diff --git a/.ast-bro-ignore b/.ast-bro-ignore new file mode 100644 index 00000000..74945dc2 --- /dev/null +++ b/.ast-bro-ignore @@ -0,0 +1,4 @@ +vendor/ +helpers/ +.agents/ +CLAUDE.md diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 00000000..80d23156 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,10 @@ +# cargo-audit configuration (CI `cargo audit` job). +# +# quick-xml < 0.41 DoS advisories. Our own untrusted-input parse paths are on +# quick-xml 0.41; the remaining old copies in Cargo.lock are pinned by the +# LATEST releases of third-party crates (xcb 0.30, docx-rs 0.36, +# rdocx-opc/oxml 0.37, citationberg-via-typst 0.38, wayland-scanner 0.39) and +# cannot be bumped without vendoring them. Mirrors the deny.toml ignore list — +# keep the two in sync. Re-check on dependency upgrades. +[advisories] +ignore = ["RUSTSEC-2026-0194", "RUSTSEC-2026-0195"] diff --git a/.cargo/config.toml b/.cargo/config.toml index beeb15f9..bdbda1c7 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -4,3 +4,8 @@ linker = "rust-lld.exe" [alias] survey = "run --profile profiler --features profiler" product = "run --profile product" + +[env] +# hotpath-meta 0.16.x looks for "hotpath-samply-meta" but the installed +# binary from hotpath 0.20.x is "hotpath-samply". Point to the correct name. +HOTPATH_META_SAMPLY_WRAPPER_BIN = "hotpath-samply" diff --git a/.claude/skills/rust-performance-optimization b/.claude/skills/rust-performance-optimization new file mode 120000 index 00000000..d7b63d82 --- /dev/null +++ b/.claude/skills/rust-performance-optimization @@ -0,0 +1 @@ +../../.agents/skills/rust-performance-optimization \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a51990f2..fcbe32e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,15 +25,27 @@ env: RUST_BACKTRACE: "1" jobs: + raw-authority-guard: + name: Raw-authority guard (Loro-first spec invariant 6) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Forbidden raw-authority surfaces must not exist in production source + run: bash tools/check_raw_authority.sh + fmt: name: fmt runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 # rustfmt config (.rustfmt.toml) uses nightly-only options. + # Nightly is date-pinned across all jobs: nightly-2026-07-13 ICEs while + # expanding test attributes (rustc_test_entrypoint_marker, see the + # 2026-07-14 build failures). Bump the pin deliberately, not implicitly. - name: Install nightly rustfmt - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@master with: + toolchain: nightly-2026-06-23 components: rustfmt # Fails the check if anything isn't formatted. The repo's pre-commit hook # runs `cargo fmt` locally, so this should normally pass on the first try. @@ -56,11 +68,14 @@ jobs: # GPUI links against Vulkan / Wayland / X11 / fontconfig and friends; # xvfb + a software Vulkan driver cover any test that builds a gpui::App. + # libdbus-1-dev: bluer (BLE collab discovery) links libdbus via + # dbus/libdbus-sys. - name: Install system dependencies run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ cmake clang pkg-config \ + libdbus-1-dev \ libfontconfig1-dev libasound2-dev \ libx11-dev libxcursor-dev libxrandr-dev libxinerama-dev libxi-dev \ libxkbcommon-dev libxkbcommon-x11-dev \ @@ -70,9 +85,11 @@ jobs: # Nightly is mandatory: the workspace uses # `cargo-features = ["profile-rustflags"]` and `-Z` flags. + # Date-pinned — see the fmt job's pin comment. - name: Install nightly toolchain - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@master with: + toolchain: nightly-2026-06-23 components: clippy - name: Cache cargo build artifacts @@ -95,6 +112,9 @@ jobs: # Runs the binaries built in "Build test binaries" (nothing to recompile); # xvfb + software Vulkan cover any test that builds a gpui::App context. + # The default (fast) suite. The multi_peer_convergence fuzzers run here + # (they converge in ~0.5s). Only the `caret_soak` reconcile-race soak is + # still `#[ignore]`d for wall-clock (~66s); run it explicitly when needed. - name: cargo test run: xvfb-run -a cargo test --workspace env: @@ -107,8 +127,11 @@ jobs: - uses: actions/checkout@v4 # cargo-deny runs `cargo metadata`, which needs nightly cargo to parse the # workspace's `cargo-features = ["profile-rustflags"]`. + # Date-pinned — see the fmt job's pin comment. - name: Install nightly toolchain - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly-2026-06-23 - name: Install cargo-deny uses: taiki-e/install-action@v2 with: diff --git a/.gitignore b/.gitignore index 7b9812c1..81641e42 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,51 @@ perf.data /VIRTUALIZED_VIEWPORT_PLAN.md flamegraph .pi/ + +# Local dev logs (flowstate::logging default dir; per-run timestamped files) +flowstate-logs/ + +# Compiled Python / caches +__pycache__/ +*.pyc + +# Local scratch documents (real .db8 fixtures live under helpers/) +/*.db8 + +# Rustc crash dumps +rustc-ice-*.txt + +# Large, regenerable ML datasets produced by tools/citation_finetune +/datasets/ + +# flowstate-autoflow: large/regenerable data (17 GB audio corpus + generated indexes) +/crates/flowstate-autoflow/audio-stash/ +/crates/flowstate-autoflow/rounds.sqlite +/crates/flowstate-autoflow/rounds.jsonl + +# Real .docx test corpus (3.3 GB, local fixture from Dropbox; gated by +# FLOWSTATE_CORPUS_DIR). NEVER track this — it bloated .git to 19 GB and was +# purged from history 2026-07-12. Regenerable from Dropbox if lost. +/helpers/corpus/ + +# Local design/spec scratch docs (kept on disk, deliberately out of git) +/flowstate_*.md +/debate-asr-production-plan.md +/lororef.md +/optimizationnotes.md + +# Profiling / probe run artifacts (regenerable captures) +/hotpath*.md +/peer1hotpath.md +/peer2hotpath.md +/*.log +/output.json +/fb_122_output.json + +# Vendored-crate build artifacts (Acts 5-14 vendoring) +vendor/*/target/ +vendor/*/*/target/ + +# Root scratch/analysis slop (keep it out of the repo) +/scratchpad_* +/citation_*.xlsx diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index b1a8e6af..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,13 +0,0 @@ -# Organization -For novel features, prefer setting them into new files. Place in the correct crates. -Avoid files over 1000 LOC. When found, suggest modularizing. - -# UI development -Always prefer using the gpui-component library components over GPUI primitives, unless it cannot be made fit. - -# Crate usage -Always consider pre-existing crates to handle operations, especially computationally heavy tasks (searching, replacement, or anything that has likely been solved externally already). Use cargo via the CLI rather than directly editing cargo files. - -# Post-edit checks -Usually avoid `cargo check`, `cargo build`, `cargo run`, or `cargo fmt`. -Main agents should run `cargo clippy` when intended edits are implemented. Fix clippy suggestions if applicable (if it is not false positive or causing regression). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..47dc3e3d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/COLLABORATION_MACOS_WINDOWS_RELEASE.md b/COLLABORATION_MACOS_WINDOWS_RELEASE.md new file mode 100644 index 00000000..3e04adeb --- /dev/null +++ b/COLLABORATION_MACOS_WINDOWS_RELEASE.md @@ -0,0 +1,200 @@ +# Collaboration release handoff: macOS and Windows + +**Status date: 2026-07-13.** This document supersedes the July-2026 draft that +previously lived at this path. That draft's "other incomplete items" list is +obsolete: the discovery coordinator, Dropbox connect flow, checkpoint +compare-and-swap, anchored comments, trust/identity settings UI, and revision +author attribution are now all implemented, wired into the application +lifecycle, and verified on Linux (unit + integration tests, clippy-clean). + +What remains is exactly two categories: + +- **Yellow zone** — code that is written and cross-compile-checked for + macOS/Windows but has never executed on those platforms. It needs a dev with + the hardware to run, verify, and fix what only a live radio/OS can reveal. +- **Red zone** — packaging, signing, entitlements, and the physical-device + acceptance matrix. Not started here by design; owned entirely by the + macOS/Windows devs. + +## Architecture you are inheriting (all verified on Linux) + +- `crates/flowstate-collab/src/discovery.rs` — signed `DiscoveryAdvertisement` + (Ed25519, domain-separated contexts, expiry, document-fingerprint scope), + `RendezvousBackend` trait, and `eligible_advertisements` — the single + verification/dedup gate every backend result must pass before UI. +- `crates/flowstate-collab/src/bluetooth.rs` — shared GATT frame codec + (u32-LE length prefix, 60 KiB cap) plus three native backends: Linux/BlueZ + (the behavioral reference), Windows/WinRT, macOS/CoreBluetooth. +- `crates/flowstate-collab/src/dropbox.rs` — PKCE OAuth, rendezvous sidecars, + revision-conditional checkpoint writes (CAS). HTTP clients have bounded + connect/read timeouts and a 256 MiB download cap. +- `crates/flowstate/src/collab/discovery_runtime.rs` — the application-level + discovery coordinator (publish on session start, 30 s renewal, 90 s ad + lifetime, scan-on-request, clear on close/shutdown). +- Comments, identity/trust settings, safety codes, and revision attribution + are application-level and platform-neutral; nothing there needs macOS- or + Windows-specific work beyond running the app. + +Security posture (for your review, not rework): discovery ads never carry +admission bearers; admission is granted server-side only to identities in the +document's standing-access set, over the encrypted iroh channel +(`net/direct.rs`); comment delete/edit are author-gated in the CRDT runtime; +the Dropbox checkpoint merge trusts the Dropbox folder ACL as its boundary +(documented at `dropbox_checkpoint.rs`); `settings.toml` (identity seed + +Dropbox tokens) is written `0600` on Unix — see the Windows note below. + +## Yellow zone: run and verify the native BLE backends + +Both backends compile cleanly today (`aarch64-apple-darwin` and +`x86_64-pc-windows-msvc`). On a real machine, plain `cargo check`/`cargo test` +works natively — no probe rig needed. Neither backend has ever executed. + +### macOS (`bluetooth.rs`, `macos_backend`) — rebuilt 2026-07-13, highest risk + +The module as originally drafted had never compiled (the +`objc2-core-bluetooth/dispatch2` feature was never enabled). It was rebuilt: + +- **Peripheral (publish) side runs on a dedicated actor thread** + (`flowstate-ble-peripheral`): CoreBluetooth objects are `!Send`, so the + backend only passes plain-data commands over a channel. All manager/service + objects live and die on that thread. +- **Both managers get dedicated serial dispatch queues** — callbacks must not + depend on the GPUI-owned main run loop (the scan side blocks its own thread). +- **`wait_for_power` distinguishes states**: fails fast on + PoweredOff/Unauthorized/Unsupported, waits up to 15 s through + Unknown/Resetting to cover the first-run permission prompt. + +Verify on hardware, in roughly this order: + +1. **Permission**: without `NSBluetoothAlwaysUsageDescription` in the bundle's + Info.plist, modern macOS kills the process on first CoreBluetooth use. This + is the first thing to wire (red zone below) before any run. +2. **Publish**: manager reaches PoweredOn on its queue; `addService` triggers + no `didAdd…error`; the ad is visible to a BLE sniffer / a Linux peer with + the Flowstate service UUID. +3. **Static characteristic long reads**: the signed record is set as the + characteristic's static value; CoreBluetooth serves offset reads itself. + Verify a Linux peer reads the full frame (records are typically 400–500 + bytes; ATT caps attribute values at 512 bytes — see the matrix item below). +4. **Scan**: 4 s scan window connects, discovers, reads, disconnects; the + delegate feeds decoded records through `eligible_advertisements`. +5. **Teardown**: Clear command and backend drop stop advertising, remove + services, and nil the delegate; repeated publish replaces cleanly (one + peripheral-manager slot). +6. **Sleep/wake and Bluetooth toggling** while a session is live. + +**Known macOS platform constraint (already handled, verify the behavior):** +CoreBluetooth cannot publish service data — macOS ads carry only the service +UUID, not the 16-byte document locator. The Linux scanner was changed +(2026-07-13) to connect-and-verify peers with a missing locator instead of +skipping them; the Windows watcher never used a locator prefilter. Verify a +macOS publisher is actually discovered by both. The post-read signed-record +check is the authoritative document scope test on every platform. + +### Windows (`bluetooth.rs`, `windows_backend`) — written by the prior agent, API-verified only + +The code compiles and its WinRT usage was reviewed; nothing has run. Verify: + +1. **Provider**: `GattServiceProvider::CreateAsync` + characteristic creation + succeed on a real adapter; `StartAdvertisingWithParameters` including the + `SetServiceData` locator (best-effort on older builds — a failure there is + deliberately ignored). +2. **Long reads**: `ReadValueWithCacheModeAsync(Uncached)` performs ATT Read + Blob continuation internally; a truncated frame fails loud in the shared + decoder. Verify against a Linux publisher with a large profile. +3. **Watcher hygiene**: repeated scans must not accumulate callbacks + (`Stop` + `RemoveReceived` run per scan; confirm with a scan loop). +4. **Apartment/threading**: WinRT activation happens from tokio worker + threads; `NativeBluetoothBackend::new` probes watcher creation so settings + gets a synchronous, actionable platform error. Verify with radio off, no + adapter, and after adapter reset. +5. **Packaged vs unpackaged**: BLE works unpackaged for development; under + MSIX the `bluetooth` device capability is required (red zone). + +### Dropbox end-to-end (all platforms; code done, no live-account run) + +The PKCE begin → system browser → `flowstate://oauth/dropbox` callback → +token exchange → persisted credentials flow, plus disconnect and the +checkpoint CAS conflict-merge path, are implemented and unit-tested. Nobody +has run them against a live Dropbox app key. On each OS verify: custom-scheme +routing to a running instance AND to a cold launch; token refresh persistence; +two clients CAS-conflicting on one bound path and converging. + +## Red zone: packaging, signing, and the acceptance matrix + +### macOS packaging and privacy + +`crates/flowstate/assets/macos/Info.plist` registers `flowstate://` but is not +a release-complete plist. + +- Add `NSBluetoothAlwaysUsageDescription` (legacy peripheral key only if the + minimum macOS requires it). Without it, CoreBluetooth use is fatal. +- Confirm hardened-runtime/App Sandbox Bluetooth entitlements for the chosen + distribution model; add only what is necessary. +- Merge URL registration + Bluetooth keys into the real release bundle; the + repository plist must not remain an unused template. +- Verify `flowstate://join` and `flowstate://oauth/dropbox` reach both a + running app and a cold launch. +- Sign, notarize, staple; test the artifact outside the build machine. + Apple Silicon and Intel if Intel remains supported. + +### Windows packaging and protocol registration + +The PowerShell helper registers associations for unpackaged dev builds only. + +- MSIX: declare the `bluetooth` device capability plus protocol/file + extensions in `Package.appxmanifest`; test activation routing through the + packaged lifecycle. +- Unpackaged installer: clean install/uninstall of per-user protocol + file + associations, safe quoting of the executable and URL argument, + upgrade/repair behavior. +- Single-instance URL activation, including with no window open. +- Sign executable and installer/MSIX; test on clean Windows 10 and 11 without + Visual Studio or developer mode. +- **Secrets at rest**: the Unix `0600` on `settings.toml` does not apply on + Windows. Decide on DPAPI (and Keychain on macOS) for the identity signing + seed and Dropbox tokens, or accept per-user-profile ACLs explicitly. + +### Physical-device acceptance matrix + +Do not enable the nearby-discovery toggle by default on either platform until +all of these pass with release artifacts and physical devices: + +- macOS publishes and Windows discovers; Windows publishes and macOS + discovers; each platform interoperates with the Linux/BlueZ reference. +- A macOS publisher (no service-data locator) is discovered by Linux and + Windows scanners; wrong-document locators are skipped pre-connect where a + locator exists, and every record is re-checked post-read. +- Records larger than one ATT payload are reassembled; records approaching + the 512-byte ATT attribute cap are measured. If a real profile (long + display name, many relay addresses) pushes a frame past 512 bytes, GATT + reads will truncate at the OS level on some stacks — the decoder fails + loudly, but delivery needs a design change (trim the endpoint address list, + or move to an L2CAP channel). Measure before shipping. +- Tampered signatures, expired records, oversized frames, invalid offsets, + and trailing bytes are rejected (the shared decoder tests cover the codec; + verify the OS paths deliver bytes unmodified). +- Two Flowstate instances publishing/scanning concurrently where the OS + permits, with a clear degraded state otherwise. +- Permission denied, radio disabled, no adapter, adapter reset, sleep/wake, + suspend/resume, and document/session closure leak no tasks, handlers, + services, or advertisements. +- Discovery never displays an unverified or out-of-scope identity and never + transports a session admission bearer. +- Battery and scan/connect rates acceptable during a long editing session + (current cadence: one BLE publish round-robin + Dropbox refresh per 30 s + tick, 4 s scan windows on demand). +- Accessibility text explains why Bluetooth is requested; discovery can stay + disabled without impairing invite links. + +### Deliberate design decisions (do not re-litigate without cause) + +- Dropbox folder membership **is** the checkpoint trust boundary; checkpoint + packages are not additionally signed. +- Discovery admission requests carry a nonce but are not replay-tracked: + a 30 s expiry, the encrypted direct channel, and the standing-access + identity gate bound the exposure. +- Trust attestations have no expiry; revocation is removing the trusted key + in settings. Key-change alerting is future UX, not release-blocking. +- Background (app-closed) BLE discovery stays out of scope until it has a + separately reviewed privacy/battery design. diff --git a/Cargo.lock b/Cargo.lock index 311e03d5..5627a842 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,31 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + [[package]] name = "aes" version = "0.8.4" @@ -31,6 +50,20 @@ dependencies = [ "cpufeatures 0.3.0", ] +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes 0.8.4", + "cipher 0.4.4", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.7.8" @@ -150,9 +183,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "append-only-bytes" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "ac436601d6bdde674a0d7fb593e829ffe7b3387c351b356dd20e2d40f5bf3ee5" [[package]] name = "approx" @@ -177,6 +216,9 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arc-swap" @@ -210,6 +252,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ccd462b64c3c72f1be8305905a85d85403d768e8690c9b8bd3b9009a5761679" + [[package]] name = "as-raw-xcb-connection" version = "1.0.1" @@ -508,6 +556,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "async_io_stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" +dependencies = [ + "futures", + "pharos", + "rustc_version", +] + [[package]] name = "async_zip" version = "0.0.17" @@ -542,6 +601,29 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomicwrites" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef1bb8d1b645fe38d51dfc331d720fb5fc2c94b440c76cc79c80ff265ca33e3" +dependencies = [ + "rustix 0.38.44", + "tempfile", + "windows-sys 0.52.0", +] + +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64", + "http", + "log", + "url", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -591,12 +673,66 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "az" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be5eb007b7cacc6c660343e96f650fedf4b5a77512399eb952ca6642cf8d13f7" +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand 2.4.1", + "gloo-timers", + "tokio", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object 0.37.3", + "rustc-demangle", + "windows-link 0.2.1", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base62" version = "2.2.4" @@ -609,6 +745,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "beef" version = "0.5.2" @@ -694,6 +836,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + [[package]] name = "bitpacking" version = "0.9.3" @@ -794,6 +945,7 @@ dependencies = [ "cfg-if", "constant_time_eq", "cpufeatures 0.3.0", + "rayon-core", ] [[package]] @@ -830,6 +982,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block2" version = "0.6.2" @@ -852,6 +1013,35 @@ dependencies = [ "piper", ] +[[package]] +name = "bluer" +version = "0.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af68112f5c60196495c8b0eea68349817855f565df5b04b2477916d09fb1a901" +dependencies = [ + "custom_debug", + "dbus", + "dbus-crossroads", + "dbus-tokio", + "displaydoc", + "futures", + "hex", + "lazy_static", + "libc", + "log", + "macaddr", + "nix 0.29.0", + "num-derive", + "num-traits", + "pin-project", + "serde", + "serde_json", + "strum 0.26.3", + "tokio", + "tokio-stream", + "uuid", +] + [[package]] name = "bon" version = "3.9.1" @@ -868,7 +1058,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -927,12 +1117,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - [[package]] name = "bytemuck" version = "1.25.0" @@ -970,6 +1154,9 @@ name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] [[package]] name = "bzip2" @@ -1015,6 +1202,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "cbindgen" version = "0.28.0" @@ -1119,6 +1315,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link 0.2.1", ] @@ -1238,6 +1435,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -1342,6 +1548,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comemo" version = "0.5.1" @@ -1441,6 +1657,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cordyceps" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a" +dependencies = [ + "loom", + "tracing", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1674,9 +1909,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1759,6 +1994,15 @@ version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2" +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1769,83 +2013,315 @@ dependencies = [ ] [[package]] -name = "darling" -version = "0.23.0" +name = "curve25519-dalek" +version = "5.0.0-pre.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "335f1947f241137a14106b6f5acc5918a5ede29c9d71d3f2cb1678d5075d9fc3" dependencies = [ - "darling_core", - "darling_macro", + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.11.3", + "fiat-crypto", + "rand_core 0.10.1", + "rustc_version", + "serde", + "subtle", + "zeroize", ] [[package]] -name = "darling_core" -version = "0.23.0" +name = "curve25519-dalek-derive" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ - "ident_case", "proc-macro2", "quote", - "strsim", "syn 2.0.117", ] [[package]] -name = "darling_macro" -version = "0.23.0" +name = "custom_debug" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2da7d1ad9567b3e11e877f1d7a0fa0360f04162f94965fc4448fbed41a65298e" dependencies = [ - "darling_core", - "quote", - "syn 2.0.117", + "custom_debug_derive", ] [[package]] -name = "data-url" -version = "0.3.2" +name = "custom_debug_derive" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" +checksum = "a707ceda8652f6c7624f2be725652e9524c815bf3b9d55a0b2320be2303f9c11" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] [[package]] -name = "datasketches" -version = "0.2.0" +name = "darling" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] [[package]] -name = "deflate64" -version = "0.1.12" +name = "darling" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] [[package]] -name = "deranged" -version = "0.5.8" +name = "darling_core" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ - "powerfmt", - "serde_core", + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] -name = "derive_more" -version = "0.99.20" +name = "darling_core" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "convert_case", + "ident_case", "proc-macro2", "quote", - "rustc_version", + "strsim", "syn 2.0.117", ] [[package]] -name = "digest" -version = "0.10.7" +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn 1.0.109", +] + +[[package]] +name = "data-url" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1e0bca6c3637f992fc1cc7cbc52a78c1ef6db076dbf1059c4323d6a2048376" + +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + +[[package]] +name = "dbus" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab69f03cc8c4340c9c8e315114e1658e6775a9b16a04357973aa21cec22b32e" +dependencies = [ + "futures-channel", + "futures-util", + "libc", + "libdbus-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "dbus-crossroads" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64bff0bd181fba667660276c6b7ebdc50cff37ce593e7adf9e734f89c8f444e8" +dependencies = [ + "dbus", +] + +[[package]] +name = "dbus-tokio" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "007688d459bc677131c063a3a77fb899526e17b7980f390b69644bdbc41fad13" +dependencies = [ + "dbus", + "libc", + "tokio", +] + +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "convert_case 0.4.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.117", + "unicode-xid", +] + +[[package]] +name = "diatomic-waker" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c" + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ @@ -1963,6 +2439,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.13.0", + "block2", + "libc", "objc2", ] @@ -1986,6 +2464,17 @@ dependencies = [ "libloading", ] +[[package]] +name = "dlopen2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2c5bd4158e66d1e215c49b837e11d62f3267b30c92f1d171c4d3105e3dc4d4" +dependencies = [ + "libc", + "once_cell", + "winapi", +] + [[package]] name = "docx-rs" version = "0.4.20" @@ -2081,11 +2570,11 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecb" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +checksum = "fbfbf3db731928d6912bc3beda911d55b834cee3df6131ba79b337a1298a3fa9" dependencies = [ - "cipher 0.4.4", + "cipher 0.5.2", ] [[package]] @@ -2097,6 +2586,33 @@ dependencies = [ "serde", ] +[[package]] +name = "ed25519" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" +dependencies = [ + "pkcs8", + "serdect", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "3.0.0-pre.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20449acd54b660981ae5caa2bcb56d1fe7f25f2e37a38ec507400fab034d4bb6" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.10.1", + "serde", + "sha2 0.11.0", + "signature", + "subtle", + "zeroize", +] + [[package]] name = "either" version = "1.16.0" @@ -2159,6 +2675,47 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66b7e2430c6dff6a955451e2cfc438f09cea1965a9d6f87f7e3b90decc014099" +[[package]] +name = "ensure-cov" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33753185802e107b8fa907192af1f0eca13b1fb33327a59266d650fef29b2b4e" + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "enum-assoc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed8956bd5c1f0415200516e78ff07ec9e16415ade83c056c230d7b7ea0d55b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "enum-iterator" version = "2.3.0" @@ -2199,6 +2756,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "enumflags2" version = "0.7.12" @@ -2460,6 +3029,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + [[package]] name = "file-id" version = "0.2.3" @@ -2496,6 +3071,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -2530,37 +3111,129 @@ name = "flowstate" version = "0.1.0" dependencies = [ "anyhow", + "async-channel 2.5.0", + "atomicwrites", + "blake3", + "chrono", "clap", "cosmic-text", "dirs 6.0.0", "fff-search", + "flowstate-collab", "flowstate-document", "flowstate-docx", + "flowstate-fidelity", "flowstate-flow", "flowstate-tub", + "futures-util", "gpui", "gpui-component", "gpui-component-assets", "gpui-symbols", "hotpath", + "iroh", + "loro", "memchr", "mimalloc", + "notify 8.2.0", "rustc-hash 2.1.2", "serde", "serde_json", "tempfile", "toml 1.1.2+spec-1.1.0", + "tracing", + "tracing-appender", + "tracing-subscriber", + "url", "uuid", ] [[package]] -name = "flowstate-document" +name = "flowstate-collab" version = "0.1.0" dependencies = [ + "anyhow", + "async-channel 2.5.0", + "base64", + "blake3", + "bluer", + "dispatch2", + "flowstate-document", + "flowstate-fidelity", + "futures-util", "gpui", "gpui-flowtext", - "rustc-hash 2.1.2", -] + "hotpath", + "iroh", + "iroh-gossip", + "iroh-tickets", + "loro", + "n0-future", + "objc2", + "objc2-core-bluetooth", + "objc2-foundation", + "pollster 0.4.0", + "postcard", + "proptest", + "rand 0.9.4", + "reqwest", + "rustc-hash 2.1.2", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "twox-hash", + "url", + "uuid", + "windows 0.62.2", + "zstd", +] + +[[package]] +name = "flowstate-corpus" +version = "0.1.0" +dependencies = [ + "anyhow", + "flowstate-collab", + "flowstate-document", + "flowstate-docx", + "gpui", + "gpui-component", + "gpui-flowtext", + "hotpath", + "ignore", + "image", + "loro", + "mimalloc", + "serde", + "serde_json", + "tempfile", + "tracing-subscriber", +] + +[[package]] +name = "flowstate-document" +version = "0.1.0" +dependencies = [ + "atomicwrites", + "blake3", + "flowstate-fidelity", + "gpui", + "gpui-flowtext", + "hotpath", + "imagesize", + "loro", + "loro-internal", + "postcard", + "rustc-hash 2.1.2", + "serde", + "serde_bytes", + "tempfile", + "uuid", +] [[package]] name = "flowstate-docx" @@ -2569,10 +3242,15 @@ dependencies = [ "docx-rs", "docxide-pdf", "flowstate-document", + "flowstate-fidelity", "gpui", "hotpath", + "image", + "imagesize", "lopdf", - "quick-xml 0.36.2", + "memchr", + "quick-xml 0.37.5", + "quick-xml 0.41.0", "rdocx", "rdocx-opc", "rdocx-oxml", @@ -2581,6 +3259,13 @@ dependencies = [ "zstd", ] +[[package]] +name = "flowstate-fidelity" +version = "0.1.0" +dependencies = [ + "tracing", +] + [[package]] name = "flowstate-flow" version = "0.1.0" @@ -2590,9 +3275,24 @@ dependencies = [ "rustc-hash 2.1.2", "serde", "serde_json", + "tracing", "uuid", ] +[[package]] +name = "flowstate-soak" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "flowstate-collab", + "flowstate-document", + "flowstate-docx", + "hotpath", + "loro", + "postcard", +] + [[package]] name = "flowstate-tub" version = "0.1.0" @@ -2624,7 +3324,7 @@ dependencies = [ "futures-core", "futures-sink", "nanorand", - "spin", + "spin 0.9.8", ] [[package]] @@ -2757,6 +3457,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -2797,6 +3503,19 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-buffered" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4421cb78ee172b6b06080093479d3c50f058e7c81b7d577bbb8d118d551d4cd5" +dependencies = [ + "cordyceps", + "diatomic-waker", + "futures-core", + "pin-project-lite", + "spin 0.10.0", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -2807,6 +3526,19 @@ dependencies = [ "futures-sink", ] +[[package]] +name = "futures-concurrency" +version = "7.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175cd8cca9e1d45b87f18ffa75088f2099e3c4fe5e2f83e42de112560bea8ea6" +dependencies = [ + "fixedbitset", + "futures-core", + "futures-lite 2.6.1", + "pin-project", + "smallvec", +] + [[package]] name = "futures-core" version = "0.3.32" @@ -2907,6 +3639,21 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link 0.2.1", + "windows-result 0.4.1", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2917,6 +3664,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "generic-btree" +version = "0.10.7" +dependencies = [ + "arref", + "heapless 0.9.3", + "itertools 0.11.0", + "loro-thunderdome", + "proc-macro2", + "rustc-hash 2.1.2", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -2947,11 +3706,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -2970,6 +3727,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gif" version = "0.13.3" @@ -2977,7 +3744,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" dependencies = [ "color_quant", - "weezl", + "weezl 0.1.12", ] [[package]] @@ -2987,9 +3754,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" dependencies = [ "color_quant", - "weezl", + "weezl 0.1.12", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "git2" version = "0.20.4" @@ -3103,6 +3876,7 @@ dependencies = [ "as-raw-xcb-connection", "ashpd 0.11.1", "async-task", + "backtrace", "bindgen", "blade-graphics", "blade-macros", @@ -3121,7 +3895,7 @@ dependencies = [ "core-video", "cosmic-text", "ctor", - "derive_more", + "derive_more 0.99.20", "embed-resource", "etagere", "filedescriptor", @@ -3179,7 +3953,7 @@ dependencies = [ "wayland-protocols-plasma", "windows 0.61.3", "windows-core 0.61.2", - "windows-numerics", + "windows-numerics 0.2.0", "windows-registry 0.5.3", "x11-clipboard", "x11rb", @@ -3253,19 +4027,25 @@ dependencies = [ name = "gpui-flowtext" version = "0.1.0" dependencies = [ + "blake3", "cosmic-text", "crop", + "flowstate-collab", + "flowstate-document", + "flowstate-fidelity", "gpui", "gpui-component", "hotpath", + "im", "imagesize", + "loro", "mitex", - "postcard", "resvg", "rustc-hash 2.1.2", "serde", "serde_json", "tempfile", + "tracing", "typst", "typst-assets", "typst-library", @@ -3330,7 +4110,7 @@ dependencies = [ "async-compression", "async-fs", "bytes", - "derive_more", + "derive_more 0.99.20", "futures", "gpui_util", "http", @@ -3417,12 +4197,15 @@ dependencies = [ "dunce", "futures", "futures-lite 1.13.0", + "git2", "globset", "gpui_collections", + "gpui_util_macros", "itertools 0.14.0", "libc", "log", "nix 0.29.0", + "rand 0.9.4", "regex", "rust-embed", "schemars", @@ -3496,6 +4279,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -3537,6 +4329,8 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -3657,10 +4451,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" dependencies = [ "atomic-polyfill", - "hash32", + "hash32 0.2.1", "rustc_version", "serde", - "spin", + "spin 0.9.8", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32 0.3.1", "stable_deref_trait", ] @@ -3732,6 +4546,83 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "bytes", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "h2", + "hickory-proto", + "http", + "idna", + "ipnet", + "jni", + "rand 0.10.1", + "rustls", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tokio-rustls", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.1", + "resolv-conf", + "rustls", + "smallvec", + "system-configuration 0.7.0", + "thiserror 2.0.18", + "tokio", + "tokio-rustls", + "tracing", +] + [[package]] name = "hidden-trait" version = "0.1.2" @@ -3920,6 +4811,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -3949,13 +4841,16 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", "futures-channel", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -4235,6 +5130,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "identity-hash" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfdd7caa900436d8f13b2346fe10257e0c05c1f1f9e351f4f5d57c03bd5f45da" + [[package]] name = "idna" version = "1.1.0" @@ -4256,6 +5157,26 @@ dependencies = [ "icu_properties 2.2.0", ] +[[package]] +name = "igd-next" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de7238d487a9aff61f81b5ab41c0a841532a115a398b5fa92a2fadd0885e2581" +dependencies = [ + "attohttpc", + "bytes", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.10.1", + "tokio", + "url", + "xmltree", +] + [[package]] name = "ignore" version = "0.4.26" @@ -4272,6 +5193,21 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core 0.6.4", + "rand_xoshiro", + "serde", + "sized-chunks", + "typenum", + "version_check", +] + [[package]] name = "image" version = "0.25.10" @@ -4303,7 +5239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" dependencies = [ "byteorder-lite", - "quick-error", + "quick-error 2.0.1", ] [[package]] @@ -4367,7 +5303,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", + "block-padding 0.3.3", "generic-array", ] @@ -4377,6 +5313,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ + "block-padding 0.4.2", "hybrid-array", ] @@ -4422,12 +5359,270 @@ dependencies = [ ] [[package]] -name = "ipnet" -version = "2.12.0" +name = "ipconfig" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - -[[package]] +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry 0.6.1", + "windows-result 0.4.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] + +[[package]] +name = "iroh" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef865dc2d11a19fe670ff217b68ffc3b511bddf473dc3a3e120090b9f691803" +dependencies = [ + "backon", + "blake3", + "bytes", + "cfg_aliases", + "ctutils", + "data-encoding", + "derive_more 2.1.1", + "ed25519-dalek", + "futures-util", + "getrandom 0.4.2", + "hickory-resolver", + "http", + "ipnet", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "iroh-relay", + "n0-error", + "n0-future", + "n0-watcher", + "netwatch", + "noq", + "noq-proto", + "noq-udp", + "papaya", + "pin-project", + "portable-atomic", + "portmapper", + "rand 0.10.1", + "reqwest", + "rustc-hash 2.1.2", + "rustls", + "rustls-pki-types", + "rustls-webpki", + "serde", + "smallvec", + "strum 0.28.0", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "wasm-bindgen-futures", + "webpki-roots", +] + +[[package]] +name = "iroh-base" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af93d67701c00c504982154569192ad384738c0450ba1196930314b955100552" +dependencies = [ + "curve25519-dalek", + "data-encoding", + "data-encoding-macro", + "derive_more 2.1.1", + "digest 0.11.3", + "ed25519-dalek", + "getrandom 0.4.2", + "n0-error", + "rand 0.10.1", + "serde", + "sha2 0.11.0", + "url", + "zeroize", + "zeroize_derive", +] + +[[package]] +name = "iroh-dns" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de4112c91eb64094d77df9d3112606dcf7ff216421afccd2dc762fda5a7b2879" +dependencies = [ + "arc-swap", + "cfg_aliases", + "derive_more 2.1.1", + "hickory-resolver", + "iroh-base", + "n0-error", + "n0-future", + "ndk-context", + "rand 0.10.1", + "reqwest", + "rustls", + "simple-dns", + "strum 0.28.0", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "iroh-gossip" +version = "0.100.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5cada641f84a2ff3d1acbd43e16490cf3b2a4f675e580c5c2caa5d68ee63b5" +dependencies = [ + "blake3", + "bytes", + "constant_time_eq", + "data-encoding", + "derive_more 2.1.1", + "ed25519-dalek", + "futures-concurrency", + "hex", + "indexmap", + "iroh", + "iroh-base", + "iroh-metrics", + "irpc", + "n0-error", + "n0-future", + "postcard", + "rand 0.10.1", + "serde", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "iroh-metrics" +version = "1.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d102597d0ee523f17fdb672c532395e634dbe945429284c811430d63bacc0d8a" +dependencies = [ + "iroh-metrics-derive", + "itoa", + "n0-error", + "portable-atomic", + "ryu", + "serde", + "tracing", +] + +[[package]] +name = "iroh-metrics-derive" +version = "1.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c8e0c97f1dc787107f388433c349397c565572fe6406d600ff7bb7b7fe3b30" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "iroh-relay" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a70030b9e71c1183bd4f88fbdbebfa1af2a5be549dd6f20a1e8ac3cd0202ee9d" +dependencies = [ + "blake3", + "bytes", + "cfg_aliases", + "data-encoding", + "derive_more 2.1.1", + "getrandom 0.4.2", + "hickory-resolver", + "http", + "http-body-util", + "hyper", + "hyper-util", + "iroh-base", + "iroh-dns", + "iroh-metrics", + "lru 0.18.0", + "n0-error", + "n0-future", + "noq", + "noq-proto", + "num_enum", + "pin-project", + "postcard", + "rand 0.10.1", + "reqwest", + "rustls", + "rustls-pki-types", + "serde", + "serde_bytes", + "strum 0.28.0", + "tokio", + "tokio-rustls", + "tokio-util", + "tokio-websockets", + "tracing", + "url", + "vergen-gitcl", + "webpki-roots", + "ws_stream_wasm", +] + +[[package]] +name = "iroh-tickets" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d4051ac9825cd834d88377dab71c1d9d483b65e9740029b2d624cc17ac58d" +dependencies = [ + "data-encoding", + "derive_more 2.1.1", + "iroh-base", + "n0-error", + "postcard", + "serde", +] + +[[package]] +name = "irpc" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05799eb70acd04843c327ef939233ccf80f607d30e9ca94857ac7f3d8f18b46" +dependencies = [ + "futures-util", + "irpc-derive", + "n0-error", + "n0-future", + "serde", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "irpc-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445d81dbc1eed4dab6379bf7f97d12ac28ce8e6f3f7d6660c9f333b7b5d8d03b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] name = "is-docker" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4472,6 +5667,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -4537,6 +5741,55 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -4549,13 +5802,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -4647,7 +5899,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -4665,6 +5917,12 @@ dependencies = [ "leak", ] +[[package]] +name = "leb128" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -4695,6 +5953,15 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libdbus-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328c4789d42200f1eeec05bd86c9c13c7f091d2ba9a6ea35acdf51f31bc0f043" +dependencies = [ + "pkg-config", +] + [[package]] name = "libfuzzer-sys" version = "0.4.13" @@ -4879,6 +6146,21 @@ dependencies = [ "logos-codegen", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + [[package]] name = "loop9" version = "0.1.5" @@ -4890,13 +6172,13 @@ dependencies = [ [[package]] name = "lopdf" -version = "0.40.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fdcbab5b237a03984f83b1394dc534e0b1960675c7f3ec4d04dccc9032b56d" +checksum = "5e2ec995d822e05cabc3f06d196ee43650af3fe4fe38012cacb35e0c3d113b68" dependencies = [ - "aes 0.8.4", + "aes 0.9.1", "bitflags 2.13.0", - "cbc", + "cbc 0.2.1", "chrono", "ecb", "encoding_rs", @@ -4906,18 +6188,153 @@ dependencies = [ "itoa", "jiff", "log", - "md-5", + "md-5 0.11.0", "nom 8.0.0", - "nom_locate", "rand 0.10.1", "rangemap", "rayon", - "sha2 0.10.9", + "sha2 0.11.0", "stringprep", "thiserror 2.0.18", "time", - "ttf-parser 0.25.1", - "weezl", + "weezl 0.2.1", +] + +[[package]] +name = "loro" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd1b63cd2f0cf66acbbb3191c41feaff03c94c3f6c6bb1e61d2ab3062c301c7" +dependencies = [ + "enum-as-inner 0.6.1", + "generic-btree", + "loro-common", + "loro-delta", + "loro-internal", + "loro-kv-store", + "rustc-hash 2.1.2", + "tracing", +] + +[[package]] +name = "loro-common" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b51042936156a1d537df8a38938ec8ed24b45de14530af933c1d02de39c9e056" +dependencies = [ + "arbitrary", + "enum-as-inner 0.6.1", + "leb128", + "loro-rle", + "nonmax", + "rustc-hash 2.1.2", + "serde", + "serde_columnar", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "loro-delta" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f381e7fb5d4bf5605b4f4a9241f3adf61403ded978d757552004c8302cccbb" +dependencies = [ + "arrayvec", + "enum-as-inner 0.5.1", + "generic-btree", + "heapless 0.8.0", +] + +[[package]] +name = "loro-internal" +version = "1.13.1" +dependencies = [ + "append-only-bytes", + "arref", + "bytes", + "either", + "ensure-cov", + "enum-as-inner 0.6.1", + "enum_dispatch", + "generic-btree", + "getrandom 0.2.17", + "hotpath", + "im", + "itertools 0.12.1", + "leb128", + "loom", + "loro-common", + "loro-delta", + "loro-kv-store", + "loro-rle", + "loro_fractional_index", + "md5", + "nonmax", + "num", + "num-traits", + "once_cell", + "parking_lot", + "pest", + "pest_derive", + "postcard", + "pretty_assertions", + "rand 0.8.6", + "rustc-hash 2.1.2", + "serde", + "serde_columnar", + "serde_json", + "smallvec", + "thiserror 1.0.69", + "thread_local", + "tracing", + "wasm-bindgen", + "xxhash-rust", +] + +[[package]] +name = "loro-kv-store" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950cc8bcc64dcff536e949fbc56ccc3d0759646fa441c7f76b3cd7c3eafa2096" +dependencies = [ + "bytes", + "ensure-cov", + "loro-common", + "lz4_flex 0.11.6", + "once_cell", + "quick_cache", + "rustc-hash 2.1.2", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "loro-rle" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76400c3eea6bb39b013406acce964a8db39311534e308286c8d8721baba8ee20" +dependencies = [ + "append-only-bytes", + "num", + "smallvec", +] + +[[package]] +name = "loro-thunderdome" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" + +[[package]] +name = "loro_fractional_index" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aca7180674d0273ddf37049a5efcde4547fd5330d24abb7519bb9d9eb6780d5b" +dependencies = [ + "once_cell", + "rand 0.8.6", + "serde", ] [[package]] @@ -4929,6 +6346,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -5000,6 +6426,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + [[package]] name = "lz4_flex" version = "0.13.1" @@ -5008,11 +6443,11 @@ checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" [[package]] name = "lzma-rust2" -version = "0.16.4" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce716bf1a316f47a280fc76295f6495b5bea4752bca01c3b3885e101b1c23c02" +checksum = "47bb1e988e6fb779cf720ad431242d3f03167c1b3f2b1aae7f1a94b2495b36ae" dependencies = [ - "sha2 0.11.0", + "sha2 0.10.9", ] [[package]] @@ -5021,6 +6456,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mac-addr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d25b0e0b648a86960ac23b7ad4abb9717601dec6f66c165f5b037f3f03065f" + +[[package]] +name = "macaddr" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -5094,6 +6541,22 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "measure_time" version = "0.9.0" @@ -5105,9 +6568,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.1" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -5270,6 +6733,23 @@ dependencies = [ "which 5.0.0", ] +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "moxcms" version = "0.7.11" @@ -5302,6 +6782,60 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" +[[package]] +name = "n0-error" +version = "1.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223e946a84aa91644507a6b7865cfebbb9a231ace499041c747ab0fd30408212" +dependencies = [ + "anyhow", + "n0-error-macros", + "spez", +] + +[[package]] +name = "n0-error-macros" +version = "1.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "565305a21e6b3bf26640ad98f05a0fda12d3ab4315394566b52a7bddb8b34828" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "n0-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ab99dfb861450e68853d34ae665243a88b8c493d01ba957321a1e9b2312bbe" +dependencies = [ + "cfg_aliases", + "derive_more 2.1.1", + "futures-buffered", + "futures-lite 2.6.1", + "futures-util", + "js-sys", + "pin-project", + "send_wrapper", + "tokio", + "tokio-util", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "n0-watcher" +version = "1.0.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928d8039a66cce5efcfd35e88b32d3defc8eba630b3ac451522997f563956a52" +dependencies = [ + "derive_more 2.1.1", + "n0-error", + "n0-future", +] + [[package]] name = "naga" version = "25.0.1" @@ -5336,6 +6870,12 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "neo_frizbee" version = "0.10.2" @@ -5346,6 +6886,126 @@ dependencies = [ "raw-cpuid", ] +[[package]] +name = "netdev" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57bacaf873ee4eab5646f99b381b271ec75e716902a67cf962c0f328c5eb5bfb" +dependencies = [ + "block2", + "dispatch2", + "dlopen2", + "ipnet", + "libc", + "mac-addr", + "netlink-packet-core", + "netlink-packet-route 0.29.0", + "netlink-sys", + "objc2-core-foundation", + "objc2-core-wlan", + "objc2-foundation", + "objc2-system-configuration", + "once_cell", + "plist", + "windows-sys 0.61.2", +] + +[[package]] +name = "netlink-packet-core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" +dependencies = [ + "paste", +] + +[[package]] +name = "netlink-packet-route" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9854ea6ad14e3f4698a7f03b65bce0833dd2d81d594a0e4a984170537146b6" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core", +] + +[[package]] +name = "netlink-packet-route" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8919612f6028ab4eacbbfe1234a9a43e3722c6e0915e7ff519066991905092" +dependencies = [ + "bitflags 2.13.0", + "libc", + "log", + "netlink-packet-core", +] + +[[package]] +name = "netlink-proto" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65d130ee111430e47eed7896ea43ca693c387f097dd97376bffafbf25812128" +dependencies = [ + "bytes", + "futures", + "log", + "netlink-packet-core", + "netlink-sys", + "thiserror 2.0.18", +] + +[[package]] +name = "netlink-sys" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" +dependencies = [ + "bytes", + "futures-util", + "libc", + "log", + "tokio", +] + +[[package]] +name = "netwatch" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2071e0c2b5b229622c459096b84f1ad51afa150cdeeefdad491ef3704e581d91" +dependencies = [ + "atomic-waker", + "bytes", + "cfg_aliases", + "derive_more 2.1.1", + "js-sys", + "libc", + "n0-error", + "n0-future", + "n0-watcher", + "netdev", + "netlink-packet-core", + "netlink-packet-route 0.30.0", + "netlink-proto", + "netlink-sys", + "noq-udp", + "objc2-core-foundation", + "objc2-system-configuration", + "pin-project-lite", + "serde", + "socket2", + "time", + "tokio", + "tokio-util", + "tracing", + "web-sys", + "windows 0.62.2", + "windows-result 0.4.1", + "wmi", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -5405,15 +7065,10 @@ dependencies = [ ] [[package]] -name = "nom_locate" -version = "5.0.0" +name = "nonmax" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" -dependencies = [ - "bytecount", - "memchr", - "nom 8.0.0", -] +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" [[package]] name = "noop_proc_macro" @@ -5421,6 +7076,68 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" +[[package]] +name = "noq" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198b99fc085a5db1f7d259edb5ede8311e59f28cdd2687920b4313613d21a73f" +dependencies = [ + "bytes", + "cfg_aliases", + "derive_more 2.1.1", + "noq-proto", + "noq-udp", + "pin-project-lite", + "rustc-hash 2.1.2", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "web-time", +] + +[[package]] +name = "noq-proto" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab0ac774795ce1e42a7e61266e71f3be8110210630441169ac8dda403dd23f1" +dependencies = [ + "aes-gcm", + "bytes", + "derive_more 2.1.1", + "enum-assoc", + "getrandom 0.4.2", + "identity-hash", + "lru-slab", + "rand 0.10.1", + "rand_pcg", + "ring", + "rustc-hash 2.1.2", + "rustls", + "rustls-pki-types", + "slab", + "sorted-index-buffer", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "noq-udp" +version = "1.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3c1520eacd33fd6b009e2e70116b05508ade51db5e0d315ff8bf6b702148c2b" +dependencies = [ + "cfg_aliases", + "libc", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "normpath" version = "1.5.1" @@ -5641,6 +7358,37 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "objc" version = "0.2.7" @@ -5684,6 +7432,19 @@ dependencies = [ "objc2-quartz-core", ] +[[package]] +name = "objc2-core-bluetooth" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b30b9eacc37434a61377866f68b27acef9fa5496f25cc9ca8b2549032e0394" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -5691,7 +7452,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.0", + "block2", "dispatch2", + "libc", "objc2", ] @@ -5705,6 +7468,20 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-core-wlan" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e34919aba0d701380d911702455038a8a3587467fe0141d6a71501e7ffe48" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-security", + "objc2-security-foundation", +] + [[package]] name = "objc2-encode" version = "4.1.0" @@ -5718,6 +7495,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.13.0", + "block2", + "libc", "objc2", "objc2-core-foundation", ] @@ -5747,6 +7526,41 @@ dependencies = [ "objc2-metal", ] +[[package]] +name = "objc2-security" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" +dependencies = [ + "bitflags 2.13.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-security-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef76382e9cedd18123099f17638715cc3d81dba3637d4c0d39ab69df2ef345a5" +dependencies = [ + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-security", +] + [[package]] name = "objc2-ui-kit" version = "0.3.2" @@ -5801,6 +7615,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -5826,7 +7644,7 @@ dependencies = [ "async-io", "async-lock", "blocking", - "cbc", + "cbc 0.1.2", "cipher 0.4.4", "digest 0.10.7", "endi", @@ -5835,7 +7653,7 @@ dependencies = [ "getrandom 0.3.4", "hkdf", "hmac 0.12.1", - "md-5", + "md-5 0.10.6", "num", "num-bigint-dig", "pbkdf2 0.12.2", @@ -5849,6 +7667,12 @@ dependencies = [ "zvariant", ] +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "open" version = "5.3.5" @@ -5934,6 +7758,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "papaya" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997ee03cd38c01469a7046643714f0ad28880bcb9e6679ff0666e24817ca19b7" +dependencies = [ + "equivalent", + "seize", +] + [[package]] name = "parking" version = "2.2.1" @@ -6032,12 +7866,74 @@ dependencies = [ "ryu", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +dependencies = [ + "pest", + "sha2 0.10.9", +] + +[[package]] +name = "pharos" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" +dependencies = [ + "futures", + "rustc_version", +] + [[package]] name = "phf" version = "0.11.3" @@ -6182,6 +8078,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -6190,13 +8096,13 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plist" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" +checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" dependencies = [ "base64", "indexmap", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "serde", "time", ] @@ -6247,6 +8153,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5da3b0203fd7ee5720aa0b5e790b591aa5d3f41c3ed2c34a3a393382198af2f7" +[[package]] +name = "pollster" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3a9f18d041e6d0e102a0a46750538147e5e8992d3b4873aaafee2520b00ce3" + [[package]] name = "polycool" version = "0.4.0" @@ -6256,11 +8168,26 @@ dependencies = [ "arrayvec", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +dependencies = [ + "serde", +] [[package]] name = "portable-atomic-util" @@ -6271,6 +8198,35 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "portmapper" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64959cbabf952c8ffcbaea13745308508f1f825922f4068353f3de08d42cf214" +dependencies = [ + "base64", + "bytes", + "derive_more 2.1.1", + "hyper-util", + "igd-next", + "iroh-metrics", + "libc", + "n0-error", + "n0-future", + "netwatch", + "num_enum", + "rand 0.10.1", + "serde", + "smallvec", + "socket2", + "time", + "tokio", + "tokio-util", + "tower-layer", + "tracing", + "url", +] + [[package]] name = "postage" version = "0.5.0" @@ -6283,7 +8239,7 @@ dependencies = [ "log", "parking_lot", "pin-project", - "pollster", + "pollster 0.2.5", "static_assertions", "thiserror 1.0.69", ] @@ -6297,10 +8253,22 @@ dependencies = [ "cobs", "embedded-io 0.4.0", "embedded-io 0.6.1", - "heapless", + "heapless 0.7.17", + "postcard-derive", "serde", ] +[[package]] +name = "postcard-derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0232bd009a197ceec9cc881ba46f727fcd8060a2d8d6a9dde7a69030a6fe2bb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -6335,7 +8303,28 @@ dependencies = [ name = "precomputed-hash" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] [[package]] name = "prettyplease" @@ -6425,6 +8414,25 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.13.0", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax 0.8.10", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "psm" version = "0.1.31" @@ -6491,6 +8499,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-error" version = "2.0.1" @@ -6544,6 +8558,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick_cache" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3db184a8b66cfe87f0263a1de147a6b554c864d1767c6f7fa4eb0e5497b565" +dependencies = [ + "ahash 0.8.12", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6566,14 +8601,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ + "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash 2.1.2", "rustls", @@ -6702,6 +8739,33 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rangemap" version = "1.7.1" @@ -6752,7 +8816,7 @@ dependencies = [ "avif-serialize", "imgref", "loop9", - "quick-error", + "quick-error 2.0.1", "rav1e", "rayon", "rgb", @@ -6808,8 +8872,6 @@ dependencies = [ [[package]] name = "rdocx" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f95247a34b31fcd322d88abb3d3ddb599d3fa288006ffc2e95ffa524aec2ec5d" dependencies = [ "rdocx-html", "rdocx-layout", @@ -7014,6 +9076,53 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "resvg" version = "0.45.1" @@ -7320,6 +9429,8 @@ version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -7359,12 +9470,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.0", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -7376,6 +9515,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error 1.2.3", + "tempfile", + "wait-timeout", +] + [[package]] name = "rustybuzz" version = "0.14.1" @@ -7525,6 +9676,16 @@ dependencies = [ "libc", ] +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "self_cell" version = "1.2.2" @@ -7537,6 +9698,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" + [[package]] name = "serde" version = "1.0.228" @@ -7547,6 +9714,41 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_columnar" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a16e404f17b16d0273460350e29b02d76ba0d70f34afdc9a4fa034c97d6c6eb" +dependencies = [ + "itertools 0.11.0", + "postcard", + "serde", + "serde_columnar_derive", + "thiserror 1.0.69", +] + +[[package]] +name = "serde_columnar_derive" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45958fce4903f67e871fbf15ac78e289269b21ebd357d6fecacdba233629112e" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -7668,6 +9870,16 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct", + "serde", +] + [[package]] name = "sfsymbols" version = "0.2.1" @@ -7753,12 +9965,28 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" + [[package]] name = "simd-adler32" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simd_helpers" version = "0.1.0" @@ -7774,6 +10002,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple-dns" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a75cbde1bf934313596a004973e462f9a82caa814dcf1a5f507bdf51597eeb4" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "simplecss" version = "0.2.2" @@ -7789,6 +10026,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + [[package]] name = "sketches-ddsketch" version = "0.4.0" @@ -7848,6 +10095,9 @@ name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] [[package]] name = "smol" @@ -7882,6 +10132,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sorted-index-buffer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea06cc588e43c632923a55450401b8f25e628131571d4e1baea1bdfdb2b5ed06" + +[[package]] +name = "spez" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87e960f4dca2788eeb86bbdde8dd246be8948790b7618d656e68f9b720a86e8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "spin" version = "0.9.8" @@ -7891,6 +10158,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spirv" version = "0.3.0+sdk-1.3.268.0" @@ -7900,6 +10173,16 @@ dependencies = [ "bitflags 2.13.0", ] +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlite-wasm-rs" version = "0.5.5" @@ -8039,6 +10322,15 @@ dependencies = [ "strum_macros 0.27.2", ] +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + [[package]] name = "strum_macros" version = "0.26.4" @@ -8064,6 +10356,18 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "subsetter" version = "0.2.6" @@ -8290,9 +10594,20 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.13.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ "bitflags 2.13.0", "core-foundation 0.9.4", @@ -8321,6 +10636,12 @@ dependencies = [ "slotmap", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "take-until" version = "0.2.0" @@ -8351,8 +10672,8 @@ dependencies = [ "itertools 0.14.0", "levenshtein_automata", "log", - "lru", - "lz4_flex", + "lru 0.16.4", + "lz4_flex 0.13.1", "measure_time", "memmap2", "once_cell", @@ -8607,21 +10928,22 @@ dependencies = [ "fax", "flate2", "half", - "quick-error", - "weezl", + "quick-error 2.0.1", + "weezl 0.1.12", "zune-jpeg 0.5.15", ] [[package]] name = "time" -version = "0.3.47" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", - "itoa", "js-sys", + "libc", "num-conv", + "num_threads", "powerfmt", "serde_core", "time-core", @@ -8630,15 +10952,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -8738,10 +11060,23 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -8764,6 +11099,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -8773,10 +11120,34 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] +[[package]] +name = "tokio-websockets" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad543404f98bfc969aeb71994105c592acfc6c43323fddcd016bb208d1c65cb" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-sink", + "getrandom 0.4.2", + "http", + "httparse", + "rand 0.10.1", + "ring", + "rustls-pki-types", + "sha1_smol", + "simdutf8", + "tokio", + "tokio-rustls", + "tokio-util", +] + [[package]] name = "toml" version = "0.8.23" @@ -8884,6 +11255,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.0", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -8902,6 +11291,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -8952,6 +11342,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -8962,12 +11362,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex-automata", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -9049,6 +11452,15 @@ dependencies = [ "syntect", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.4", +] + [[package]] name = "typed-arena" version = "2.0.2" @@ -9365,6 +11777,12 @@ dependencies = [ "unicode-math-class", ] +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "uds_windows" version = "1.2.1" @@ -9376,6 +11794,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unic-langid" version = "0.9.6" @@ -9531,6 +11955,16 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -9685,6 +12119,43 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vergen" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "vergen-lib", +] + +[[package]] +name = "vergen-gitcl" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ff3b5300a085d6bcd8fc96a507f706a28ae3814693236c9b409db71a1d15b9" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", + "time", + "vergen", + "vergen-lib", +] + +[[package]] +name = "vergen-lib" +version = "9.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569" +dependencies = [ + "anyhow", + "derive_builder", + "rustversion", +] + [[package]] name = "version_check" version = "0.9.5" @@ -9711,6 +12182,15 @@ dependencies = [ "libc", ] +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.2.0" @@ -9744,9 +12224,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen 0.57.1", ] @@ -9762,9 +12242,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -9775,9 +12255,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.72" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -9785,9 +12265,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9795,9 +12275,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", @@ -9808,9 +12288,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.122" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -9850,13 +12330,26 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmi" version = "0.51.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb321403ce594274827657a908e13d1d9918aa02257b8bf8391949d9764023ff" dependencies = [ - "spin", + "spin 0.9.8", "wasmi_collections", "wasmi_core", "wasmi_ir", @@ -10007,9 +12500,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.99" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -10025,12 +12518,36 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "weezl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + [[package]] name = "which" version = "5.0.0" @@ -10056,6 +12573,12 @@ dependencies = [ "winsafe", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -10103,11 +12626,23 @@ version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ - "windows-collections", + "windows-collections 0.2.0", "windows-core 0.61.2", - "windows-future", + "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections 0.3.2", + "windows-core 0.62.2", + "windows-future 0.3.2", + "windows-numerics 0.3.1", ] [[package]] @@ -10120,7 +12655,7 @@ dependencies = [ "rayon", "thiserror 2.0.18", "windows 0.61.3", - "windows-future", + "windows-future 0.2.1", ] [[package]] @@ -10132,6 +12667,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core 0.62.2", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -10178,7 +12722,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", + "windows-threading 0.2.1", ] [[package]] @@ -10247,6 +12802,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core 0.62.2", + "windows-link 0.2.1", +] + [[package]] name = "windows-registry" version = "0.4.0" @@ -10269,6 +12834,17 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.1.2" @@ -10425,6 +13001,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -10700,6 +13285,21 @@ dependencies = [ "wasmparser 0.244.0", ] +[[package]] +name = "wmi" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c81b85c57a57500e56669586496bf2abd5cf082b9d32995251185d105208b64" +dependencies = [ + "chrono", + "futures", + "log", + "serde", + "thiserror 2.0.18", + "windows 0.62.2", + "windows-core 0.62.2", +] + [[package]] name = "workspace-hack" version = "0.1.0" @@ -10731,6 +13331,25 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "ws_stream_wasm" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c173014acad22e83f16403ee360115b38846fe754e735c5d9d3803fe70c6abc" +dependencies = [ + "async_io_stream", + "futures", + "js-sys", + "log", + "pharos", + "rustc_version", + "send_wrapper", + "thiserror 2.0.18", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wyz" version = "0.5.1" @@ -10843,6 +13462,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + [[package]] name = "xml5ever" version = "0.18.1" @@ -10860,6 +13485,15 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + [[package]] name = "xmlwriter" version = "0.1.0" @@ -10887,6 +13521,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yazi" version = "0.2.1" @@ -11086,7 +13726,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", + "system-configuration 0.6.1", "tokio", "tokio-rustls", "tokio-socks", @@ -11096,7 +13736,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "windows-registry 0.4.0", ] diff --git a/Cargo.toml b/Cargo.toml index 7cdeca66..bbcf9946 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,11 @@ publish = false [workspace.dependencies] anyhow = "1.0.102" +async-channel = "2" +base64 = "0.22" clap = { version = "4.5", features = ["derive"] } cosmic-text = { version = "0.14.2", default-features = false, features = [ + "std", "shape-run-cache", ] } crop = "0.4.3" @@ -23,6 +26,8 @@ docxide-pdf = { version = "0.15.1", default-features = false, git = "https://git fff-search = "0.8.2-nightly.ffb39a8" flowstate-document = { version = "0.1.0", path = "crates/flowstate-document" } flowstate-docx = { version = "0.1.0", path = "crates/flowstate-docx" } +flowstate-fidelity = { version = "0.1.0", path = "crates/flowstate-fidelity" } +flowstate-collab = { version = "0.1.0", path = "crates/flowstate-collab" } flowstate-flow = { version = "0.1.0", path = "crates/flowstate-flow" } flowstate-tub = { version = "0.1.0", path = "crates/flowstate-tub" } gpui-flowtext = { version = "0.1.0", path = "crates/gpui-flowtext" } @@ -33,11 +38,20 @@ gpui-symbols = { version = "0.6.1", features = ["component"] } hotpath = "0.16.1" ignore = "0.4.25" imagesize = "0.13" -lopdf = "0.40.0" +iroh = "1.0.0-rc.1" +iroh-gossip = "0.100" +iroh-tickets = "1.0.0-rc.1" +lopdf = "0.44.0" +loro = "1.13" mimalloc = "0.1.52" mitex = "0.2" notify = "8.2.0" -quick-xml = "0.36.2" +n0-future = "0.3" +postcard = { version = "1", features = ["use-std"] } +proptest = "1" +quick-xml = "0.41.0" +rand = "0.9" +reqwest = { version = "0.13.4", default-features = false, features = ["form", "json", "rustls"] } rdocx = "0.1.2" rdocx-opc = "0.1.2" rdocx-oxml = "0.1.2" @@ -46,20 +60,44 @@ rusqlite = { version = "0.40.0", features = ["bundled"] } rustc-hash = "2.1.2" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" +sha2 = "0.10" tantivy = "0.26.1" tempfile = "3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "sync"] } toml = "1.1.2" +tracing = "0.1" +tracing-appender = "0.2" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +twox-hash = "2" typst = "0.14" typst-assets = "0.14" typst-library = "0.14" typst-svg = "0.14" unicode-segmentation = "1" +url = "2.5" uuid = { version = "1.23.1", features = ["serde", "v4"] } zip = { version = "0.6.6", default-features = false, features = ["deflate"] } zstd = "0.13.3" [patch.crates-io] +# §perf (bimodal-undo fix): vendored to add `BTree::next_leaf_matching`, a +# cache-predicate-skipping next-leaf walk. The rich-text tracker's +# `CrdtRope::insert` needs "first NON-FUTURE element to the right" per fed op; +# upstream only offers linear iteration, which goes O(ops²) when a tracker +# rebuild retreats a large concurrent branch (the 80-195s dirty-history undo +# regime). One added method; re-apply on generic-btree upgrades. +generic-btree = { path = "vendor/generic-btree" } gpui-component = { path = "vendor/gpui-component" } +# §perf: vendored to add `LoroDoc::query_text_id_positions`, a single-pass batch +# resolver for many text cursor ids. Upstream Loro 1.13 only offers per-id cursor +# resolution (`get_cursor_pos`), which is O(elements) each; resolving every +# paragraph/block boundary per projection was therefore O(paragraphs²) and hung +# the CRDT actor on large documents. The patch is one added method (see +# `richtext_state.rs` get_text_indices_of_ids); re-apply it on Loro upgrades. +loro-internal = { path = "vendor/loro-internal" } +# §act-eleven A11.2: vendored for the 2→1 parse collapse (from_parsed_parts + +# into_typed_document accessors; upstream 0.1.2 keeps Document internals private). +rdocx = { path = "vendor/rdocx" } [workspace.lints.clippy] correctness = { level = "deny", priority = -1 } @@ -227,15 +265,20 @@ unreadable_literal = "allow" collapsible_else_if = "allow" +[profile.dev] +split-debuginfo = "unpacked" + [profile.release] rustflags = ["-Z", "threads=0"] incremental = true +split-debuginfo = "unpacked" [profile.profiler] inherits = "release" opt-level = 3 overflow-checks = false -debug = true +debug = "line-tables-only" +split-debuginfo = "unpacked" strip = false [profile.product] @@ -247,3 +290,9 @@ strip = true lto = "fat" codegen-units = 1 rustflags = ["-C", "target-cpu=native", "-Z", "tune-cpu=native"] + +# Profiling builds: release codegen + debug symbols, separate cache so samply +# runs never invalidate the plain release artifacts. +[profile.release-debug] +inherits = "release" +debug = true diff --git a/crates/flowstate-collab/Cargo.toml b/crates/flowstate-collab/Cargo.toml new file mode 100644 index 00000000..39a397fc --- /dev/null +++ b/crates/flowstate-collab/Cargo.toml @@ -0,0 +1,81 @@ +[package] +name = "flowstate-collab" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow = { workspace = true } +async-channel = { workspace = true } +base64 = { workspace = true } +blake3 = "1.8.5" +flowstate-document.workspace = true +flowstate-fidelity.workspace = true +gpui-flowtext = { workspace = true } +hotpath.workspace = true +iroh = { workspace = true } +iroh-gossip = { workspace = true } +iroh-tickets = { workspace = true } +loro = { workspace = true } +n0-future = { workspace = true } +postcard = { workspace = true } +rand = { workspace = true } +reqwest = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +twox-hash = { workspace = true } +url = { workspace = true } +uuid = { workspace = true } +zstd.workspace = true + +[features] +hotpath = ["hotpath/hotpath", "flowstate-document/hotpath", "gpui-flowtext/hotpath"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "flowstate-document/hotpath-cpu", "gpui-flowtext/hotpath-cpu"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "flowstate-document/hotpath-alloc", "gpui-flowtext/hotpath-alloc"] + +[dev-dependencies] +gpui = { workspace = true, features = ["test-support"] } +pollster = "0.4.0" +proptest = { workspace = true } +tempfile.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +[target.'cfg(target_os = "linux")'.dependencies] +bluer = { version = "0.17.4", features = ["bluetoothd"] } +futures-util = "0.3" + +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.62.2", features = [ + "Devices_Bluetooth", + "Devices_Bluetooth_Advertisement", + "Devices_Bluetooth_GenericAttributeProfile", + "Foundation", + "Foundation_Collections", + "Storage_Streams", +] } + +[target.'cfg(target_os = "macos")'.dependencies] +dispatch2 = "0.3.1" +objc2 = "0.6.4" +objc2-foundation = { version = "0.3.2", features = ["NSArray", "NSData", "NSDictionary", "NSError", "NSObject", "NSString"] } +objc2-core-bluetooth = { version = "0.3.2", features = [ + "CBATTRequest", + "CBAttribute", + "CBCentral", + "CBCentralManager", + "CBCharacteristic", + "CBManager", + "CBPeer", + "CBPeripheral", + "CBPeripheralManager", + "CBService", + "CBUUID", + "dispatch2", +] } + +[lints] +workspace = true diff --git a/crates/flowstate-collab/assets/flowstate-cards-256k.zdict b/crates/flowstate-collab/assets/flowstate-cards-256k.zdict new file mode 100644 index 00000000..2fd08a95 Binary files /dev/null and b/crates/flowstate-collab/assets/flowstate-cards-256k.zdict differ diff --git a/crates/flowstate-collab/src/admission.rs b/crates/flowstate-collab/src/admission.rs new file mode 100644 index 00000000..f91b9e24 --- /dev/null +++ b/crates/flowstate-collab/src/admission.rs @@ -0,0 +1,57 @@ +//! Symmetric editor admission for live collaboration sessions. + +use rand::{TryRngCore as _, rngs::OsRng}; +use serde::{Deserialize, Serialize}; + +pub const SESSION_SECRET_LEN: usize = 32; + +/// Bearer secret shared by every admitted editor in one ephemeral session. +/// It is deliberately not persisted: the session and all of its links die +/// when the last participant disconnects. +#[derive(Clone, Serialize, Deserialize)] +pub struct SessionAdmission([u8; SESSION_SECRET_LEN]); + +impl std::fmt::Debug for SessionAdmission { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SessionAdmission([REDACTED])") + } +} + +impl PartialEq for SessionAdmission { + fn eq(&self, other: &Self) -> bool { + self + .0 + .iter() + .zip(other.0.iter()) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 + } +} + +impl Eq for SessionAdmission {} + +impl SessionAdmission { + #[must_use] + pub fn generate() -> Self { + let mut bytes = [0; SESSION_SECRET_LEN]; + OsRng + .try_fill_bytes(&mut bytes) + .expect("OS randomness must be available to create a collaboration admission secret"); + Self(bytes) + } + + #[must_use] + pub const fn from_bytes(bytes: [u8; SESSION_SECRET_LEN]) -> Self { + Self(bytes) + } + + #[must_use] + pub const fn as_bytes(&self) -> &[u8; SESSION_SECRET_LEN] { + &self.0 + } + + #[must_use] + pub fn tag(&self, payload: &[u8]) -> [u8; 32] { + *blake3::keyed_hash(&self.0, payload).as_bytes() + } +} diff --git a/crates/flowstate-collab/src/bluetooth.rs b/crates/flowstate-collab/src/bluetooth.rs new file mode 100644 index 00000000..b6012344 --- /dev/null +++ b/crates/flowstate-collab/src/bluetooth.rs @@ -0,0 +1,1063 @@ +//! Nearby rendezvous over Bluetooth Low Energy. +//! +//! BLE advertising payloads are intentionally only locators. The complete +//! signed advertisement is read from a GATT characteristic, avoiding the +//! 20–28 byte legacy advertising limits without truncating security data. + +use std::{future::Future, pin::Pin, time::Duration}; + +use anyhow::{Result, bail}; +use iroh::PublicKey; + +use crate::discovery::{DiscoveryAdvertisement, DiscoveryTransport, RendezvousBackend}; + +pub const FLOWSTATE_SERVICE_UUID: u128 = 0x0198_f10e_7679_7cd0_961d_f107_2fc0_b002; +pub const FLOWSTATE_ADVERTISEMENT_CHARACTERISTIC_UUID: u128 = 0x0198_f10e_7679_7cd0_961d_f107_2fc0_b003; +const MAX_GATT_ADVERTISEMENT_BYTES: usize = 60 * 1024; + +/// Common framing used by every native adapter. A length prefix lets clients +/// issue offset reads until the entire postcard value has arrived. +pub fn encode_gatt_advertisement(advertisement: &DiscoveryAdvertisement) -> Result> { + let bytes = postcard::to_stdvec(advertisement)?; + if bytes.len() > MAX_GATT_ADVERTISEMENT_BYTES { + bail!("Bluetooth discovery advertisement is too large"); + } + let len = u32::try_from(bytes.len()).expect("bounded advertisement length"); + let mut framed = Vec::with_capacity(4 + bytes.len()); + framed.extend_from_slice(&len.to_le_bytes()); + framed.extend_from_slice(&bytes); + Ok(framed) +} + +pub fn decode_gatt_advertisement(framed: &[u8]) -> Result { + let header: [u8; 4] = framed + .get(..4) + .ok_or_else(|| anyhow::anyhow!("Bluetooth advertisement is missing its length"))? + .try_into() + .expect("four-byte slice"); + let len = usize::try_from(u32::from_le_bytes(header)).expect("u32 fits usize"); + if len > MAX_GATT_ADVERTISEMENT_BYTES || framed.len() != len + 4 { + bail!("Bluetooth advertisement length is invalid"); + } + postcard::from_bytes(&framed[4..]).map_err(Into::into) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::{collections::BTreeMap, sync::Arc}; + + use anyhow::{Context as _, ensure}; + use bluer::{ + Adapter, AdapterEvent, DiscoveryFilter, Uuid, + adv::{Advertisement, AdvertisementHandle, Type}, + gatt::{ + local::{Application, ApplicationHandle, Characteristic, CharacteristicRead, Service}, + remote::CharacteristicReadRequest, + }, + }; + use futures_util::{FutureExt as _, StreamExt as _}; + use tokio::sync::Mutex; + + use super::*; + + struct PublishedAdvertisement { + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + _application: ApplicationHandle, + _advertisement: AdvertisementHandle, + } + + /// `BlueZ` implementation used on Linux. Registration handles remain live for + /// exactly as long as the signed advertisement is published. + pub struct NativeBluetoothBackend { + adapter: Adapter, + published: Mutex>, + scan_lock: Mutex<()>, + scan_duration: Duration, + } + + impl NativeBluetoothBackend { + pub async fn new(scan_duration: Duration) -> Result { + let session = bluer::Session::new().await.context("connecting to BlueZ")?; + let adapter = session + .default_adapter() + .await + .context("finding the default Bluetooth adapter")?; + if !adapter + .is_powered() + .await + .context("reading Bluetooth power state")? + { + adapter + .set_powered(true) + .await + .context("powering on Bluetooth adapter")?; + } + Ok(Self { + adapter, + published: Mutex::new(None), + scan_lock: Mutex::new(()), + scan_duration, + }) + } + + async fn publish_inner(&self, advertisement: DiscoveryAdvertisement) -> Result<()> { + // Most desktop adapters expose one legacy advertising slot. Release the + // old GATT/advertisement pair before registering its replacement. + *self.published.lock().await = None; + let framed = Arc::new(encode_gatt_advertisement(&advertisement)?); + let read_value = Arc::clone(&framed); + let application = Application { + services: vec![Service { + uuid: service_uuid(), + primary: true, + characteristics: vec![Characteristic { + uuid: characteristic_uuid(), + read: Some(CharacteristicRead { + read: true, + fun: Box::new(move |request| { + let value = Arc::clone(&read_value); + async move { + let offset = usize::from(request.offset); + if offset > value.len() { + return Err(bluer::gatt::local::ReqError::InvalidOffset); + } + let payload = request.mtu.saturating_sub(1).max(1) as usize; + Ok(value[offset..value.len().min(offset.saturating_add(payload))].to_vec()) + } + .boxed() + }), + ..Default::default() + }), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }; + let application_handle = self + .adapter + .serve_gatt_application(application) + .await + .context("registering Flowstate Bluetooth GATT service")?; + + let mut service_data = BTreeMap::new(); + service_data.insert(service_uuid(), advertisement.document_fingerprint[..16].to_vec()); + let ble_advertisement = Advertisement { + advertisement_type: Type::Peripheral, + service_uuids: [service_uuid()].into_iter().collect(), + service_data, + discoverable: Some(true), + local_name: Some("Flowstate".into()), + ..Default::default() + }; + let advertisement_handle = match self.adapter.advertise(ble_advertisement).await { + Ok(handle) => handle, + Err(error) => { + drop(application_handle); + return Err(error).context("advertising Flowstate Bluetooth service"); + }, + }; + *self.published.lock().await = Some(PublishedAdvertisement { + identity: advertisement.identity, + device_id: advertisement.device_id, + document_fingerprint: advertisement.document_fingerprint, + _application: application_handle, + _advertisement: advertisement_handle, + }); + Ok(()) + } + + async fn scan_inner(&self, document_fingerprint: [u8; 32]) -> Result> { + let _scan_guard = self.scan_lock.lock().await; + self + .adapter + .set_discovery_filter(DiscoveryFilter { + uuids: [service_uuid()].into_iter().collect(), + transport: bluer::DiscoveryTransport::Le, + duplicate_data: false, + ..Default::default() + }) + .await + .context("configuring Flowstate Bluetooth scan")?; + let stream = self + .adapter + .discover_devices() + .await + .context("starting Flowstate Bluetooth scan")?; + futures_util::pin_mut!(stream); + let deadline = tokio::time::Instant::now() + self.scan_duration; + let mut found = Vec::new(); + loop { + let event = match tokio::time::timeout_at(deadline, stream.next()).await { + Ok(Some(event)) => event, + Ok(None) | Err(_) => break, + }; + let AdapterEvent::DeviceAdded(address) = event else { continue }; + let device = self + .adapter + .device(address) + .context("opening discovered Bluetooth device")?; + if device.rssi().await.ok().flatten().is_none() { + continue; + } + let service_data = device + .service_data() + .await + .unwrap_or_default() + .unwrap_or_default(); + // A locator for a DIFFERENT document is skipped pre-connect. A missing + // locator is not: CoreBluetooth cannot publish service data, so macOS + // peers advertise only the service UUID and are verified after the + // full signed record is read (the authoritative check either way). + if service_data + .get(&service_uuid()) + .is_some_and(|locator| locator.as_slice() != &document_fingerprint[..16]) + { + continue; + } + let remaining = deadline + .saturating_duration_since(tokio::time::Instant::now()) + .min(Duration::from_secs(3)); + if remaining.is_zero() { + break; + } + match tokio::time::timeout(remaining, read_advertisement(&device)).await { + Ok(Ok(advertisement)) if advertisement.document_fingerprint == document_fingerprint => found.push(advertisement), + Ok(Ok(_)) => {}, + Ok(Err(error)) => tracing::debug!(%error, %address, "nearby Flowstate Bluetooth record could not be read"), + Err(_) => tracing::debug!(%address, "nearby Flowstate Bluetooth read timed out"), + } + // Leaving the link up occupies the peer's (often single) peripheral + // connection slot and suppresses its advertising for other scanners. + if tokio::time::timeout(Duration::from_secs(2), device.disconnect()) + .await + .is_err() + { + tracing::debug!(%address, "nearby Flowstate Bluetooth disconnect timed out"); + } + } + Ok(found) + } + } + + impl RendezvousBackend for NativeBluetoothBackend { + fn transport(&self) -> DiscoveryTransport { + DiscoveryTransport::Bluetooth + } + + fn publish(&self, advertisement: DiscoveryAdvertisement) -> Pin> + Send + '_>> { + Box::pin(self.publish_inner(advertisement)) + } + + fn scan(&self, document_fingerprint: [u8; 32]) -> Pin>> + Send + '_>> { + Box::pin(self.scan_inner(document_fingerprint)) + } + + fn clear( + &self, + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let mut published = self.published.lock().await; + if published.as_ref().is_some_and(|current| { + current.identity == identity && current.device_id == device_id && current.document_fingerprint == document_fingerprint + }) { + *published = None; + } + drop(published); + Ok(()) + }) + } + } + + async fn read_advertisement(device: &bluer::Device) -> Result { + if !device + .is_connected() + .await + .context("reading Bluetooth connection state")? + { + device + .connect() + .await + .context("connecting to nearby Flowstate")?; + } + let mut characteristic = None; + for service in device + .services() + .await + .context("enumerating nearby Flowstate services")? + { + if service.uuid().await? != service_uuid() { + continue; + } + for candidate in service.characteristics().await? { + if candidate.uuid().await? == characteristic_uuid() { + characteristic = Some(candidate); + break; + } + } + } + let characteristic = characteristic.context("Flowstate GATT characteristic is missing")?; + let mut framed = Vec::new(); + loop { + ensure!( + framed.len() <= MAX_GATT_ADVERTISEMENT_BYTES + 4, + "Bluetooth advertisement exceeded its maximum length" + ); + let offset = u16::try_from(framed.len()).context("Bluetooth advertisement offset overflow")?; + let chunk = characteristic + .read_ext(&CharacteristicReadRequest { + offset, + ..Default::default() + }) + .await + .context("reading Flowstate GATT characteristic")?; + ensure!(!chunk.is_empty(), "Bluetooth advertisement ended early"); + framed.extend_from_slice(&chunk); + if let Some(expected) = expected_framed_len(&framed)? + && framed.len() >= expected + { + ensure!(framed.len() == expected, "Bluetooth advertisement returned trailing bytes"); + break; + } + } + decode_gatt_advertisement(&framed) + } + + fn expected_framed_len(bytes: &[u8]) -> Result> { + let Some(header) = bytes.get(..4) else { return Ok(None) }; + let len = u32::from_le_bytes(header.try_into().expect("four-byte slice")) as usize; + ensure!( + len <= MAX_GATT_ADVERTISEMENT_BYTES, + "Bluetooth advertisement declares an excessive length" + ); + Ok(Some(len + 4)) + } + + fn service_uuid() -> Uuid { + Uuid::from_u128(FLOWSTATE_SERVICE_UUID) + } + + fn characteristic_uuid() -> Uuid { + Uuid::from_u128(FLOWSTATE_ADVERTISEMENT_CHARACTERISTIC_UUID) + } +} + +#[cfg(target_os = "linux")] +pub use linux::NativeBluetoothBackend; + +#[cfg(target_os = "windows")] +mod windows_backend { + use std::collections::HashSet; + + use anyhow::{Context as _, ensure}; + use tokio::sync::Mutex; + use windows::{ + Devices::Bluetooth::{ + Advertisement::{ + BluetoothLEAdvertisement, BluetoothLEAdvertisementFilter, BluetoothLEAdvertisementReceivedEventArgs, BluetoothLEAdvertisementWatcher, + BluetoothLEScanningMode, + }, + BluetoothCacheMode, BluetoothError, BluetoothLEDevice, + GenericAttributeProfile::{ + GattCharacteristicProperties, GattCommunicationStatus, GattLocalCharacteristic, GattLocalCharacteristicParameters, GattProtectionLevel, + GattServiceProvider, GattServiceProviderAdvertisingParameters, + }, + }, + Foundation::TypedEventHandler, + Storage::Streams::{DataReader, DataWriter, IBuffer}, + core::GUID, + }; + + use super::*; + + struct PublishedAdvertisement { + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + provider: GattServiceProvider, + _characteristic: GattLocalCharacteristic, + } + + pub struct NativeBluetoothBackend { + published: Mutex>, + scan_lock: Mutex<()>, + scan_duration: Duration, + } + + impl NativeBluetoothBackend { + pub async fn new(scan_duration: Duration) -> Result { + // WinRT's activation factory initializes MTA usage on demand. Opening a + // watcher here gives settings a synchronous, actionable platform error. + BluetoothLEAdvertisementWatcher::new().context("opening Windows Bluetooth LE watcher")?; + Ok(Self { + published: Mutex::new(None), + scan_lock: Mutex::new(()), + scan_duration, + }) + } + + async fn publish_inner(&self, advertisement: DiscoveryAdvertisement) -> Result<()> { + if let Some(current) = self.published.lock().await.take() { + let _ = current.provider.StopAdvertising(); + } + + let framed = encode_gatt_advertisement(&advertisement)?; + let provider_result = GattServiceProvider::CreateAsync(service_guid()) + .context("creating Flowstate Windows GATT provider")? + .await + .context("creating Flowstate Windows GATT provider")?; + ensure!( + provider_result.Error()? == BluetoothError::Success, + "Windows could not create the Flowstate GATT service" + ); + let provider = provider_result.ServiceProvider()?; + + let parameters = GattLocalCharacteristicParameters::new()?; + parameters.SetCharacteristicProperties(GattCharacteristicProperties::Read)?; + parameters.SetReadProtectionLevel(GattProtectionLevel::Plain)?; + parameters.SetUserDescription(&"Flowstate signed discovery record".into())?; + { + let framed_buffer = bytes_to_buffer(&framed)?; + parameters.SetStaticValue(&framed_buffer)?; + } + let characteristic_result = provider + .Service()? + .CreateCharacteristicAsync(characteristic_guid(), ¶meters)? + .await?; + ensure!( + characteristic_result.Error()? == BluetoothError::Success, + "Windows could not create the Flowstate GATT characteristic" + ); + let characteristic = characteristic_result.Characteristic()?; + + let advertising = GattServiceProviderAdvertisingParameters::new()?; + advertising.SetIsConnectable(true)?; + advertising.SetIsDiscoverable(true)?; + // ServiceData is optional on older Windows builds. The service UUID is + // still filtered before connecting and the full signed document scope is + // always checked after reading the characteristic. + { + let locator = bytes_to_buffer(&advertisement.document_fingerprint[..16])?; + let _ = advertising.SetServiceData(&locator); + } + provider + .StartAdvertisingWithParameters(&advertising) + .context("starting Flowstate Windows Bluetooth advertisement")?; + + *self.published.lock().await = Some(PublishedAdvertisement { + identity: advertisement.identity, + device_id: advertisement.device_id, + document_fingerprint: advertisement.document_fingerprint, + provider, + _characteristic: characteristic, + }); + Ok(()) + } + + async fn scan_inner(&self, document_fingerprint: [u8; 32]) -> Result> { + let _guard = self.scan_lock.lock().await; + let filter_record = BluetoothLEAdvertisement::new()?; + filter_record.ServiceUuids()?.Append(service_guid())?; + let filter = BluetoothLEAdvertisementFilter::new()?; + filter.SetAdvertisement(&filter_record)?; + let watcher = BluetoothLEAdvertisementWatcher::Create(&filter)?; + watcher.SetScanningMode(BluetoothLEScanningMode::Active)?; + + let (addresses_tx, addresses_rx) = async_channel::bounded(64); + let received_token = { + let received = TypedEventHandler::::new(move |_, args| { + if let Ok(args) = args.ok() + && let Ok(address) = args.BluetoothAddress() + { + let _ = addresses_tx.try_send(address); + } + Ok(()) + }); + watcher.Received(&received)? + }; + watcher.Start().context("starting Windows Bluetooth scan")?; + + let deadline = tokio::time::Instant::now() + self.scan_duration; + let mut seen = HashSet::new(); + let mut found = Vec::new(); + loop { + let address = match tokio::time::timeout_at(deadline, addresses_rx.recv()).await { + Ok(Ok(address)) => address, + Ok(Err(_)) | Err(_) => break, + }; + if !seen.insert(address) { + continue; + } + let remaining = deadline + .saturating_duration_since(tokio::time::Instant::now()) + .min(Duration::from_secs(3)); + if remaining.is_zero() { + break; + } + match tokio::time::timeout(remaining, read_advertisement(address)).await { + Ok(Ok(advertisement)) if advertisement.document_fingerprint == document_fingerprint => found.push(advertisement), + Ok(Ok(_)) => {}, + Ok(Err(error)) => tracing::debug!(%error, address, "nearby Windows Flowstate Bluetooth record could not be read"), + Err(_) => tracing::debug!(address, "nearby Windows Flowstate Bluetooth read timed out"), + } + } + let _ = watcher.Stop(); + let _ = watcher.RemoveReceived(received_token); + Ok(found) + } + } + + impl RendezvousBackend for NativeBluetoothBackend { + fn transport(&self) -> DiscoveryTransport { + DiscoveryTransport::Bluetooth + } + + fn publish(&self, advertisement: DiscoveryAdvertisement) -> Pin> + Send + '_>> { + Box::pin(self.publish_inner(advertisement)) + } + + fn scan(&self, document_fingerprint: [u8; 32]) -> Pin>> + Send + '_>> { + Box::pin(self.scan_inner(document_fingerprint)) + } + + fn clear( + &self, + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let mut published = self.published.lock().await; + if published.as_ref().is_some_and(|current| { + current.identity == identity && current.device_id == device_id && current.document_fingerprint == document_fingerprint + }) { + if let Some(current) = published.take() { + current.provider.StopAdvertising()?; + } + } + Ok(()) + }) + } + } + + async fn read_advertisement(address: u64) -> Result { + let device = BluetoothLEDevice::FromBluetoothAddressAsync(address)? + .await + .context("opening nearby Windows Bluetooth device")?; + let services = device + .GetGattServicesForUuidWithCacheModeAsync(service_guid(), BluetoothCacheMode::Uncached)? + .await?; + ensure!( + services.Status()? == GattCommunicationStatus::Success, + "reading Flowstate GATT services failed" + ); + let service = { + let services = services.Services()?; + ensure!(services.Size()? > 0, "Flowstate GATT service is missing"); + services.GetAt(0)? + }; + let characteristics = service + .GetCharacteristicsForUuidWithCacheModeAsync(characteristic_guid(), BluetoothCacheMode::Uncached)? + .await?; + ensure!( + characteristics.Status()? == GattCommunicationStatus::Success, + "reading Flowstate GATT characteristics failed" + ); + let characteristic = { + let characteristics = characteristics.Characteristics()?; + ensure!(characteristics.Size()? > 0, "Flowstate GATT characteristic is missing"); + characteristics.GetAt(0)? + }; + // WinRT performs ATT long reads (Read Blob continuation) internally, so + // this single call returns the complete characteristic value; a truncated + // frame still fails loud on the length prefix in `decode_gatt_advertisement`. + let value = characteristic + .ReadValueWithCacheModeAsync(BluetoothCacheMode::Uncached)? + .await?; + ensure!(value.Status()? == GattCommunicationStatus::Success, "reading Flowstate GATT value failed"); + decode_gatt_advertisement(&buffer_to_bytes(value.Value()?)?) + } + + fn bytes_to_buffer(bytes: &[u8]) -> Result { + let writer = DataWriter::new()?; + writer.WriteBytes(bytes)?; + writer.DetachBuffer().map_err(Into::into) + } + + fn buffer_to_bytes(buffer: IBuffer) -> Result> { + let reader = DataReader::FromBuffer(&buffer)?; + let mut bytes = vec![0; reader.UnconsumedBufferLength()? as usize]; + reader.ReadBytes(&mut bytes)?; + Ok(bytes) + } + + const fn service_guid() -> GUID { + GUID::from_u128(FLOWSTATE_SERVICE_UUID) + } + + const fn characteristic_guid() -> GUID { + GUID::from_u128(FLOWSTATE_ADVERTISEMENT_CHARACTERISTIC_UUID) + } +} + +#[cfg(target_os = "windows")] +pub use windows_backend::NativeBluetoothBackend; + +#[cfg(target_os = "macos")] +mod macos_backend { + use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicIsize, Ordering}, + }, + thread, + }; + + use anyhow::Context as _; + use dispatch2::DispatchQueue; + use objc2::{ + AnyThread, DefinedClass, Message as _, define_class, msg_send, + rc::Retained, + runtime::{AnyObject, ProtocolObject}, + }; + use objc2_core_bluetooth::{ + CBAdvertisementDataServiceUUIDsKey, CBAttributePermissions, CBCentralManager, CBCentralManagerDelegate, CBCharacteristic, + CBCharacteristicProperties, CBManagerState, CBMutableCharacteristic, CBMutableService, CBPeripheral, CBPeripheralDelegate, + CBPeripheralManager, CBPeripheralManagerDelegate, CBService, CBUUID, + }; + use objc2_foundation::{NSArray, NSData, NSDictionary, NSError, NSNumber, NSObject, NSObjectProtocol, NSString}; + + use super::*; + + /// `manager_state` holds the raw `CBManagerState` (`Unknown` = 0 matches the + /// atomic's default), so waiters can tell a pending permission prompt apart + /// from Bluetooth that is definitively off. + #[derive(Default)] + struct MacDelegateState { + manager_state: AtomicIsize, + scan_sender: Mutex>>, + peripherals: Mutex>>, + } + + struct MacDelegateIvars { + state: Arc, + } + + define_class!( + // SAFETY: NSObject has no subclassing requirements and this class has no + // Drop implementation. All mutable Rust state is synchronized. + #[unsafe(super = NSObject)] + #[thread_kind = AnyThread] + #[ivars = MacDelegateIvars] + struct MacBluetoothDelegate; + + // SAFETY: These delegate protocols impose callback signatures only. + unsafe impl NSObjectProtocol for MacBluetoothDelegate {} + + unsafe impl CBCentralManagerDelegate for MacBluetoothDelegate { + #[unsafe(method(centralManagerDidUpdateState:))] + unsafe fn central_manager_did_update_state(&self, central: &CBCentralManager) { + let state = unsafe { central.state() }; + self + .ivars() + .state + .manager_state + .store(state.0, Ordering::Release); + } + + #[unsafe(method(centralManager:didDiscoverPeripheral:advertisementData:RSSI:))] + unsafe fn central_manager_did_discover( + &self, + central: &CBCentralManager, + peripheral: &CBPeripheral, + _advertisement_data: &NSDictionary, + _rssi: &NSNumber, + ) { + let retained = peripheral.retain(); + unsafe { + peripheral.setDelegate(Some(ProtocolObject::from_ref(self))); + central.connectPeripheral_options(peripheral, None); + } + self + .ivars() + .state + .peripherals + .lock() + .expect("Bluetooth peripheral lock poisoned") + .push(retained); + } + + #[unsafe(method(centralManager:didConnectPeripheral:))] + unsafe fn central_manager_did_connect(&self, _central: &CBCentralManager, peripheral: &CBPeripheral) { + let services = NSArray::from_retained_slice(&[service_uuid()]); + unsafe { peripheral.discoverServices(Some(&*services)) }; + } + } + + unsafe impl CBPeripheralDelegate for MacBluetoothDelegate { + #[unsafe(method(peripheral:didDiscoverServices:))] + unsafe fn peripheral_did_discover_services(&self, peripheral: &CBPeripheral, error: Option<&NSError>) { + if error.is_some() { + return; + } + let expected = service_uuid(); + if let Some(services) = unsafe { peripheral.services() } { + for service in &*services { + if unsafe { service.UUID() } == expected { + let characteristics = NSArray::from_retained_slice(&[characteristic_uuid()]); + unsafe { peripheral.discoverCharacteristics_forService(Some(&*characteristics), &service) }; + } + } + } + } + + #[unsafe(method(peripheral:didDiscoverCharacteristicsForService:error:))] + unsafe fn peripheral_did_discover_characteristics(&self, peripheral: &CBPeripheral, service: &CBService, error: Option<&NSError>) { + if error.is_some() { + return; + } + let expected = characteristic_uuid(); + if let Some(characteristics) = unsafe { service.characteristics() } { + for characteristic in &*characteristics { + if unsafe { characteristic.UUID() } == expected { + unsafe { peripheral.readValueForCharacteristic(&characteristic) }; + } + } + } + } + + #[unsafe(method(peripheral:didUpdateValueForCharacteristic:error:))] + unsafe fn peripheral_did_update_value(&self, _peripheral: &CBPeripheral, characteristic: &CBCharacteristic, error: Option<&NSError>) { + if error.is_some() { + return; + } + let Some(value) = (unsafe { characteristic.value() }) else { return }; + let bytes = unsafe { value.as_bytes_unchecked() }; + let Ok(advertisement) = decode_gatt_advertisement(bytes) else { return }; + if let Some(sender) = self + .ivars() + .state + .scan_sender + .lock() + .expect("Bluetooth scan sender lock poisoned") + .as_ref() + { + let _ = sender.try_send(advertisement); + } + } + } + + unsafe impl CBPeripheralManagerDelegate for MacBluetoothDelegate { + #[unsafe(method(peripheralManagerDidUpdateState:))] + unsafe fn peripheral_manager_did_update_state(&self, peripheral: &CBPeripheralManager) { + let state = unsafe { peripheral.state() }; + self + .ivars() + .state + .manager_state + .store(state.0, Ordering::Release); + } + } + ); + + impl MacBluetoothDelegate { + fn new(state: Arc) -> Retained { + let this = Self::alloc().set_ivars(MacDelegateIvars { state }); + unsafe { msg_send![super(this), init] } + } + } + + struct PublishedAdvertisement { + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + manager: Retained, + _delegate: Retained, + _service: Retained, + _characteristic: Retained, + _advertisement: Retained>, + } + + enum PeripheralCommand { + Publish { + advertisement: DiscoveryAdvertisement, + reply: async_channel::Sender>, + }, + Clear { + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + reply: async_channel::Sender>, + }, + } + + /// CoreBluetooth objects are not `Send`, so every peripheral-role object is + /// confined to one dedicated OS thread; the backend holds only plain-data + /// channels, keeping it `Send + Sync` for the discovery actor. + pub struct NativeBluetoothBackend { + peripheral: async_channel::Sender, + scan_lock: tokio::sync::Mutex<()>, + scan_duration: Duration, + } + + impl NativeBluetoothBackend { + pub async fn new(scan_duration: Duration) -> Result { + let (peripheral, commands) = async_channel::unbounded(); + thread::Builder::new() + .name("flowstate-ble-peripheral".into()) + .spawn(move || peripheral_actor(commands)) + .context("starting the macOS Bluetooth peripheral thread")?; + Ok(Self { + peripheral, + scan_lock: tokio::sync::Mutex::new(()), + scan_duration, + }) + } + + async fn request(&self, build: impl FnOnce(async_channel::Sender>) -> PeripheralCommand) -> Result<()> { + let (reply, response) = async_channel::bounded(1); + self + .peripheral + .send(build(reply)) + .await + .map_err(|_| anyhow::anyhow!("macOS Bluetooth peripheral thread stopped"))?; + response + .recv() + .await + .map_err(|_| anyhow::anyhow!("macOS Bluetooth peripheral thread stopped"))? + } + + async fn scan_inner(&self, document_fingerprint: [u8; 32]) -> Result> { + let _guard = self.scan_lock.lock().await; + let scan_duration = self.scan_duration; + tokio::task::spawn_blocking(move || scan_blocking(document_fingerprint, scan_duration)) + .await + .context("macOS Bluetooth scanner task stopped")? + } + } + + impl RendezvousBackend for NativeBluetoothBackend { + fn transport(&self) -> DiscoveryTransport { + DiscoveryTransport::Bluetooth + } + + fn publish(&self, advertisement: DiscoveryAdvertisement) -> Pin> + Send + '_>> { + Box::pin(self.request(move |reply| PeripheralCommand::Publish { advertisement, reply })) + } + + fn scan(&self, document_fingerprint: [u8; 32]) -> Pin>> + Send + '_>> { + Box::pin(self.scan_inner(document_fingerprint)) + } + + fn clear( + &self, + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + ) -> Pin> + Send + '_>> { + Box::pin(self.request(move |reply| PeripheralCommand::Clear { + identity, + device_id, + document_fingerprint, + reply, + })) + } + } + + fn peripheral_actor(commands: async_channel::Receiver) { + let mut published: Option = None; + while let Ok(command) = commands.recv_blocking() { + match command { + PeripheralCommand::Publish { advertisement, reply } => { + let _ = reply.try_send(publish_on_thread(&mut published, advertisement)); + }, + PeripheralCommand::Clear { + identity, + device_id, + document_fingerprint, + reply, + } => { + if published.as_ref().is_some_and(|current| { + current.identity == identity && current.device_id == device_id && current.document_fingerprint == document_fingerprint + }) { + stop_publishing(published.take()); + } + let _ = reply.try_send(Ok(())); + }, + } + } + // The backend was dropped: withdraw the advertisement with the thread. + stop_publishing(published.take()); + } + + fn stop_publishing(published: Option) { + if let Some(current) = published { + unsafe { + current.manager.stopAdvertising(); + current.manager.removeAllServices(); + current.manager.setDelegate(None); + } + } + } + + fn publish_on_thread(published: &mut Option, advertisement: DiscoveryAdvertisement) -> Result<()> { + // One peripheral-manager slot: release the old GATT/advertisement pair + // before registering its replacement. + stop_publishing(published.take()); + + let state = Arc::new(MacDelegateState::default()); + let delegate = MacBluetoothDelegate::new(Arc::clone(&state)); + // A dedicated serial queue keeps delegate callbacks (and therefore + // `wait_for_power` progress) independent of the main thread's run loop. + let queue = DispatchQueue::new("dev.flowstate.bluetooth.peripheral", None); + let manager = unsafe { + CBPeripheralManager::initWithDelegate_queue(CBPeripheralManager::alloc(), Some(ProtocolObject::from_ref(&*delegate)), Some(&queue)) + }; + wait_for_power(&state, "macOS Bluetooth peripheral manager")?; + + let framed = NSData::from_vec(encode_gatt_advertisement(&advertisement)?); + let characteristic = unsafe { + CBMutableCharacteristic::initWithType_properties_value_permissions( + CBMutableCharacteristic::alloc(), + &characteristic_uuid(), + CBCharacteristicProperties::Read, + Some(&*framed), + CBAttributePermissions::Readable, + ) + }; + let service = unsafe { CBMutableService::initWithType_primary(CBMutableService::alloc(), &service_uuid(), true) }; + let characteristic_ref: &CBCharacteristic = characteristic.as_ref(); + let characteristics = NSArray::from_slice(&[characteristic_ref]); + unsafe { service.setCharacteristics(Some(&*characteristics)) }; + unsafe { manager.addService(&service) }; + + let advertised_services = NSArray::from_retained_slice(&[service_uuid()]); + let advertised_services_object: &AnyObject = advertised_services.as_ref(); + let advertisement_data = + unsafe { NSDictionary::::from_slices(&[CBAdvertisementDataServiceUUIDsKey], &[advertised_services_object]) }; + unsafe { manager.startAdvertising(Some(&*advertisement_data)) }; + + *published = Some(PublishedAdvertisement { + identity: advertisement.identity, + device_id: advertisement.device_id, + document_fingerprint: advertisement.document_fingerprint, + manager, + _delegate: delegate, + _service: service, + _characteristic: characteristic, + _advertisement: advertisement_data, + }); + Ok(()) + } + + fn scan_blocking(document_fingerprint: [u8; 32], duration: Duration) -> Result> { + let state = Arc::new(MacDelegateState::default()); + let (sender, receiver) = async_channel::bounded(64); + *state + .scan_sender + .lock() + .expect("Bluetooth scan sender lock poisoned") = Some(sender); + let delegate = MacBluetoothDelegate::new(Arc::clone(&state)); + // This blocking scan sleeps its own thread, so callbacks must arrive on a + // dedicated queue — on the main queue they would deadlock against a busy + // (or GPUI-owned) main thread. + let queue = DispatchQueue::new("dev.flowstate.bluetooth.central", None); + let manager = + unsafe { CBCentralManager::initWithDelegate_queue(CBCentralManager::alloc(), Some(ProtocolObject::from_ref(&*delegate)), Some(&queue)) }; + wait_for_power(&state, "macOS Bluetooth central manager")?; + let services = NSArray::from_retained_slice(&[service_uuid()]); + unsafe { manager.scanForPeripheralsWithServices_options(Some(&*services), None) }; + thread::sleep(duration); + unsafe { manager.stopScan() }; + for peripheral in state + .peripherals + .lock() + .expect("Bluetooth peripheral lock poisoned") + .iter() + { + unsafe { manager.cancelPeripheralConnection(peripheral) }; + } + unsafe { manager.setDelegate(None) }; + + let mut found = Vec::new(); + while let Ok(advertisement) = receiver.try_recv() { + if advertisement.document_fingerprint == document_fingerprint { + found.push(advertisement); + } + } + Ok(found) + } + + /// Definitive states (off/unauthorized/unsupported) fail fast so a refresh + /// tick never stalls the discovery actor; `Unknown`/`Resetting` wait long + /// enough to cover the first-run Bluetooth permission prompt. + fn wait_for_power(state: &MacDelegateState, label: &str) -> Result<()> { + for _ in 0..300 { + match CBManagerState(state.manager_state.load(Ordering::Acquire)) { + CBManagerState::PoweredOn => return Ok(()), + CBManagerState::PoweredOff => anyhow::bail!("{label}: Bluetooth is turned off"), + CBManagerState::Unauthorized => anyhow::bail!("{label}: Bluetooth permission was denied"), + CBManagerState::Unsupported => anyhow::bail!("{label}: Bluetooth LE is not supported on this device"), + _ => thread::sleep(Duration::from_millis(50)), + } + } + anyhow::bail!("{label} did not become ready") + } + + fn service_uuid() -> Retained { + let string = NSString::from_str(&uuid::Uuid::from_u128(FLOWSTATE_SERVICE_UUID).to_string()); + unsafe { CBUUID::UUIDWithString(&string) } + } + + fn characteristic_uuid() -> Retained { + let string = NSString::from_str(&uuid::Uuid::from_u128(FLOWSTATE_ADVERTISEMENT_CHARACTERISTIC_UUID).to_string()); + unsafe { CBUUID::UUIDWithString(&string) } + } +} + +#[cfg(target_os = "macos")] +pub use macos_backend::NativeBluetoothBackend; + +/// Other targets keep the wire format shared, but must provide their native +/// CoreBluetooth/WinRT host from the application target. This explicit error +/// avoids silently pretending that nearby discovery is active. +#[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))] +pub struct NativeBluetoothBackend; + +#[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))] +impl NativeBluetoothBackend { + pub async fn new(_scan_duration: Duration) -> Result { + bail!("native Bluetooth rendezvous is not available in this build") + } +} + +#[cfg(test)] +mod tests { + use iroh::{EndpointAddr, SecretKey}; + + use crate::{SessionId, identity::PortableIdentitySecret}; + + use super::*; + + #[test] + fn gatt_framing_round_trips_signed_advertisements() { + let identity = PortableIdentitySecret::generate(); + let advertisement = DiscoveryAdvertisement::issue( + &identity, + 7, + [8; 32], + SessionId::from_bytes([9; 32]), + EndpointAddr::new(SecretKey::generate().public()), + 100, + identity.sign_profile(1, "Alex".into(), 0x334455, None), + ); + let framed = encode_gatt_advertisement(&advertisement).unwrap(); + assert_eq!(decode_gatt_advertisement(&framed).unwrap(), advertisement); + assert!(decode_gatt_advertisement(&framed[..framed.len() - 1]).is_err()); + } +} diff --git a/crates/flowstate-collab/src/crdt_runtime.rs b/crates/flowstate-collab/src/crdt_runtime.rs new file mode 100644 index 00000000..2b3d42d0 --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime.rs @@ -0,0 +1,8748 @@ +use std::{ + collections::BTreeMap, + io, + path::{Path, PathBuf}, + sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering as AtomicOrdering}, + }, +}; + +use anyhow::{Context as _, Result}; +use flowstate_document::{ + AssetId, AssetRecord, BLOCKS_BY_ID, Block, BlockId, CellId, ColumnId, DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD, DocumentPackage, + DocumentProjection, FLOW_ATTRS_KEY, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, ImportedLoroDocument, InputBlock, + InputBlockAlignment, InputEquationDisplay, InputImageSizing, InputParagraph, InputTableBlock, InputTableCell, InputTableCellBlock, + InputTableColumnWidth, MAIN_BODY_BLOCK_ID, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, + MARK_STRIKETHROUGH, OBJECT_REPLACEMENT, PARAGRAPHS_BY_ID, Paragraph, ParagraphId, ParagraphStyle, ProjectionDefect, ProjectionPatch, + ProjectionStructuralBlock, ROOT, ROOT_BODY_FLOW_ID, ROOT_FIRST_PARAGRAPH_ID, RowId, RunSemanticStyle, RunStyles, SENTINEL_NEWLINE, SectionId, + TableBlock, cell_loro_id, cell_loro_id_for, column_loro_id, document_from_loro, document_from_loro_with_defects, import_document_projection, + loro_import::assets_from_document, loro_schema::body_text, new_loro_document, row_loro_id, +}; +use loro::{ + Container, ContainerID, ExportMode, Frontiers, ID, ImportStatus, LoroDoc, LoroMap, LoroMovableList, LoroText, LoroValue, Subscription, + UndoItemMeta, ValueOrContainer, VersionRange, VersionVector, + cursor::{Cursor, Side}, + event::{Diff, DiffEvent}, +}; +// §perf act-three B.1: the vendored `loro-internal` UndoManager (re-exported by +// the wrapper as `InnerUndoManager`) — needed for the external fast-path undo +// APIs (`peek_top_span` / `external_step_*`), which the thin wrapper type does +// not surface. The wrapper's `UndoManager` is a passthrough over this exact +// type, so behavior is otherwise identical. +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use loro::InnerUndoManager as UndoManager; +use rustc_hash::{FxHashMap, FxHashSet}; +use uuid::Uuid; + +/// §P2a: commit origin stamped on canonical projection-repair batches. Excluded +/// from the local undo stack and used by peers/telemetry to identify repairs. +const REPAIR_ORIGIN: &str = "repair"; + +/// §P2a: maximum canonical-repair attempts per defect `stable_key`. A defect that +/// survives this many repair passes is quarantined (left as the deterministic +/// projection) instead of looping forever. +const PROJECTION_REPAIR_ATTEMPT_CAP: u64 = 3; + +#[path = "crdt_runtime/import_delta.rs"] +pub(crate) mod import_delta; +#[path = "crdt_runtime/projection_patch.rs"] +mod projection_patch; +#[path = "crdt_runtime/projection_repair.rs"] +mod projection_repair; +#[path = "crdt_runtime/table_ops.rs"] +pub(crate) mod table_ops; +#[path = "crdt_runtime/types.rs"] +mod types; +use crate::presence::{PresenceSelection, SelectionAffinity, SelectionDirection, SelectionEndpoint, VisualGravity}; +use gpui_flowtext::{DocumentOffset, EditorSelection, ExternalCaret, ExternalSelection, ProjectionPatchBatch, apply_projection_patch_batch}; +use loro::{ContainerTrait as _, cursor::PosType}; +pub(crate) use projection_patch::{ + body_input_paragraph_at, paragraph_projection_patches_ranged, projection_patches_between, remote_nonstructural_projection_patches, + splice_region_patches, text_delta_between, +}; + +/// A successful §6-R regional derivation: the splice patch set, the region's +/// defects, and the OLD projection's paragraph-index range the region covered +/// (so the caller can route out-of-region changes through the ranged readback). +struct RegionalPatches { + patches: Vec, + defects: Vec, + region_paragraphs: std::ops::Range, +} + +/// How a repair pass surfaces the repaired projection to the editor stream — +/// see [`CrdtRuntime::schedule_projection_repairs`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RepairEmission { + /// Push the repaired projection onto the ordered editor stream (repairs + /// running after a patched emission left the editor pre-repair). + Streamed, + /// Do not stream: an enclosing full-rebuild emission (or the absence of any + /// attached editor, at construction) carries the repaired state instead. + Silent, +} +use types::UndoSelectionState; +pub use types::{ + ProjectionFallbackStats, ProjectionInvalidation, ProjectionTextRange, RuntimeAssetMetadata, RuntimeCommentMessage, RuntimeCommentThread, + RuntimeEvent, RuntimePresenceCaretRequest, RuntimePresenceCarets, RuntimeRevisionInfo, SemanticCommand, UndoSelectionAffinity, + UndoSelectionDirection, UndoSelectionSnapshot, +}; + +#[derive(Debug)] +pub struct CrdtRuntime { + doc: LoroDoc, + projection: DocumentProjection, + projection_index: ProjectionRuntimeIndex, + undo: UndoManager, + defer_undo_checkpoints: bool, + undo_checkpoint_pending: bool, + // §15: optional durable author identity. When set, revisions record this user + // as their author; when `None`, authorship stays unset (behavior unchanged). + author_user_id: Option, + // Loro-first spec §16 (lazy author identity): identity supplied by the app, + // registered into `users_by_id`/`replicas_by_id` only when this replica + // actually commits an edit — a replica that never edits leaves no history. + pending_author_identity: Option<(u128, Option)>, + /// §16 always-on commit metrics (ops from Loro's own `ChangeMeta.len`). + ops_committed_total: Arc, + commits_total: Arc, + _pre_commit_subscription: Subscription, + package: Option, + package_path: Option, + package_journal_prepared: bool, + last_persisted_frontier: Frontiers, + last_persisted_vv: VersionVector, + undo_selection: Arc>, + subscription_events: Arc>>, + // §23: monotonic runtime epoch. Bumped on every full projection rebuild/reset so + // the permanent subscription can discard buffered summaries stamped before the + // reset instead of relying on synchronous drain timing during import/checkout. + runtime_epoch: Arc, + local_subscription_updates: Arc>>>, + projection_fallback_counts: Mutex>, + // §P2a: per-`stable_key` projection-defect repair attempt counter (mirrors + // `projection_fallback_counts`). A defect that survives `PROJECTION_REPAIR_ATTEMPT_CAP` + // repair passes is quarantined instead of spinning. + projection_repair_counts: Mutex>, + // §P2a re-entrancy guard: `schedule_projection_repairs` re-projects via + // `refresh_projection`, which itself collects defects — the guard stops that + // inner refresh from scheduling another repair pass (no infinite recursion). + repairing_projection_defects: bool, + // §act-twelve A12.1.1: refresh the replica record's `last_seen_at` on the + // FIRST local edit instead of at open (open commits invalidate the + // package's frontier-stamped caches). + pending_replica_touch: bool, + /// Field fix 2026-07-07: the ORDERED editor-bound projection stream. Every + /// projection change (local intent patches, remote import patches, full + /// replaces) is pushed here IN COMMIT ORDER under the write gate; the editor + /// drains it as its sole projection input. See + /// `gpui_flowtext::local_intents::ProjectionStreamItem`. + editor_stream: Vec, + /// Loro-first publish queue (spec §6): committed local-change events + /// (`LocalUpdate` bytes and friends) buffered under the write gate for the + /// I/O service to drain and broadcast. Local intents never block on the + /// network; the network never reaches into an intent. + pending_publish: Vec, + /// §6-R: the paragraph/block registry container ids, resolved once so the + /// subscription drain can recognize registry-map events by target and + /// harvest imported record keys for regional rematerialization. + paragraph_registry_container_id: String, + block_registry_container_id: String, + /// §act-three B.1: single-slot recorded inverse for the last qualifying mass + /// `DeleteRange` commit. Consumed (ping-ponged) by the undo/redo fast path; + /// any interleaving commit or import fails its frontier check and the slot + /// is dropped. See `local_write::recorded_inverse`. + /// §act-five P3-deep: the recorded-inverse fast path is now a STACK, not a + /// single slot — so a RUN of consecutive local undos (the Ctrl-Z mash) each + /// replay `O(change)` instead of only the first, with the rest falling to the + /// `O(doc/history)` Loro `UndoManager`. `recorded_undo` holds inverses that can + /// be fast-undone (LIFO); `recorded_redo` holds their opposites after an undo. + /// A forward edit pushes to `recorded_undo` and clears `recorded_redo`. Both + /// clear on any remote import / frontier break. FAIL-SAFE: every fast step is + /// still validated against the Loro `UndoManager`'s actual top span + /// (`peek_top_span`), so a stale entry bails to the correct slow path. + recorded_undo: Vec, + recorded_redo: Vec, + /// §act-eleven C8: count of batched-import calls (one per gate hold). + import_batches_served: u64, + _root_subscription: Subscription, + _local_update_subscription: Subscription, +} + +/// §24 style interval index entry: one run's byte span within a paragraph plus +/// its styles. `start`/`len` are byte offsets into the paragraph text. +#[derive(Clone, Copy, Debug, PartialEq)] +struct StyleInterval { + start: usize, + len: usize, + styles: RunStyles, +} + +/// §24/§P2b table row/column/cell index entry for one table block. +/// +/// The projection's [`TableBlock`] now carries durable [`RowId`]/[`ColumnId`]/ +/// [`CellId`] identifiers, so the index is keyed by those durable ids rather than +/// positional indices: `cells` maps each `(RowId, ColumnId)` coordinate to the +/// cell's deterministic [`CellId`]. This keeps the diagnostic invalidation index +/// stable under concurrent structural table edits (insert/remove/reorder). +#[derive(Clone, Debug, Default, PartialEq)] +struct TableIndexEntry { + row_ids: Vec, + column_ids: Vec, + cells: FxHashMap<(RowId, ColumnId), CellId>, +} + +/// Loro-first spec §6.3: the composed body delta drained from one +/// subscription-buffer sweep, with its conservatism flags. +#[derive(Debug, Default)] +pub(crate) struct DrainedBodyDelta { + pub(crate) net: import_delta::NetBodyDelta, + pub(crate) has_remote_origin: bool, + pub(crate) checkout_seen: bool, + /// Spec §6-R: record keys this batch wrote into (or deleted from) the + /// paragraph registry — the identity candidates a regional + /// rematerialization needs beyond the old projection's own rows. + pub(crate) imported_paragraph_record_keys: Vec, + /// Same, for the block registry (paragraph blocks and objects alike). + pub(crate) imported_block_record_keys: Vec, +} + +/// §24 search unit span: a lightweight body-unicode range for one search unit. +/// `paragraph` is `Some(ix)` for a paragraph unit and `None` for an object +/// placeholder unit. Used to map changed body ranges onto affected search units. +#[derive(Clone, Copy, Debug, PartialEq)] +struct SearchUnitSpan { + paragraph: Option, + unicode_start: usize, + unicode_len: usize, +} + +#[derive(Debug, Default)] +pub(crate) struct ProjectionRuntimeIndex { + // Original three indexes (§ earlier work): behavior and population unchanged. + paragraph_body_unicode_starts: Vec, + /// One-past-the-end unicode position of the live body flow at index-build + /// time. Used to clamp degraded (id-less) cursor resolutions, which Loro + /// resolves to the WHOLE-BODY end (spec §8 / semantics-audit F7). + body_unicode_end: usize, + paragraph_boundary_positions: Vec, + object_placeholder_positions: Vec, + // §24 additive projection indexes, all derived from the projection in + // `from_projection` and rebuilt whenever the original three are rebuilt. + /// Paragraph metadata index: paragraph id → paragraph index. + paragraph_metadata_by_id: FxHashMap, + /// Block anchor index: block id → block index in `projection.blocks`. + block_anchor_by_id: FxHashMap, + /// Table row/column/cell index: table block id → its positional cell layout. + table_cells_by_block: FxHashMap, + /// Style interval index: paragraph index → its run style intervals. + style_runs_by_paragraph: FxHashMap>, + /// Section anchor index: section id → its start paragraph id. + section_anchor_by_id: FxHashMap, + /// Asset reference index: asset id → blocks (image blocks) referencing it. + asset_refs_by_id: FxHashMap>, + /// Search unit index: per-paragraph and per-object body-unicode spans. + search_unit_spans: Vec, + /// Cursor resolution cache: encoded Loro cursor bytes → resolved document + /// offset. Memoizes `resolve_undo_cursor`. Positions shift on any edit, so the + /// cache is emptied on every rebuild (a fresh `from_projection`) and cleared in + /// `update_for_patches` for incremental updates. + cursor_resolution_cache: FxHashMap, DocumentOffset>, +} + +impl ProjectionRuntimeIndex { + /// Paragraph id → paragraph index in the projection this index was built + /// from. O(1); the identity basis of the local-write resolution law. + pub(crate) fn paragraph_index(&self, id: ParagraphId) -> Option { + self.paragraph_metadata_by_id.get(&id).copied() + } + + /// Block id → block index in the projection this index was built from. + pub(crate) fn block_index(&self, id: BlockId) -> Option { + self.block_anchor_by_id.get(&id).copied() + } + + /// Clamp a body-unicode position into the live body range recorded at index + /// build time (degraded-cursor defense, spec §8). + pub(crate) fn clamp_body_unicode(&self, pos: usize) -> usize { + pos.min(self.body_unicode_end) + } + + /// Sorted pre-state paragraph-boundary positions (spec §6.3 exact + /// structural-delete detection). + pub(crate) fn boundary_positions(&self) -> &[usize] { + &self.paragraph_boundary_positions + } + + /// Sorted pre-state object placeholder positions. + pub(crate) fn object_positions(&self) -> &[usize] { + &self.object_placeholder_positions + } + + /// Pre-state paragraph start positions (body unicode). + pub(crate) fn paragraph_starts(&self) -> &[usize] { + &self.paragraph_body_unicode_starts + } + + #[hotpath::measure] + pub(crate) fn from_projection(projection: &DocumentProjection) -> Self { + let mut index = Self::default(); + let mut body_unicode = 1usize; + let mut paragraph_ix = 0usize; + let mut has_body_content = false; + + for (block_ix, block) in projection.blocks.iter().enumerate() { + // §24 block anchor index: block id → block index. `block_ids` is parallel + // to `blocks`, so the first occurrence matches the prior `.position` scans. + if let Some(block_id) = projection.ids.block_ids.get(block_ix) { + index + .block_anchor_by_id + .entry(*block_id) + .or_insert(block_ix); + } + match block { + Block::Paragraph(paragraph) => { + if has_body_content { + index.paragraph_boundary_positions.push(body_unicode); + body_unicode = body_unicode.saturating_add(1); + } else { + index.paragraph_boundary_positions.push(0); + } + let paragraph_start = body_unicode; + index.paragraph_body_unicode_starts.push(paragraph_start); + // Count chars straight off the rope slice — materializing a String + // per paragraph (the old `paragraph_text` call) allocated the whole + // body once per index rebuild. + let byte_range = flowstate_document::paragraph_byte_range(projection, paragraph_ix); + let char_count = projection.text.byte_slice(byte_range).chars().count(); + body_unicode = body_unicode.saturating_add(char_count); + // §24 paragraph metadata index: paragraph id → paragraph index. + if let Some(paragraph_id) = projection.ids.paragraph_ids.get(paragraph_ix) { + index + .paragraph_metadata_by_id + .entry(*paragraph_id) + .or_insert(paragraph_ix); + } + // §24 style interval index: per-paragraph run intervals. + index + .style_runs_by_paragraph + .insert(paragraph_ix, style_intervals_for_paragraph(paragraph)); + // §24 search unit index: one body-text span per paragraph. + index.search_unit_spans.push(SearchUnitSpan { + paragraph: Some(paragraph_ix), + unicode_start: paragraph_start, + unicode_len: char_count, + }); + paragraph_ix = paragraph_ix.saturating_add(1); + has_body_content = true; + }, + Block::Image(image) => { + index.object_placeholder_positions.push(body_unicode); + // §24 asset reference index: asset id → referencing image block ids. + if let Some(block_id) = projection.ids.block_ids.get(block_ix) { + index + .asset_refs_by_id + .entry(image.asset_id) + .or_default() + .push(*block_id); + } + index.search_unit_spans.push(SearchUnitSpan { + paragraph: None, + unicode_start: body_unicode, + unicode_len: 1, + }); + body_unicode = body_unicode.saturating_add(1); + has_body_content = true; + }, + Block::Equation(_) => { + index.object_placeholder_positions.push(body_unicode); + index.search_unit_spans.push(SearchUnitSpan { + paragraph: None, + unicode_start: body_unicode, + unicode_len: 1, + }); + body_unicode = body_unicode.saturating_add(1); + has_body_content = true; + }, + Block::Table(table) => { + index.object_placeholder_positions.push(body_unicode); + // §24 table row/column/cell index. + if let Some(block_id) = projection.ids.block_ids.get(block_ix) { + index + .table_cells_by_block + .insert(*block_id, table_index_entry(table)); + } + index.search_unit_spans.push(SearchUnitSpan { + paragraph: None, + unicode_start: body_unicode, + unicode_len: 1, + }); + body_unicode = body_unicode.saturating_add(1); + has_body_content = true; + }, + } + } + + // §24 section anchor index: section id → start paragraph id. + for section in projection.sections.iter() { + index + .section_anchor_by_id + .entry(section.id) + .or_insert(section.start_paragraph); + } + + // FS-170: surface any object placeholder whose two caret sides collapse to the + // same document offset. Gated so a disabled build pays only one atomic load. + if fidelity::enabled() { + index.fidelity_check_object_sides(projection); + } + + index.body_unicode_end = body_unicode; + index + } + + fn body_unicode_for_offset(&self, projection: &DocumentProjection, offset: DocumentOffset) -> Option { + let paragraph = projection.paragraphs.get(offset.paragraph)?; + let paragraph_text = flowstate_document::paragraph_text(projection, offset.paragraph); + let byte = offset + .byte + .min(flowstate_document::paragraph_text_len(paragraph)); + if !paragraph_text.is_char_boundary(byte) { + return None; + } + let unicode = *self.paragraph_body_unicode_starts.get(offset.paragraph)? + paragraph_text[..byte].chars().count(); + fidelity::event(FidelityClass::Caret, "offset->unicode", || { + format!("offset={offset:?} byte={byte} -> body_unicode={unicode}") + }); + Some(unicode) + } + + /// Like [`body_unicode_for_offset`], but returns a position in the ACTUAL live Loro + /// body flow rather than the projection's coordinate space. Use this whenever the + /// result feeds a Loro body mutation or a Loro cursor: `push_flow_blocks` can + /// coalesce an object-adjacent empty paragraph out of the projection while its + /// boundary newline stays PHYSICALLY in the body, so the projection-derived index + /// runs short of the live body by one unicode per coalesced empty. Resolving the + /// paragraph's start from its durable boundary cursor keeps the position in Loro + /// space, so an edit lands where the projection intends instead of the phantom slot + /// (which would materialize the coalesced paragraph and diverge the incremental + /// projection from the full rebuild). Falls back to the projection-space start when + /// the durable record can't be resolved (e.g. the boundary-0 sentinel). + pub(crate) fn body_unicode_for_offset_in_loro(&self, doc: &LoroDoc, projection: &DocumentProjection, offset: DocumentOffset) -> Option { + let paragraph = projection.paragraphs.get(offset.paragraph)?; + let paragraph_text = flowstate_document::paragraph_text(projection, offset.paragraph); + let byte = offset + .byte + .min(flowstate_document::paragraph_text_len(paragraph)); + if !paragraph_text.is_char_boundary(byte) { + return None; + } + // §perf (measured 2026-07-07): the MAINTAINED start first, content-verified + // against the live body (same trust law as `paragraph_boundary_from_record`), + // with the durable-cursor resolution as the fallback. The old order ran + // `paragraph_body_start_in_loro` — a linear cursor chunk scan, 2.6ms on the + // reference doc — on EVERY keystroke's anchor resolution; the maintained + // index is in-sync inside the gate and the verification catches the + // coalesced-row/rot cases where it is not. + let maintained = self + .paragraph_body_unicode_starts + .get(offset.paragraph) + .copied() + .filter(|&start| body_start_matches_paragraph(doc, projection, offset.paragraph, start)); + let paragraph_start = maintained.or_else(|| { + projection + .ids + .paragraph_ids + .get(offset.paragraph) + .and_then(|paragraph_id| paragraph_body_start_in_loro(doc, *paragraph_id)) + .or_else(|| { + self + .paragraph_body_unicode_starts + .get(offset.paragraph) + .copied() + }) + })?; + let unicode = paragraph_start + paragraph_text[..byte].chars().count(); + fidelity::event(FidelityClass::Caret, "offset->unicode-loro", || { + format!("offset={offset:?} byte={byte} -> body_unicode={unicode}") + }); + Some(unicode) + } + + pub(crate) fn offset_for_body_unicode(&self, projection: &DocumentProjection, unicode: usize) -> Option { + // FS-170: the body-unicode slot immediately AFTER a block-object placeholder + // is the interstitial paragraph boundary. A caret resting there is "after the + // object" and belongs to the FOLLOWING paragraph's start — not the preceding + // paragraph's end, which the object's own slot (`position`) already resolves + // to. Without this redirect both sides of the object collapse onto the same + // offset (the `object-side-collapse` fidelity violation). Only fires when a + // paragraph actually starts just past the boundary (an object between two + // paragraphs); a trailing object falls through to the normal clamp. + if unicode > 0 && self.object_placeholder_positions.contains(&(unicode - 1)) { + let following_start = unicode + 1; + if let Ok(start_ix) = self + .paragraph_body_unicode_starts + .binary_search(&following_start) + { + let paragraph_ix = start_ix.min(projection.paragraphs.len().saturating_sub(1)); + let offset = DocumentOffset { + paragraph: paragraph_ix, + byte: 0, + }; + fidelity::event(FidelityClass::Caret, "unicode->offset-after-object", || { + format!("body_unicode={unicode} -> offset={offset:?}") + }); + return Some(offset); + } + } + let paragraph_ix = self.paragraph_at_body_unicode(unicode, projection.paragraphs.len())?; + let paragraph_start = self + .paragraph_body_unicode_starts + .get(paragraph_ix) + .copied() + .unwrap_or_default(); + let local_unicode = unicode.saturating_sub(paragraph_start); + let paragraph_text = flowstate_document::paragraph_text(projection, paragraph_ix); + let byte = paragraph_text + .char_indices() + .nth(local_unicode) + .map_or(paragraph_text.len(), |(byte, _)| byte); + let offset = DocumentOffset { + paragraph: paragraph_ix, + byte, + }; + fidelity::event(FidelityClass::Caret, "unicode->offset", || { + format!("body_unicode={unicode} -> offset={offset:?}") + }); + Some(offset) + } + + /// FS-170 diagnostic: a U+FFFC object placeholder occupies a single body-unicode + /// slot, but a caret can rest on either side of it. The slot before the object + /// (its own position) and the slot after it (`position + 1`) must resolve to + /// distinct document offsets; a collapse means carets on the two sides of the + /// object are indistinguishable. Emits the resolved offsets and, when both + /// resolve, checks they differ (kind `object-side-collapse`). Runs only when + /// fidelity tracing is enabled; read-only. + fn fidelity_check_object_sides(&self, projection: &DocumentProjection) { + for &position in &self.object_placeholder_positions { + let left = self.offset_for_body_unicode(projection, position); + let right = self.offset_for_body_unicode(projection, position.saturating_add(1)); + fidelity::event(FidelityClass::Caret, "object-side-offsets", || { + format!("placeholder_body_unicode={position} left_offset={left:?} right_offset={right:?}") + }); + if let (Some(left), Some(right)) = (left, right) { + fidelity::check(left != right, FidelityClass::Caret, "object-side-collapse", || { + format!("U+FFFC object at body-unicode {position} collapses caret sides: both map to {left:?}") + }); + } + } + } + + fn paragraphs_for_changed_ranges(&self, ranges: &[ProjectionTextRange], paragraph_count: usize, live_starts: &[usize]) -> Vec { + let mut touched = std::collections::BTreeSet::new(); + for range in ranges + .iter() + .filter(|range| range.flow_id == ROOT_BODY_FLOW_ID) + { + let start = self.paragraph_at_body_unicode_with(live_starts, range.unicode_start, paragraph_count); + let end = self.paragraph_at_body_unicode_with(live_starts, range.unicode_start.saturating_add(range.unicode_len), paragraph_count); + // Widen by one paragraph on each edge: a paragraph-STYLE mark rides the + // boundary `\n`, which text-start attribution assigns to the PRECEDING + // paragraph, and mark expand semantics can carry one position past the + // range end — either way the styled paragraph sits one slot outside the + // strict span. Over-inclusion is harmless (the ranged readback is exact + // and produces no patch for untouched paragraphs); under-inclusion left + // a stale paragraph style after an undo of a multi-paragraph restyle + // (object-fuzz undo arm). + if let Some(start) = start { + touched.insert(start.saturating_sub(1)); + touched.insert(start); + } + if let Some(end) = end { + touched.insert(end); + touched.insert((end + 1).min(paragraph_count.saturating_sub(1))); + } + if let (Some(start), Some(end)) = (start, end) { + touched.extend(start.min(end)..=start.max(end)); + } + } + touched.into_iter().collect() + } + + fn paragraph_at_body_unicode_with(&self, live_starts: &[usize], unicode: usize, paragraph_count: usize) -> Option { + // The changed-range unicode positions are in POST-import (new-body) coordinates, + // so map them against paragraph starts derived from the CURRENT Loro body, not the + // stale pre-import `paragraph_body_unicode_starts`. On the incremental path any + // boundary (`\n`) insert/delete forces a full rebuild, so the paragraph count is + // stable and `live_starts[i]` aligns with projection paragraph `i`. Falls back to + // the prebuilt index only when live starts are unavailable/mismatched. + let starts = if live_starts.len() == paragraph_count && !live_starts.is_empty() { + live_starts + } else { + self.paragraph_body_unicode_starts.as_slice() + }; + if paragraph_count == 0 || starts.is_empty() { + return None; + } + match starts.binary_search(&unicode) { + Ok(ix) => Some(ix.min(paragraph_count - 1)), + Err(0) => Some(0), + Err(ix) => Some((ix - 1).min(paragraph_count - 1)), + } + } + + fn paragraph_at_body_unicode(&self, unicode: usize, paragraph_count: usize) -> Option { + if paragraph_count == 0 || self.paragraph_body_unicode_starts.is_empty() { + return None; + } + match self.paragraph_body_unicode_starts.binary_search(&unicode) { + Ok(ix) => Some(ix.min(paragraph_count - 1)), + Err(0) => Some(0), + Err(ix) => Some((ix - 1).min(paragraph_count - 1)), + } + } + + fn deleted_range_contains_structure(&self, start: usize, len: usize) -> bool { + if len == 0 { + return false; + } + let end = start.saturating_add(len); + self + .paragraph_boundary_positions + .iter() + .chain(&self.object_placeholder_positions) + .any(|position| (start..end).contains(position)) + } + + /// §24 style interval index lookup: run styles covering `byte` in a paragraph. + fn run_styles_at(&self, paragraph_ix: usize, byte: usize) -> Option { + self + .style_runs_by_paragraph + .get(¶graph_ix)? + .iter() + .find(|interval| byte >= interval.start && byte < interval.start.saturating_add(interval.len)) + .map(|interval| interval.styles) + } + + /// §24 search unit index lookup: indices of search-unit spans overlapping any + /// body-text range in `ranges`. Mirrors `paragraphs_for_changed_ranges` but + /// resolves search units rather than paragraphs. + fn search_units_for_changed_ranges(&self, ranges: &[ProjectionTextRange]) -> Vec { + // §act-ten A10.8: search-unit spans TILE the body flow (non-overlapping, + // ascending in both start and end — paragraph spans cover their ranges + // with the boundary separators between, object spans are single slots), so + // each range's overlap window is two binary searches instead of the old + // full scan per commit. + let mut touched = Vec::new(); + for range in ranges { + if range.flow_id != ROOT_BODY_FLOW_ID { + continue; + } + let range_end = range.unicode_start.saturating_add(range.unicode_len.max(1)); + let window_start = self + .search_unit_spans + .partition_point(|span| span.unicode_start.saturating_add(span.unicode_len.max(1)) <= range.unicode_start); + let window_end = self + .search_unit_spans + .partition_point(|span| span.unicode_start < range_end); + touched.extend(window_start..window_end); + } + touched.sort_unstable(); + touched.dedup(); + touched + } + + /// Returns `(rebuild_required, table_ids_to_refresh_post_apply)`. + fn update_for_patches(&mut self, projection: &DocumentProjection, patches: &[ProjectionPatch]) -> (bool, Vec) { + // §24: resolved cursor offsets shift whenever the projection's positions + // change, so invalidate the memoized cursor cache on every incremental + // update. (Full rebuilds construct a fresh index, which starts empty.) + self.cursor_resolution_cache.clear(); + let mut text_deltas = Vec::new(); + let mut structural: Option<&ProjectionPatch> = None; + let mut table_refresh: Vec = Vec::new(); + let mut rebuild = false; + for patch in patches { + match patch { + ProjectionPatch::ParagraphText { + block_id, + paragraph_id, + row_hint, + new, + .. + } => { + let Some(paragraph_ix) = paragraph_index_for_patch(projection, *block_id, *paragraph_id, *row_hint) else { + rebuild = true; + break; + }; + let old_len = flowstate_document::paragraph_text(projection, paragraph_ix) + .chars() + .count(); + let new_len = new + .runs + .iter() + .map(|run| run.text.chars().count()) + .sum::(); + text_deltas.push((paragraph_ix, new_len as isize - old_len as isize)); + // A10.3 oracle catch (pre-existing): the incremental path left the + // edited paragraph's style intervals stale — refresh them from the + // patch payload. + let mut intervals = Vec::with_capacity(new.runs.len()); + let mut start = 0usize; + for run in &new.runs { + intervals.push(StyleInterval { + start, + len: run.text.len(), + styles: run.styles, + }); + start = start.saturating_add(run.text.len()); + } + self.style_runs_by_paragraph.insert(paragraph_ix, intervals); + }, + ProjectionPatch::InsertBlocks { .. } | ProjectionPatch::DeleteBlocks { .. } => { + // §act-ten A10.3: ONE structural patch can splice incrementally (the + // Enter/join/cross-delete shapes patch synthesis emits). A second + // structural patch, or any unresolvable shape, falls back to rebuild. + if structural.replace(patch).is_some() { + rebuild = true; + break; + } + }, + ProjectionPatch::MoveBlock { .. } => { + rebuild = true; + break; + }, + ProjectionPatch::ParagraphRuns { + block_id, + paragraph_id, + row_hint, + runs, + } => { + let Some(paragraph_ix) = paragraph_index_for_patch(projection, *block_id, *paragraph_id, *row_hint) else { + rebuild = true; + break; + }; + let mut intervals = Vec::with_capacity(runs.len()); + let mut start = 0usize; + for run in runs { + intervals.push(StyleInterval { + start, + len: run.len, + styles: run.styles, + }); + start = start.saturating_add(run.len); + } + self.style_runs_by_paragraph.insert(paragraph_ix, intervals); + }, + ProjectionPatch::ReplaceObjectBlock { block_id, .. } => { + // A10.3 oracle catches (pre-existing): a replaced TABLE left its + // `table_cells_by_block` entry stale (wrong cell resolution for the + // next local table op), and a replaced IMAGE left `asset_refs_by_id` + // pointing at the old asset. The fresh entries need the POST-apply + // block, so the caller refreshes them after the batch applies. + table_refresh.push(*block_id); + }, + ProjectionPatch::ParagraphStyle { .. } | ProjectionPatch::AssetArrived { .. } => {}, + } + } + if rebuild { + return (true, Vec::new()); + } + // §11 anti-amplification: ONE merge-walk pass per array instead of one + // full O(doc) shift pass PER TEXT PATCH — a replace-all/mass-restyle batch + // carries thousands of ParagraphText patches, which made this + // O(patches × doc). Deltas come from DISJOINT paragraphs (patch synthesis + // groups per paragraph), so each position's final shift is the prefix sum + // of the deltas of paragraphs strictly before it — same result as the + // sequential per-patch loops this replaces. + let mut text_deltas: Vec<(usize, isize)> = text_deltas + .into_iter() + .filter(|(_, delta)| *delta != 0) + .collect(); + if text_deltas.is_empty() && structural.is_none() { + return (false, table_refresh); + } + text_deltas.sort_unstable_by_key(|(paragraph_ix, _)| *paragraph_ix); + // A10.3 oracle catch (pre-existing): the merge-walk shifted every position + // array but left `body_unicode_end` stale after every text edit — the + // degraded-cursor clamp was drifting from the real body end. + let total_delta: isize = text_deltas.iter().map(|(_, delta)| *delta).sum(); + self.body_unicode_end = self.body_unicode_end.saturating_add_signed(total_delta); + // Pre-shift start values key the value-ordered arrays (placeholders, + // search spans); index-ordered arrays walk by paragraph index. + let thresholds: Vec<(usize, isize)> = text_deltas + .iter() + .map(|&(paragraph_ix, delta)| { + ( + self + .paragraph_body_unicode_starts + .get(paragraph_ix) + .copied() + .unwrap_or_default(), + delta, + ) + }) + .collect(); + { + let mut shift = 0isize; + let mut next = 0usize; + for (ix, start) in self.paragraph_body_unicode_starts.iter_mut().enumerate() { + while next < text_deltas.len() && text_deltas[next].0 < ix { + shift += text_deltas[next].1; + next += 1; + } + if shift != 0 { + *start = start.saturating_add_signed(shift); + } + } + } + { + let mut shift = 0isize; + let mut next = 0usize; + for (ix, boundary) in self.paragraph_boundary_positions.iter_mut().enumerate() { + while next < text_deltas.len() && text_deltas[next].0 < ix { + shift += text_deltas[next].1; + next += 1; + } + if shift != 0 { + *boundary = boundary.saturating_add_signed(shift); + } + } + } + { + // `object_placeholder_positions` is ascending by construction; shift + // each placeholder by the deltas whose (pre-shift) threshold lies AT or + // before it. A10.3 oracle catch (pre-existing): the old strict `<` (and + // the per-delta `> threshold` filter before it) missed an object sitting + // at the SAME position as an empty paragraph's start — typing into that + // empty paragraph left the object's cached position stale. + let mut shift = 0isize; + let mut next = 0usize; + for position in &mut self.object_placeholder_positions { + while next < thresholds.len() && thresholds[next].0 <= *position { + shift += thresholds[next].1; + next += 1; + } + if shift != 0 { + *position = position.saturating_add_signed(shift); + } + } + } + { + let mut delta_by_paragraph: FxHashMap = FxHashMap::default(); + for &(paragraph_ix, delta) in &text_deltas { + *delta_by_paragraph.entry(paragraph_ix).or_default() += delta; + } + // Spans are ascending by `unicode_start` (built in flow order). An + // OBJECT span at the same position as an edited empty paragraph's start + // must shift (same equality case as the placeholder walk above); the + // edited paragraph's OWN span must not (its start is the threshold — only + // its length changes, applied via `own_delta` below). + let mut shift = 0isize; + let mut next = 0usize; + for span in &mut self.search_unit_spans { + let bound = if span.paragraph.is_none() { + span.unicode_start.saturating_add(1) + } else { + span.unicode_start + }; + while next < thresholds.len() && thresholds[next].0 < bound { + shift += thresholds[next].1; + next += 1; + } + if shift != 0 { + span.unicode_start = span.unicode_start.saturating_add_signed(shift); + } + if let Some(own_delta) = span + .paragraph + .and_then(|paragraph_ix| delta_by_paragraph.get(¶graph_ix)) + { + span.unicode_len = span.unicode_len.saturating_add_signed(*own_delta); + } + } + } + // §act-ten A10.3: the trailing structural patch, spliced incrementally. + // Runs AFTER the text merge-walk so positions reflect the content patches + // (patch synthesis touches disjoint paragraphs: e.g. a join edits the + // surviving paragraph and deletes the removed rows). Every Enter and + // paragraph-joining Backspace used to pay a full `from_projection` here — + // O(doc chars) rope counting plus five map rebuilds on the actor thread. + if let Some(patch) = structural { + return (!self.try_splice_structural(projection, patch), table_refresh); + } + (false, table_refresh) + } + + /// §act-ten A10.3: refresh object-derived index entries from the POST-apply + /// projection (`ReplaceObjectBlock` patches). Tables refresh their cell + /// layout entry; the asset-reference map is rebuilt outright when any object + /// was replaced — an image replace can change `asset_id`, and the map's + /// per-asset vectors are block-ordered, so an in-place remove/re-add would + /// break the ordering invariant the fresh build guarantees. One O(blocks) + /// filter pass, no rope work; object replaces are rare. + fn refresh_object_entries(&mut self, projection: &DocumentProjection, replaced_ids: &[flowstate_document::BlockId]) { + if replaced_ids.is_empty() { + return; + } + for id in replaced_ids { + let Some(&row) = self.block_anchor_by_id.get(id) else { + continue; + }; + if let Some(Block::Table(table)) = projection.blocks.get(row) { + self + .table_cells_by_block + .insert(*id, table_index_entry(table)); + } + } + self.asset_refs_by_id.clear(); + for (block_ix, block) in projection.blocks.iter().enumerate() { + if let Block::Image(image) = block + && let Some(block_id) = projection.ids.block_ids.get(block_ix) + { + self + .asset_refs_by_id + .entry(image.asset_id) + .or_default() + .push(*block_id); + } + } + } + + /// §act-ten A10.3 oracle: field-by-field equality against a fresh + /// `from_projection` of the POST-patch projection. Debug builds only — + /// O(doc), exercised by the convergence/intent fuzz suites. + #[cfg(debug_assertions)] + fn debug_assert_matches_fresh(&self, projection: &DocumentProjection) { + let fresh = Self::from_projection(projection); + debug_assert_eq!( + self.paragraph_body_unicode_starts, fresh.paragraph_body_unicode_starts, + "A10.3 splice: paragraph starts diverged" + ); + debug_assert_eq!(self.body_unicode_end, fresh.body_unicode_end, "A10.3 splice: body end diverged"); + debug_assert_eq!( + self.paragraph_boundary_positions, fresh.paragraph_boundary_positions, + "A10.3 splice: boundaries diverged" + ); + debug_assert_eq!( + self.object_placeholder_positions, fresh.object_placeholder_positions, + "A10.3 splice: object positions diverged" + ); + debug_assert_eq!( + self.paragraph_metadata_by_id, fresh.paragraph_metadata_by_id, + "A10.3 splice: paragraph id index diverged" + ); + debug_assert_eq!(self.block_anchor_by_id, fresh.block_anchor_by_id, "A10.3 splice: block id index diverged"); + debug_assert_eq!( + self.table_cells_by_block, fresh.table_cells_by_block, + "A10.3 splice: table index diverged" + ); + debug_assert_eq!( + self.style_runs_by_paragraph, fresh.style_runs_by_paragraph, + "A10.3 splice: style intervals diverged" + ); + debug_assert_eq!( + self.section_anchor_by_id, fresh.section_anchor_by_id, + "A10.3 splice: section anchors diverged" + ); + debug_assert_eq!(self.asset_refs_by_id, fresh.asset_refs_by_id, "A10.3 splice: asset refs diverged"); + debug_assert_eq!(self.search_unit_spans, fresh.search_unit_spans, "A10.3 splice: search spans diverged"); + } + + /// §act-ten A10.3: incrementally splice ONE `InsertBlocks`/`DeleteBlocks` of + /// PARAGRAPH rows into the index (the shapes local split/join/cross-delete + /// synthesize). Returns `false` (caller rebuilds) for anything outside the + /// gated shape: object rows, non-contiguous deletes, doc-head edits, or an + /// object inside the ±2 row window (mirroring the editor's in-place gate — + /// outside the window the unicode math is unambiguous because neighbors are + /// paragraphs). `projection` is the PRE-patch state. + fn try_splice_structural(&mut self, projection: &DocumentProjection, patch: &ProjectionPatch) -> bool { + const SPLICE_MAX_ROWS: usize = 8; + let old_paragraph_count = projection.paragraphs.len(); + let old_block_count = projection.blocks.len(); + let object_free_window = |center: usize| { + let start = center.saturating_sub(2); + let end = (center + 2).min(old_block_count.saturating_sub(1)); + (start..=end).all(|row| matches!(projection.blocks.get(row), None | Some(Block::Paragraph(_)))) + }; + // Paragraph rank of the first row at-or-after `row` (rows in the window are + // paragraphs, so the O(log N) tree rank query applies directly). + let paragraph_rank_at = |row: usize| -> Option { + if row == old_block_count { + return Some(old_paragraph_count); + } + projection.blocks.paragraph_ix_for_block_row(row) + }; + match patch { + ProjectionPatch::InsertBlocks { before, row_hint, blocks } => { + if blocks.len() > SPLICE_MAX_ROWS || blocks.is_empty() { + return false; + } + let mut inserted: Vec<(flowstate_document::BlockId, ParagraphId, usize, Vec)> = Vec::with_capacity(blocks.len()); + for block in blocks { + let (InputBlock::Paragraph(paragraph), Some(paragraph_id)) = (&block.block, block.paragraph_id) else { + return false; + }; + let char_count: usize = paragraph + .runs + .iter() + .map(|run| run.text.chars().count()) + .sum(); + let mut intervals = Vec::with_capacity(paragraph.runs.len()); + let mut start = 0usize; + for run in ¶graph.runs { + intervals.push(StyleInterval { + start, + len: run.text.len(), + styles: run.styles, + }); + start = start.saturating_add(run.text.len()); + } + inserted.push((block.block_id, paragraph_id, char_count, intervals)); + } + let row = match before { + Some(before_id) => match self.block_anchor_by_id.get(before_id).copied() { + Some(row) => row, + None => return false, + }, + None => old_block_count, + }; + let _ = row_hint; + if !object_free_window(row) { + return false; + } + let Some(paragraph_at) = paragraph_rank_at(row) else { + return false; + }; + // Doc-head inserts hit the boundary-0 special case — leave to rebuild. + if paragraph_at == 0 || old_paragraph_count == 0 { + return false; + } + let base = match self.paragraph_body_unicode_starts.get(paragraph_at) { + Some(start) => *start, + None => self.body_unicode_end.saturating_add(1), + }; + let mut acc = 0usize; + let mut new_starts = Vec::with_capacity(inserted.len()); + let mut new_boundaries = Vec::with_capacity(inserted.len()); + for (_, _, char_count, _) in &inserted { + new_boundaries.push(base + acc - 1); + new_starts.push(base + acc); + acc += char_count + 1; + } + let shift = acc as isize; + let k = inserted.len(); + self + .paragraph_body_unicode_starts + .splice(paragraph_at..paragraph_at, new_starts.iter().copied()); + for start in &mut self.paragraph_body_unicode_starts[paragraph_at + k..] { + *start = start.saturating_add_signed(shift); + } + self + .paragraph_boundary_positions + .splice(paragraph_at..paragraph_at, new_boundaries.iter().copied()); + for boundary in &mut self.paragraph_boundary_positions[paragraph_at + k..] { + *boundary = boundary.saturating_add_signed(shift); + } + for position in &mut self.object_placeholder_positions { + if *position >= base - 1 { + *position = position.saturating_add_signed(shift); + } + } + // Suffix re-index: ids after the splice point move by k (paragraphs) + // and k rows (blocks). O(suffix) hash updates — no rope work. + for (offset, id) in projection.ids.paragraph_ids[paragraph_at.min(projection.ids.paragraph_ids.len())..] + .iter() + .enumerate() + { + self + .paragraph_metadata_by_id + .insert(*id, paragraph_at + k + offset); + } + for (offset, id) in projection.ids.block_ids[row.min(projection.ids.block_ids.len())..] + .iter() + .enumerate() + { + self.block_anchor_by_id.insert(*id, row + k + offset); + } + for ix in (paragraph_at..old_paragraph_count).rev() { + if let Some(intervals) = self.style_runs_by_paragraph.remove(&ix) { + self.style_runs_by_paragraph.insert(ix + k, intervals); + } + } + let span_from = row.min(self.search_unit_spans.len()); + for span in &mut self.search_unit_spans[span_from..] { + span.unicode_start = span.unicode_start.saturating_add_signed(shift); + if let Some(paragraph_ix) = &mut span.paragraph { + *paragraph_ix += k; + } + } + for (i, (block_id, paragraph_id, char_count, intervals)) in inserted.into_iter().enumerate() { + self + .paragraph_metadata_by_id + .insert(paragraph_id, paragraph_at + i); + self.block_anchor_by_id.insert(block_id, row + i); + self + .style_runs_by_paragraph + .insert(paragraph_at + i, intervals); + self.search_unit_spans.insert( + row + i, + SearchUnitSpan { + paragraph: Some(paragraph_at + i), + unicode_start: new_starts[i], + unicode_len: char_count, + }, + ); + } + self.body_unicode_end = self.body_unicode_end.saturating_add_signed(shift); + true + }, + ProjectionPatch::DeleteBlocks { block_ids, .. } => { + let k = block_ids.len(); + if k == 0 || k > SPLICE_MAX_ROWS { + return false; + } + let mut rows: Vec = Vec::with_capacity(k); + for id in block_ids { + match self.block_anchor_by_id.get(id).copied() { + Some(row) => rows.push(row), + None => return false, + } + } + rows.sort_unstable(); + let row = rows[0]; + if rows.windows(2).any(|pair| pair[1] != pair[0] + 1) { + return false; + } + if rows + .iter() + .any(|row| !matches!(projection.blocks.get(*row), Some(Block::Paragraph(_)))) + { + return false; + } + if !object_free_window(row) || !object_free_window(row + k - 1) { + return false; + } + let Some(paragraph_at) = paragraph_rank_at(row) else { + return false; + }; + if paragraph_at == 0 || paragraph_at + k > old_paragraph_count { + return false; + } + let Some(&lower) = self.paragraph_body_unicode_starts.get(paragraph_at) else { + return false; + }; + let upper = match self.paragraph_body_unicode_starts.get(paragraph_at + k) { + Some(start) => *start, + None => self.body_unicode_end.saturating_add(1), + }; + let shift = (upper - lower) as isize; + self + .paragraph_body_unicode_starts + .drain(paragraph_at..paragraph_at + k); + for start in &mut self.paragraph_body_unicode_starts[paragraph_at..] { + *start = start.saturating_add_signed(-shift); + } + self + .paragraph_boundary_positions + .drain(paragraph_at..paragraph_at + k); + for boundary in &mut self.paragraph_boundary_positions[paragraph_at..] { + *boundary = boundary.saturating_add_signed(-shift); + } + for position in &mut self.object_placeholder_positions { + if *position >= lower { + *position = position.saturating_add_signed(-shift); + } + } + for id in &projection.ids.paragraph_ids[paragraph_at..(paragraph_at + k).min(projection.ids.paragraph_ids.len())] { + self.paragraph_metadata_by_id.remove(id); + } + for (offset, id) in projection.ids.paragraph_ids[(paragraph_at + k).min(projection.ids.paragraph_ids.len())..] + .iter() + .enumerate() + { + self + .paragraph_metadata_by_id + .insert(*id, paragraph_at + offset); + } + for id in block_ids { + self.block_anchor_by_id.remove(id); + } + for (offset, id) in projection.ids.block_ids[(row + k).min(projection.ids.block_ids.len())..] + .iter() + .enumerate() + { + self.block_anchor_by_id.insert(*id, row + offset); + } + for ix in paragraph_at..paragraph_at + k { + self.style_runs_by_paragraph.remove(&ix); + } + for ix in paragraph_at + k..old_paragraph_count { + if let Some(intervals) = self.style_runs_by_paragraph.remove(&ix) { + self.style_runs_by_paragraph.insert(ix - k, intervals); + } + } + self + .search_unit_spans + .drain(row..(row + k).min(self.search_unit_spans.len())); + let span_from = row.min(self.search_unit_spans.len()); + for span in &mut self.search_unit_spans[span_from..] { + span.unicode_start = span.unicode_start.saturating_add_signed(-shift); + if let Some(paragraph_ix) = &mut span.paragraph { + *paragraph_ix -= k; + } + } + self.body_unicode_end = self.body_unicode_end.saturating_add_signed(-shift); + true + }, + _ => false, + } + } +} + +fn paragraph_index_for_block_row(projection: &DocumentProjection, row: usize) -> Option { + // §act-ten A10.8: O(log N) tree rank (was an O(row) block walk per patch). + projection.blocks.paragraph_ix_for_block_row(row) +} + +fn paragraph_index_for_patch( + projection: &DocumentProjection, + block_id: flowstate_document::BlockId, + paragraph_id: flowstate_document::ParagraphId, + row_hint: usize, +) -> Option { + let row = if projection.ids.block_ids.get(row_hint).copied() == Some(block_id) { + row_hint + } else { + projection + .ids + .block_ids + .iter() + .position(|id| *id == block_id)? + }; + let paragraph_ix = paragraph_index_for_block_row(projection, row)?; + (projection.ids.paragraph_ids.get(paragraph_ix).copied() == Some(paragraph_id)).then_some(paragraph_ix) +} + +/// §24 style interval index builder: maps a paragraph's runs to byte-offset +/// `{start, len, styles}` intervals. Run lengths are byte lengths, matching the +/// projection's `TextRun::len`. +fn style_intervals_for_paragraph(paragraph: &Paragraph) -> Vec { + let mut intervals = Vec::with_capacity(paragraph.runs.len()); + let mut start = 0usize; + for run in ¶graph.runs { + intervals.push(StyleInterval { + start, + len: run.len, + styles: run.styles, + }); + start = start.saturating_add(run.len); + } + intervals +} + +/// §24/§P2b table row/column/cell index builder, keyed by the model's durable +/// [`RowId`]/[`ColumnId`]/[`CellId`] identifiers so the index survives concurrent +/// structural table edits. +fn table_index_entry(table: &TableBlock) -> TableIndexEntry { + let mut cells = FxHashMap::default(); + for row in &table.rows { + for cell in &row.cells { + cells.insert((cell.row_id, cell.column_id), cell.id); + } + } + TableIndexEntry { + row_ids: table.rows.iter().map(|row| row.id).collect(), + column_ids: table.columns.iter().map(|column| column.id).collect(), + cells, + } +} + +impl CrdtRuntime { + pub fn new_empty(title: &str) -> Result { + let doc = new_loro_document(title).context("initializing Loro document")?; + Self::from_doc(doc, None, None) + } + + pub fn open_package(path: impl AsRef) -> Result { + // §act-twelve A12.1.0 (`FLOWSTATE_OPEN_PROBE=1`): stage timings for the + // cold-open pipeline — where the seconds actually sit, not where they + // sat before patch #18/#24. + static OPEN_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + let probe = *OPEN_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_OPEN_PROBE").is_some()); + let path = path.as_ref(); + let t0 = std::time::Instant::now(); + let package = DocumentPackage::read(path).with_context(|| format!("reading Flowstate package {}", path.display()))?; + let t_read = t0.elapsed(); + let t1 = std::time::Instant::now(); + let projection = package + .current_projection_document() + .context("reading frontier-matched package projection cache")?; + let t_cache = t1.elapsed(); + let cache_hit = projection.is_some(); + // `read` just validated the package; the plain `load_loro_doc` would run + // a SECOND full validate (re-hashing every segment + asset) on every open. + let t2 = std::time::Instant::now(); + // §act-twelve A12.1.3 (default ON): decode the shallow accelerator chunk + // instead of the full history when the package carries one. Every + // anomaly — no chunk, stale chunk, a segment the shallow state cannot + // accept — falls back to the full decode below. + static SHALLOW_OPEN: std::sync::OnceLock = std::sync::OnceLock::new(); + let shallow_armed = *SHALLOW_OPEN.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_SHALLOW_OPEN").is_none()); + let mut opened_shallow = false; + let shallow_doc = if shallow_armed { + package.load_loro_doc_shallow().unwrap_or_else(|error| { + eprintln!("[flowstate-shallow-open] shallow decode failed; falling back to full history: {error}"); + None + }) + } else { + None + }; + let doc = match shallow_doc { + Some(doc) => { + opened_shallow = true; + doc + }, + None => package + .load_loro_doc_from_validated() + .context("loading Loro document from package")?, + }; + let t_doc = t2.elapsed(); + let t3 = std::time::Instant::now(); + let mut runtime = Self::from_doc_with_projection(doc, Some(package), Some(path.to_path_buf()), projection)?; + runtime.package_journal_prepared = true; + if probe { + eprintln!( + "[flowstate-open-probe] read={t_read:?} projection_cache={t_cache:?} (hit={cache_hit}) loro_doc={t_doc:?} (shallow={opened_shallow}) runtime_construct={:?} total={:?}", + t3.elapsed(), + t0.elapsed() + ); + } + Ok(runtime) + } + + pub fn from_package(package: DocumentPackage, package_path: Option) -> Result { + let projection = package + .current_projection_document() + .context("reading frontier-matched package projection cache")?; + let doc = package + .load_loro_doc() + .context("loading Loro document from package")?; + Self::from_doc_with_projection(doc, Some(package), package_path, projection) + } + + pub fn from_document_projection(document: &DocumentProjection, title: &str) -> Result { + let imported = import_document_projection(document.clone(), title).context("importing projected document into canonical Loro runtime")?; + Self::from_imported_document(imported) + } + + pub fn from_imported_document(imported: ImportedLoroDocument) -> Result { + let ImportedLoroDocument { doc, projection } = imported; + Self::from_doc_with_projection_options(doc, None, None, Some(projection), false) + } + + pub fn from_doc(doc: LoroDoc, package: Option, package_path: Option) -> Result { + Self::from_doc_with_projection(doc, package, package_path, None) + } + + fn from_doc_with_projection( + doc: LoroDoc, + package: Option, + package_path: Option, + projection: Option, + ) -> Result { + Self::from_doc_with_projection_options(doc, package, package_path, projection, true) + } + + fn from_doc_with_projection_options( + doc: LoroDoc, + mut package: Option, + package_path: Option, + projection: Option, + repair_paragraph_style_marks: bool, + ) -> Result { + // Every `CrdtRuntime` marks paragraph/run styles on its canonical body text, + // and `text.mark(..)` errors ("Style configuration missing") unless the doc + // has been style-configured. `new_loro_document` and the package loaders + // configure the docs they build, but a runtime can also be constructed from a + // bare `LoroDoc` received over the network or restored elsewhere; guarantee + // the invariant here so no construction path can leave marking broken. The + // call is idempotent, so re-configuring an already-configured doc is a no-op. + static OPEN_PROBE2: std::sync::OnceLock = std::sync::OnceLock::new(); + let open_probe = *OPEN_PROBE2.get_or_init(|| std::env::var_os("FLOWSTATE_OPEN_PROBE").is_some()); + let probe_t0 = std::time::Instant::now(); + flowstate_document::loro_schema::configure_text_styles(&doc); + let frontier_before_startup_metadata = doc.state_frontiers().encode(); + let projection_content_repaired = if repair_paragraph_style_marks { + persist_body_paragraph_style_mark_repair(&doc, package.as_mut(), package_path.as_deref())? + } else { + // Trusted import builders apply every paragraph boundary mark while constructing + // the body. Avoid materializing a full rich-text delta only to verify it again. + flowstate_document::register_replica(&doc, None)?; + false + }; + let current_frontier = doc.state_frontiers().encode(); + // §P2a: projecting from canonical Loro reports defects; the two trusted-input + // arms below skip projection, so they contribute none. + let mut startup_defects: Vec = Vec::new(); + let probe_t_repair = probe_t0.elapsed(); + let probe_t1 = std::time::Instant::now(); + let mut projection = match projection { + Some(projection) if projection.frontier == current_frontier => projection, + Some(mut projection) if !projection_content_repaired && projection.frontier == frontier_before_startup_metadata => { + projection.frontier.clone_from(¤t_frontier); + projection + }, + None => { + let (projection, defects) = document_from_loro_with_defects(&doc).context("building initial projection from canonical Loro state")?; + startup_defects = defects; + projection + }, + Some(_) => { + let (projection, defects) = document_from_loro_with_defects(&doc).context("rebuilding stale initial projection")?; + startup_defects = defects; + projection + }, + }; + let probe_t_projection = probe_t1.elapsed(); + if open_probe { + eprintln!("[flowstate-open-probe] construct: repair_scan={probe_t_repair:?} projection_decide={probe_t_projection:?}"); + } + if let Some(package) = &package { + attach_package_assets(&mut projection, package); + } + let last_persisted_frontier = doc.state_frontiers(); + let last_persisted_vv = doc.state_vv(); + let subscription_events = Arc::new(Mutex::new(Vec::new())); + let subscription_events_for_callback = Arc::clone(&subscription_events); + let runtime_epoch = Arc::new(AtomicU64::new(0)); + let runtime_epoch_for_callback = Arc::clone(&runtime_epoch); + // §23: the callback owns a reference clone of the document (a shared handle, not a + // deep fork) so it can stamp the live state frontier on every summary. This is + // deadlock-safe: Loro releases the doc state lock before dispatching observers + // (see `emit_events`: "we should not hold the lock when emitting events"), and + // `state_frontiers()` only takes that same lock briefly to clone the frontier. + let doc_for_callback = doc.clone(); + let root_subscription = doc.subscribe_root(Arc::new(move |event: DiffEvent<'_>| { + let mut summary = summarize_subscription_event(&event); + // §23: stamp the runtime epoch and the emit-time frontier. Summaries stamped + // before a full rebuild (different epoch) or ahead of the drain target (later + // frontier) are filtered in `merge_subscription_invalidation`. + summary.epoch = runtime_epoch_for_callback.load(AtomicOrdering::SeqCst); + summary.frontier = doc_for_callback.state_frontiers().encode(); + tracing::trace!( + origin = %summary.origin, + trigger = %summary.triggered_by, + epoch = summary.epoch, + frontier_len = summary.frontier.len(), + changes = summary.changes.len(), + "Flowstate Loro root event", + ); + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + " subscription: origin {} trigger {} epoch {} changes {}", + summary.origin, + summary.triggered_by, + summary.epoch, + summary.changes.len() + ); + } + if let Ok(mut events) = subscription_events_for_callback.lock() { + events.push(summary); + } + })); + // Loro-first spec §16: source-of-truth ops-per-commit tap. `ChangeMeta.len` + // is measured by Loro itself at commit time — immune to instrumentation + // drift. Callback discipline per I-9b: copy the numbers out, touch nothing. + let ops_committed_total = Arc::new(AtomicU64::new(0)); + let commits_total = Arc::new(AtomicU64::new(0)); + let ops_for_callback = Arc::clone(&ops_committed_total); + let commits_for_callback = Arc::clone(&commits_total); + let pre_commit_subscription = doc.subscribe_pre_commit(Box::new(move |payload| { + ops_for_callback.fetch_add(payload.change_meta.len as u64, AtomicOrdering::Relaxed); + commits_for_callback.fetch_add(1, AtomicOrdering::Relaxed); + true + })); + let local_subscription_updates = Arc::new(Mutex::new(Vec::new())); + let local_updates_for_callback = Arc::clone(&local_subscription_updates); + let local_update_subscription = doc.subscribe_local_update(Box::new(move |bytes| { + tracing::trace!(bytes = bytes.len(), "Flowstate Loro local update"); + if let Ok(mut updates) = local_updates_for_callback.lock() { + updates.push(bytes.clone()); + } + true + })); + let undo = UndoManager::new(doc.inner()); + // Loro-first spec §10 (D6): time-based merge OFF — undo units are explicit + // group_start/group_end boundaries driven by input semantics. + undo.set_merge_interval(0); + undo.set_max_undo_steps(300); + undo.add_exclude_origin_prefix("remote"); + // §P2a: canonical repair commits carry the `repair` origin and must never + // enter the local undo stack (they are convergent housekeeping, not edits). + undo.add_exclude_origin_prefix("repair"); + // §act-ten A10.2 (vendor patch #14): metadata/revision commits (autosave + // checkpoints) are INERT for undo — they must not create undo steps, and + // must not mark the stacks dirty either (the plain excluded-origin path + // composes them as remote-like diffs, flipping `peek_top_span`'s `clean` + // flag and forcing the O(doc) slow path after every autosave). Sound + // because "meta" commits write only meta/revisions containers, never + // content an undo item references. + undo.add_inert_origin_prefix("meta"); + let undo_selection = Arc::new(Mutex::new(UndoSelectionState::default())); + install_undo_selection_callbacks(&undo, &undo_selection); + let projection_index = ProjectionRuntimeIndex::from_projection(&projection); + // §6-R: registry container ids, read-only (no `ensure_*` — construction + // must not leave uncommitted marker writes in the pending txn). + let registry_root = doc.get_map(ROOT); + let paragraph_registry_container_id = child_map(®istry_root, PARAGRAPHS_BY_ID) + .map(|map| map.id().to_string()) + .unwrap_or_default(); + let block_registry_container_id = child_map(®istry_root, BLOCKS_BY_ID) + .map(|map| map.id().to_string()) + .unwrap_or_default(); + let mut runtime = Self { + doc, + projection, + projection_index, + undo, + defer_undo_checkpoints: false, + undo_checkpoint_pending: false, + author_user_id: None, + pending_author_identity: None, + ops_committed_total, + commits_total, + _pre_commit_subscription: pre_commit_subscription, + package, + package_path, + package_journal_prepared: false, + last_persisted_frontier, + last_persisted_vv, + undo_selection, + subscription_events, + runtime_epoch, + local_subscription_updates, + projection_fallback_counts: Mutex::new(BTreeMap::new()), + projection_repair_counts: Mutex::new(BTreeMap::new()), + repairing_projection_defects: false, + pending_replica_touch: true, + editor_stream: Vec::new(), + pending_publish: Vec::new(), + paragraph_registry_container_id, + block_registry_container_id, + recorded_undo: Vec::new(), + recorded_redo: Vec::new(), + import_batches_served: 0, + _root_subscription: root_subscription, + _local_update_subscription: local_update_subscription, + }; + // §P2a: repair any malformed canonical state the initial projection reported. + // At construction there is no peer channel to emit onto, but the repair is + // committed + persisted, so peers receive it via the update segment / next + // anti-entropy pull. Errors are logged, never fatal to opening the document. + if !startup_defects.is_empty() + && let Err(error) = runtime.schedule_projection_repairs(startup_defects, RepairEmission::Silent) + { + tracing::error!(%error, "scheduling projection repairs during runtime construction failed"); + } + Ok(runtime) + } + + /// The live doc. Callers must hold the write gate for any barrier-capable + /// use (spec I-9a); integration tests access it through their gate guards. + pub fn doc(&self) -> &LoroDoc { + &self.doc + } + + /// §16 commit metrics: (total commits, total Loro ops) recorded at the + /// pre-commit tap since this core was constructed. + #[must_use] + pub fn commit_metrics(&self) -> (u64, u64) { + ( + self.commits_total.load(AtomicOrdering::Relaxed), + self.ops_committed_total.load(AtomicOrdering::Relaxed), + ) + } + + // ---- Loro-first local-write core surface (spec §4/§6) -------------------- + + pub(crate) fn projection_ref(&self) -> &DocumentProjection { + &self.projection + } + + pub(crate) fn projection_index_ref(&self) -> &ProjectionRuntimeIndex { + &self.projection_index + } + + pub(crate) fn set_projection_frontier(&mut self, frontier: Vec) { + self.projection.frontier = frontier; + } + + pub(crate) fn undo_manager_mut(&mut self) -> &mut UndoManager { + &mut self.undo + } + + /// §act-three B.1 / §act-five P3-deep: the recorded-inverse fast-path stacks + /// (see `local_write::recorded_inverse`). + pub(crate) fn recorded_undo_stack(&mut self) -> &mut Vec { + &mut self.recorded_undo + } + + pub(crate) fn recorded_redo_stack(&mut self) -> &mut Vec { + &mut self.recorded_redo + } + + /// §act-eleven C8: batched-import calls served (one per gate hold). Test + /// observability for the `doc_io` coalescing bound. + #[must_use] + pub fn import_batches_served(&self) -> u64 { + self.import_batches_served + } + + /// Drop the whole fast-path cache — a frontier break (remote import, checkout, + /// or a slow-path step) invalidates every recorded inverse at once. + pub(crate) fn clear_recorded_inverse(&mut self) { + self.recorded_undo.clear(); + self.recorded_redo.clear(); + } + + /// §oom-leads #9 (mass-op collab): decide the recorded fast-path stacks' fate + /// after an APPLYING remote import — rebase them through hull-disjoint remote + /// changes instead of the old unconditional clear (which made the O(change) + /// undo fast path effectively dead under live collab: any partner keystroke + /// between an edit and Ctrl-Z forced the O(doc/history) slow path — measured + /// seconds-to-minutes on large docs). Clears for anything the disjointness + /// proof cannot cover: checkouts, structural churn, unclassifiable diffs, + /// section-map changes, or remote body changes at/after the recorded hull. + fn reconcile_recorded_inverse_after_remote(&mut self, invalidation: &ProjectionInvalidation, drained: &DrainedBodyDelta) { + if self.recorded_undo.is_empty() && self.recorded_redo.is_empty() { + return; + } + let unrebasable = drained.checkout_seen + || drained.net.structural_churn + || !invalidation.changed_sections.is_empty() + || matches!( + invalidation.fallback_reason, + Some("checkout_trigger_projection_rebuild" | "unknown_loro_subscription_diff") + ); + if unrebasable { + self.clear_recorded_inverse(); + return; + } + let body_ranges_post: Vec<(usize, usize)> = invalidation + .changed_text_ranges + .iter() + .filter(|range| range.flow_id == ROOT_BODY_FLOW_ID) + .map(|range| (range.unicode_start, range.unicode_len)) + .collect(); + // Same structure law the derive ladder classifies by: the import neither + // inserted a boundary/placeholder nor deleted a pre-existing boundary or + // object placeholder (any of those changes the projection's ROW set, which + // pre-recorded patches index into). + let structure_neutral = !drained.net.inserts_structure() + && !drained + .net + .deletes_any_position(self.projection_index.boundary_positions()) + && !drained + .net + .deletes_any_position(self.projection_index.object_positions()); + let net = drained.net.clone(); + if !crate::local_write::recorded_inverse::rebase_recorded_inverse_through_remote(self, &net, &body_ranges_post, structure_neutral) { + tracing::debug!("recorded-inverse stacks cleared by remote import (rebase declined)"); + } + } + + /// Push an editor-bound projection change onto the ordered stream (gate-held + /// callers only — ordering is the whole point). + pub(crate) fn push_editor_stream(&mut self, item: gpui_flowtext::ProjectionStreamItem) { + self.editor_stream.push(item); + } + + /// Drain the ordered projection stream for the editor (single consumer). + pub(crate) fn take_editor_stream(&mut self) -> Vec { + std::mem::take(&mut self.editor_stream) + } + + /// Queue committed-change events for the I/O service to drain (spec §6). + pub(crate) fn queue_publish(&mut self, events: Vec) { + self.pending_publish.extend(events); + } + + /// Drain queued publish events. Called by the I/O service under the gate. + pub(crate) fn take_pending_publish(&mut self) -> Vec { + std::mem::take(&mut self.pending_publish) + } + + /// Spec §7 audit law: the patch-applied projection must equal an independent + /// full materialization (modulo the documented whitelist, which is embodied + /// by `projections_semantically_equal`). On mismatch: loud metric + snapshot + /// repair by installing the full rebuild. + #[hotpath::measure] + pub(crate) fn audit_projection_against_full_rebuild(&mut self, class: &str) -> Result<()> { + let full = document_from_loro(&self.doc).context("audit: full materialization failed")?; + if projections_semantically_equal(&self.projection, &full) { + return Ok(()); + } + let detail = first_projection_divergence(&self.projection, &full); + tracing::error!( + class, + detail, + "audit-mismatch: patch-applied projection != full rebuild; installing snapshot repair" + ); + if let Ok(mut counts) = self.projection_fallback_counts.lock() { + *counts.entry("audit-mismatch".to_string()).or_default() += 1; + } + let frontier = self.projection.frontier.clone(); + self.projection = full; + self.projection.frontier = frontier; + self.projection_index = ProjectionRuntimeIndex::from_projection(&self.projection); + anyhow::bail!("projection audit mismatch for intent class {class}: {detail}") + } + + /// §15: bind a stable durable author identity to this live replica. + /// + /// Registers (or refreshes) the user record in `users_by_id`, links the current + /// Loro replica to that user, and stores the id so later revisions record it as + /// their author. Until this is called, `author_user_id` stays `None` and + /// authorship is left unset (behavior unchanged). + pub fn set_author_identity(&mut self, user_id: u128, display_name: Option) -> Result> { + // Loro-first spec §16: registration is LAZY — deferred to this replica's + // first actual edit so an open-and-look session commits no identity noise. + self.author_user_id = Some(user_id); + self.pending_author_identity = Some((user_id, display_name)); + Ok(Vec::new()) + } + + /// Register the deferred author identity (spec §16). Called by the local + /// write path right after this replica's first committed edit; the identity + /// commit rides the excluded `repair` origin so it never enters undo stacks. + pub(crate) fn register_pending_author_identity(&mut self) { + let identity = self.pending_author_identity.take(); + // §act-twelve A12.1.1: the replica `last_seen_at` touch is DEFERRED from + // open to the first real edit (an open-time commit invalidated the + // package's frontier-stamped caches for every later open). + let replica_touch = std::mem::take(&mut self.pending_replica_touch); + if identity.is_none() && !replica_touch { + return; + } + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + // §act-twelve A12.1.1: the INERT "meta" origin (vendor #14), not the + // excluded "repair" one — an excluded-origin commit composes into the + // Loro undo stacks as a remote-like diff and flips `clean` for the whole + // session, which silently killed the recorded-inverse fast path on every + // doc once the replica touch moved here (fast undo died after the FIRST + // edit). Identity/replica writes only touch meta-class containers, so + // inert is sound; the recorded stacks' frontier tops are re-armed below, + // mirroring the autosave checkpoint (A10.2). + self.doc.set_next_commit_origin("meta"); + self.doc.set_next_commit_message("author-identity"); + if replica_touch && let Err(error) = flowstate_document::register_replica(&self.doc, None) { + tracing::warn!(%error, "deferred replica registration failed"); + } + if let Some((user_id, display_name)) = identity + && let Err(error) = flowstate_document::register_user(&self.doc, user_id, display_name.as_deref()) + { + tracing::warn!(%error, "registering deferred author identity failed"); + return; + } + let frontier_before_meta = self.projection.frontier.clone(); + match self.events_after_local_change(from_frontier, from_vv, ProjectionInvalidation::default(), false) { + Ok(events) => self.queue_publish(events), + Err(error) => tracing::warn!(%error, "publishing deferred author-identity update failed; anti-entropy recovers it"), + } + // The meta commit advanced the frontier: (a) keep the recorded fast-path + // tops armed across it (A10.2 law); (b) advance the maintained projection + // frontier and stream an EMPTY patch batch so the editor's base-frontier + // chain stays gapless (the ordered-stream field bug class — the #40 + // neutral-repair emission, same law). + crate::local_write::recorded_inverse::rearm_recorded_inverse_frontier(self); + self.projection.frontier = self.doc.state_frontiers().encode(); + let invalidation = ProjectionInvalidation { + frontier_before: frontier_before_meta, + frontier_after: self.projection.frontier.clone(), + ..Default::default() + }; + let empty: std::sync::Arc<[ProjectionPatch]> = std::sync::Arc::from(Vec::new()); + let event = self.projection_patched_event(empty, invalidation); + self.queue_publish(vec![event]); + } + + pub fn set_pending_undo_selection(&mut self, selection: Option) -> Result<()> { + let pending_selection = selection + .map(|selection| postcard::to_stdvec(&selection).context("encoding undo selection snapshot failed")) + .transpose()?; + if let Ok(mut state) = self.undo_selection.lock() { + state.pending_selection = pending_selection; + } + Ok(()) + } + + pub fn take_restored_undo_selection(&mut self) -> Option { + self + .undo_selection + .lock() + .ok() + .and_then(|mut state| state.restored_selection.take()) + } + + /// Re-park a restored-selection snapshot that could not resolve against the + /// current projection yet (mirrors the slow undo path's deferral). + pub(crate) fn restore_undo_selection_later(&mut self, snapshot: UndoSelectionSnapshot) { + if let Ok(mut state) = self.undo_selection.lock() { + state.restored_selection = Some(snapshot); + } + } + + #[hotpath::measure] + pub(crate) fn record_undo_checkpoint(&mut self) -> Result<()> { + if self.defer_undo_checkpoints { + self.undo_checkpoint_pending = true; + return Ok(()); + } + self + .undo + .record_new_checkpoint() + .context("recording Loro undo checkpoint")?; + Ok(()) + } + + pub fn projection_snapshot(&self) -> Result { + Ok(self.projection.clone()) + } + + pub fn asset_metadata(&self) -> Result> { + let root = self.doc.get_map(ROOT); + let Some(ValueOrContainer::Container(Container::Map(assets_by_id))) = root.get(flowstate_document::loro_schema::ASSETS_BY_ID) else { + return Ok(Vec::new()); + }; + let mut assets = Vec::new(); + for key in assets_by_id.keys() { + let Some(ValueOrContainer::Container(Container::Map(map))) = assets_by_id.get(&key) else { + continue; + }; + let Some(asset_id) = map_string_opt(&map, "asset_id").and_then(|value| value.parse::().ok()) else { + continue; + }; + let byte_length = map_i64_opt(&map, "byte_length").unwrap_or_default().max(0) as u64; + let Some(content_hash) = map_string_opt(&map, "content_hash").and_then(|hash| parse_blake3_hex(&hash)) else { + tracing::warn!(asset_id, "ignoring asset metadata with an invalid BLAKE3 digest"); + continue; + }; + if byte_length == 0 { + continue; + } + assets.push(RuntimeAssetMetadata { + asset_id, + content_hash, + mime_type: map_string_opt(&map, "mime_type").unwrap_or_else(|| "application/octet-stream".to_string()), + original_name: map_string_opt(&map, "original_name"), + byte_length, + }); + } + Ok(assets) + } + + pub fn revisions(&self) -> Vec { + let root = flowstate_document::loro_schema::root_map(&self.doc); + let users = root.get(flowstate_document::loro_schema::USERS_BY_ID); + self + .package + .as_ref() + .map(|package| { + package + .revisions + .iter() + .rev() + .map(|revision| { + let author_display_name = revision.author_user_id.and_then(|user_id| { + let ValueOrContainer::Container(Container::Map(users)) = users.as_ref()? else { + return None; + }; + let Some(ValueOrContainer::Container(Container::Map(user))) = users.get(&user_id.to_string()) else { + return None; + }; + map_string_opt(&user, "display_name") + }); + RuntimeRevisionInfo { + revision_id: revision.revision_id, + title: revision.title.clone(), + summary: revision.summary.clone(), + created_at_unix_secs: revision.created_at_unix_secs, + author_user_id: revision.author_user_id, + author_display_name, + replica_id: revision.replica_id, + } + }) + .collect() + }) + .unwrap_or_default() + } + + /// Materialize durable comment threads and resolve their CRDT anchors + /// against the current document. A deleted anchor remains discoverable with + /// `anchor: None` and its quoted-text fallback intact. + pub fn comments(&self) -> Vec { + let root = flowstate_document::loro_schema::root_map(&self.doc); + let Some(ValueOrContainer::Container(Container::Map(comments))) = root.get(flowstate_document::COMMENTS_BY_ID) else { + return Vec::new(); + }; + let mut threads = Vec::new(); + comments.for_each(|key, value| { + let ValueOrContainer::Container(Container::Map(thread)) = value else { + return; + }; + let Some(comment_id) = key.parse::().ok() else { + return; + }; + let head_cursor = map_binary_opt(&thread, "head_cursor"); + let anchor_cursor = map_binary_opt(&thread, "anchor_cursor"); + let anchor = head_cursor + .zip(anchor_cursor) + .and_then(|(head, anchor)| self.resolve_selection_anchor(&head, &anchor)) + .map(|(head, anchor)| if anchor <= head { (anchor, head) } else { (head, anchor) }) + .filter(|(start, end)| start != end); + let mut messages = Vec::new(); + if let Some(ValueOrContainer::Container(Container::Map(message_map))) = thread.get("messages_by_id") { + message_map.for_each(|message_key, value| { + let ValueOrContainer::Container(Container::Map(message)) = value else { + return; + }; + let Some(message_id) = message_key.parse().ok() else { + return; + }; + let Some(author_user_id) = map_string_opt(&message, "author_user_id").and_then(|id| id.parse().ok()) else { + return; + }; + messages.push(RuntimeCommentMessage { + message_id, + author_user_id, + author_display_name: map_string_opt(&message, "author_display_name").unwrap_or_else(|| "Unknown author".into()), + body: map_string_opt(&message, "body").unwrap_or_default(), + created_at_unix_secs: map_i64_opt(&message, "created_at").unwrap_or_default(), + updated_at_unix_secs: map_i64_opt(&message, "updated_at").unwrap_or_default(), + }); + }); + } + messages.sort_by_key(|message| (message.created_at_unix_secs, message.message_id)); + threads.push(RuntimeCommentThread { + comment_id, + quoted_text: map_string_opt(&thread, "quoted_text").unwrap_or_default(), + resolved: map_bool_opt(&thread, "resolved").unwrap_or(false), + created_at_unix_secs: map_i64_opt(&thread, "created_at").unwrap_or_default(), + updated_at_unix_secs: map_i64_opt(&thread, "updated_at").unwrap_or_default(), + anchor, + messages, + }); + }); + threads.sort_by_key(|thread| (thread.resolved, thread.created_at_unix_secs, thread.comment_id)); + threads + } + + pub fn create_comment(&mut self, selection: &EditorSelection, body: &str, author_user_id: u128, author_display_name: &str) -> Result { + let body = validated_comment_body(body)?; + anyhow::ensure!(selection.anchor != selection.head, "Select text before adding a comment"); + self.register_pending_author_identity(); + let (head_cursor, anchor_cursor) = self + .encode_selection_anchor(selection) + .context("The selected text cannot be anchored in this document")?; + let start = gpui_flowtext::global_byte(&self.projection, selection.anchor.min(selection.head)); + let end = gpui_flowtext::global_byte(&self.projection, selection.anchor.max(selection.head)); + let quoted_text = gpui_flowtext::document_text_slice(&self.projection, start..end); + let comment_id = Uuid::new_v4().as_u128(); + let message_id = Uuid::new_v4().as_u128(); + let now = unix_time_secs(); + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + flowstate_document::touch_document_metadata(&self.doc)?; + let comments = flowstate_document::loro_schema::root_map(&self.doc).ensure_mergeable_map(flowstate_document::COMMENTS_BY_ID)?; + let thread = comments.ensure_mergeable_map(&comment_id.to_string())?; + thread.insert("id", comment_id.to_string())?; + thread.insert("author_user_id", author_user_id.to_string())?; + thread.insert("head_cursor", LoroValue::Binary(head_cursor.into()))?; + thread.insert("anchor_cursor", LoroValue::Binary(anchor_cursor.into()))?; + thread.insert("quoted_text", quoted_text)?; + thread.insert("resolved", false)?; + thread.insert("created_at", now)?; + thread.insert("updated_at", now)?; + let messages = thread.ensure_mergeable_map("messages_by_id")?; + write_comment_message(&messages, message_id, &body, author_user_id, author_display_name, now)?; + self.finish_comment_mutation(from_frontier, from_vv, "comment-create")?; + Ok(comment_id) + } + + pub fn reply_to_comment(&mut self, comment_id: u128, body: &str, author_user_id: u128, author_display_name: &str) -> Result { + let body = validated_comment_body(body)?; + self.register_pending_author_identity(); + let message_id = Uuid::new_v4().as_u128(); + let now = unix_time_secs(); + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let thread = existing_comment_thread(&self.doc, comment_id)?; + let messages = thread.ensure_mergeable_map("messages_by_id")?; + write_comment_message(&messages, message_id, &body, author_user_id, author_display_name, now)?; + thread.insert("updated_at", now)?; + self.finish_comment_mutation(from_frontier, from_vv, "comment-reply")?; + Ok(message_id) + } + + pub fn set_comment_resolved(&mut self, comment_id: u128, resolved: bool) -> Result<()> { + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let thread = existing_comment_thread(&self.doc, comment_id)?; + thread.insert("resolved", resolved)?; + thread.insert("updated_at", unix_time_secs())?; + self.finish_comment_mutation(from_frontier, from_vv, if resolved { "comment-resolve" } else { "comment-reopen" }) + } + + pub fn edit_comment_message(&mut self, comment_id: u128, message_id: u128, body: &str, actor_user_id: u128) -> Result<()> { + let body = validated_comment_body(body)?; + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let thread = existing_comment_thread(&self.doc, comment_id)?; + let messages = existing_child_map(&thread, "messages_by_id")?; + let message = existing_child_map(&messages, &message_id.to_string())?; + anyhow::ensure!( + map_string_opt(&message, "author_user_id").and_then(|id| id.parse().ok()) == Some(actor_user_id), + "Only the message author can edit it" + ); + message.insert("body", body.as_str())?; + message.insert("updated_at", unix_time_secs())?; + thread.insert("updated_at", unix_time_secs())?; + self.finish_comment_mutation(from_frontier, from_vv, "comment-edit") + } + + pub fn delete_comment(&mut self, comment_id: u128, actor_user_id: u128) -> Result<()> { + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let root = flowstate_document::loro_schema::root_map(&self.doc); + let comments = existing_child_map(&root, flowstate_document::COMMENTS_BY_ID)?; + let thread = existing_child_map(&comments, &comment_id.to_string())?; + // Threads written before the author field existed fall back to the + // earliest message's author. + let thread_author = map_string_opt(&thread, "author_user_id") + .and_then(|id| id.parse::().ok()) + .or_else(|| { + let messages = existing_child_map(&thread, "messages_by_id").ok()?; + messages + .values() + .filter_map(|value| match value { + ValueOrContainer::Container(Container::Map(message)) => Some(message), + _ => None, + }) + .min_by_key(|message| map_i64_opt(message, "created_at").unwrap_or_default()) + .and_then(|message| map_string_opt(&message, "author_user_id")) + .and_then(|id| id.parse::().ok()) + }); + anyhow::ensure!(thread_author == Some(actor_user_id), "Only the thread author can delete it"); + comments.delete(&comment_id.to_string())?; + self.finish_comment_mutation(from_frontier, from_vv, "comment-delete") + } + + fn finish_comment_mutation(&mut self, from_frontier: Frontiers, from_vv: VersionVector, message: &str) -> Result<()> { + flowstate_document::touch_document_metadata(&self.doc)?; + self.doc.set_next_commit_origin("comment"); + self.doc.set_next_commit_message(message); + self.doc.commit(); + let frontier_before = self.projection.frontier.clone(); + let mut invalidation = ProjectionInvalidation::default(); + self.merge_subscription_invalidation(&mut invalidation); + let events = self.events_after_local_change(from_frontier, from_vv, invalidation, false)?; + self.queue_publish(events); + crate::local_write::recorded_inverse::rearm_recorded_inverse_frontier(self); + self.projection.frontier = self.doc.state_frontiers().encode(); + let empty: Arc<[ProjectionPatch]> = Arc::from(Vec::new()); + let _ = self.projection_patched_event( + empty, + ProjectionInvalidation { + frontier_before, + frontier_after: self.projection.frontier.clone(), + ..Default::default() + }, + ); + Ok(()) + } + + /// §caret-anchor: re-anchor an editor selection across a remote import using + /// Loro cursors, so a concurrent insert/delete BEFORE the caret repositions it + /// instead of stranding it at a stale projection offset (the concurrent-typing + /// interleave bug). The caret is projection-space for rendering/navigation, but + /// its STABILITY across remote edits comes from the CRDT — exactly as paragraph + /// boundaries and object anchors already do. + /// + /// `before` is the editor's PRE-patch projection and `base_frontier` its + /// pre-patch frontier. We reconstruct the pre-patch body (`fork_at`), anchor a + /// cursor at each endpoint there, then resolve it in the CURRENT canonical doc. + /// Returns `None` (empty/edge body, unresolvable cursor, structural mismatch) so + /// the caller can fall back to clamping — never a silent wrong caret. + pub fn rebase_selection_across_import( + &self, + selection: &EditorSelection, + before: &DocumentProjection, + base_frontier: &[u8], + ) -> Option { + // Instrument the O(doc) fork so a test can assert the fast path (stored + // cursors) is taken in the common case rather than silently regressing to a + // fork on every remote batch. + CARET_REBASE_FORKS.with(|count| count.set(count.get().wrapping_add(1))); + let base = Frontiers::decode(base_frontier).ok()?; + let old_doc = self.doc.fork_at(&base).ok()?; + let old_text = body_text(&old_doc); + let cur_text = body_text(&self.doc); + let before_index = ProjectionRuntimeIndex::from_projection(before); + let rebase = |offset: DocumentOffset, affinity: SelectionAffinity| -> Option { + let old_unicode = before_index.body_unicode_for_offset(before, offset)?; + let cursor = cursor_for_boundary(&old_text, old_unicode, affinity)?; + let resolved = self.doc.get_cursor_pos(&cursor).ok()?; + let new_unicode = resolved_cursor_boundary_unicode(&cur_text, &resolved)?; + self + .projection_index + .offset_for_body_unicode(&self.projection, new_unicode) + }; + let head = rebase(selection.head, SelectionAffinity::from(selection.head_affinity))?; + let anchor = rebase(selection.anchor, SelectionAffinity::from(selection.anchor_affinity))?; + Some(EditorSelection { + anchor, + head, + ..selection.clone() + }) + } + + /// §caret-anchor FAST path: encode a selection's endpoints as Loro cursors + /// against the CURRENT (synced) body. The editor stores these at synced moments + /// (after a sync/local write) and, when a remote import arrives while the caret + /// has NOT moved, resolves them with [`Self::resolve_selection_anchor`] to + /// reposition the caret in O(log n) — avoiding the O(doc) `fork_at` of the + /// general [`Self::rebase_selection_across_import`] fallback. Reuses the presence + /// cursor machinery, so both stay bit-consistent. + pub fn encode_selection_anchor(&self, selection: &EditorSelection) -> Option<(Vec, Vec)> { + let head = self.presence_endpoint( + selection.head, + SelectionAffinity::from(selection.head_affinity), + VisualGravity::from(selection.head_gravity), + )?; + let anchor = self.presence_endpoint( + selection.anchor, + SelectionAffinity::from(selection.anchor_affinity), + VisualGravity::from(selection.anchor_gravity), + )?; + Some((head.cursor, anchor.cursor)) + } + + /// Resolve encoded caret cursors (from [`Self::encode_selection_anchor`]) to + /// offsets in the CURRENT canonical state. `None` if either endpoint no longer + /// resolves (e.g. its container was removed) so the caller can fall back. + pub fn resolve_selection_anchor(&self, head_cursor: &[u8], anchor_cursor: &[u8]) -> Option<(DocumentOffset, DocumentOffset)> { + Some(( + self.resolve_body_cursor_offset(head_cursor)?, + self.resolve_body_cursor_offset(anchor_cursor)?, + )) + } + + fn resolve_body_cursor_offset(&self, encoded: &[u8]) -> Option { + let text = body_text(&self.doc); + let cursor = Cursor::decode(encoded).ok()?; + if cursor.container != text.id() { + return None; + } + let resolved = self.doc.get_cursor_pos(&cursor).ok()?; + let unicode = resolved_cursor_boundary_unicode(&text, &resolved)?; + self + .projection_index + .offset_for_body_unicode(&self.projection, unicode) + } + + pub fn presence_selection(&self, selection: &EditorSelection) -> Option { + let direction = selection_direction(selection.anchor, selection.head); + // §16: read the genuine, stored affinity/gravity off each endpoint instead + // of guessing a side from the selection's direction. + let anchor_affinity = SelectionAffinity::from(selection.anchor_affinity); + let head_affinity = SelectionAffinity::from(selection.head_affinity); + let anchor_gravity = VisualGravity::from(selection.anchor_gravity); + let head_gravity = VisualGravity::from(selection.head_gravity); + Some(PresenceSelection { + anchor: self.presence_endpoint(selection.anchor, anchor_affinity, anchor_gravity)?, + head: self.presence_endpoint(selection.head, head_affinity, head_gravity)?, + direction, + }) + } + + pub fn resolve_presence_carets(&self, requests: Vec) -> RuntimePresenceCarets { + let text = body_text(&self.doc); + // Resolve one presence endpoint's encoded cursor to a projection-space + // DocumentOffset in the CURRENT canonical state (`None` if the cursor's + // container is gone or no longer resolves). + let resolve_endpoint = |cursor_bytes: &[u8]| -> Option { + let cursor = Cursor::decode(cursor_bytes).ok()?; + if cursor.container != text.id() { + return None; + } + let resolved = self.doc.get_cursor_pos(&cursor).ok()?; + let unicode = resolved_cursor_boundary_unicode(&text, &resolved)?; + self + .projection_index + .offset_for_body_unicode(&self.projection, unicode) + }; + let mut carets = Vec::with_capacity(requests.len()); + let mut selections = Vec::new(); + for request in requests { + let Some(head_offset) = resolve_endpoint(&request.selection.head.cursor) else { + continue; + }; + carets.push(ExternalCaret { + offset: head_offset, + visual_gravity: gpui_gravity_from_presence(request.selection.head.visual_gravity), + color_rgb: request.color_rgb, + }); + // A non-collapsed selection also paints a highlighted span behind the + // text. Skip it when the anchor no longer resolves or coincides with the + // head (a bare caret) — the caret above already covers that case. + if let Some(anchor_offset) = resolve_endpoint(&request.selection.anchor.cursor) + && anchor_offset != head_offset + { + selections.push(ExternalSelection { + selection: EditorSelection { + anchor: anchor_offset, + head: head_offset, + ..EditorSelection::default() + }, + color_rgb: request.color_rgb, + }); + } + } + RuntimePresenceCarets { carets, selections } + } + + fn presence_endpoint(&self, offset: DocumentOffset, affinity: SelectionAffinity, visual_gravity: VisualGravity) -> Option { + let text = body_text(&self.doc); + let offset = clamp_projection_offset(&self.projection, offset); + // Intentionally projection-space: the endpoint encodes to a Loro cursor and is + // decoded back via projection-space `offset_for_body_unicode`; both sides must + // share a space or the presence caret drifts on object-adjacent-empty docs. Only + // direct (non-round-tripping) body mutations use the Loro-space resolver (5d50099). + let pos = self + .projection_index + .body_unicode_for_offset(&self.projection, offset)?; + cursor_for_boundary(&text, pos, affinity).map(|cursor| SelectionEndpoint { + cursor: cursor.encode(), + affinity, + visual_gravity, + }) + } + + /// Record asset bytes/metadata into canonical state (Loro-first spec §6): + /// additive `assets_by_id` metadata + the maintained projection's asset + /// store, published like any local change and patched to editors as + /// `AssetArrived`. Asset records are content-addressed sideband — never + /// document text authority. + pub fn merge_asset_records(&mut self, records: Vec) -> Result> { + if records.is_empty() { + return Ok(Vec::new()); + } + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let root = self.doc.get_map(ROOT); + let assets = root + .ensure_mergeable_map(flowstate_document::loro_schema::ASSETS_BY_ID) + .context("opening canonical assets map")?; + for record in &records { + let asset_id = record.id.0.to_string(); + let asset_map = assets + .ensure_mergeable_map(&asset_id) + .context("opening canonical asset record")?; + let hash = blake3::hash(&record.bytes); + asset_map + .insert("asset_id", asset_id.as_str()) + .context("writing asset id")?; + asset_map + .insert("content_hash", hash.to_hex().as_str()) + .context("writing asset content hash")?; + asset_map + .insert("mime_type", record.mime_type.as_ref()) + .context("writing asset mime type")?; + asset_map + .insert("byte_length", i64::try_from(record.bytes.len()).unwrap_or(i64::MAX)) + .context("writing asset byte length")?; + if let Some(original_name) = &record.original_name { + asset_map + .insert("original_name", original_name.as_ref()) + .context("writing asset original name")?; + } + } + refresh_image_asset_metadata(&self.doc).context("refreshing image asset metadata")?; + // Asset records ride the repair origin: convergent housekeeping, never an + // undo unit. + self.doc.set_next_commit_origin(REPAIR_ORIGIN); + self.doc.set_next_commit_message("asset-records"); + self.doc.commit(); + + merge_asset_records_into_projection(&mut self.projection, &records); + self.projection.frontier = self.doc.state_frontiers().encode(); + + let mut invalidation = ProjectionInvalidation { + frontier_before: from_frontier.encode(), + frontier_after: self.doc.state_frontiers().encode(), + changed_assets: records + .iter() + .map(|record| record.id.0.to_string()) + .collect(), + ..ProjectionInvalidation::default() + }; + self.merge_subscription_invalidation(&mut invalidation); + let mut events = self.events_after_local_change(from_frontier, from_vv, invalidation.clone(), false)?; + let patches: Vec<_> = records + .into_iter() + .map(|record| flowstate_document::ProjectionPatch::AssetArrived { id: record.id, record }) + .collect(); + events.push(self.projection_patched_event(patches.into(), invalidation)); + Ok(events) + } + + pub fn command(&mut self, command: SemanticCommand) -> Result> { + let restore_undo_selection = matches!(&command, SemanticCommand::Undo | SemanticCommand::Redo); + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let mutates_document = match &command { + SemanticCommand::InsertText { text, .. } => !text.is_empty(), + SemanticCommand::DeleteRange { unicode_len, .. } => *unicode_len > 0, + SemanticCommand::OpenRevision { .. } | SemanticCommand::ForkRevision { .. } | SemanticCommand::Undo | SemanticCommand::Redo => false, + _ => true, + }; + if mutates_document { + flowstate_document::touch_document_metadata(&self.doc).context("updating canonical document metadata for semantic command")?; + // §act-three B.1: any other mutation invalidates the recorded inverse + // (the frontier check would catch it; this frees the capture eagerly). + self.clear_recorded_inverse(); + } + #[allow( + clippy::needless_late_init, + reason = "assigned across match arms that interleave with diverging early-return arms" + )] + let projection_invalidation; + match command { + SemanticCommand::InsertText { unicode_index, text, styles } => { + if text.is_empty() { + return Ok(Vec::new()); + } + let body = body_text(&self.doc); + let newline_boundaries = inserted_newline_boundaries(unicode_index, &text); + body + .insert(unicode_index, &text) + .context("inserting text into Loro body flow")?; + let inserted_len = text.chars().count(); + if inserted_len > 0 { + mark_run_styles(&body, unicode_index..unicode_index + inserted_len, styles).context("marking inserted run styles")?; + } + repair_paragraph_metadata_after_text_flow_edit(&self.doc, &body, &newline_boundaries, "semantic_insert_text")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_text(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_index, inserted_len); + }, + SemanticCommand::DeleteRange { unicode_index, unicode_len } => { + // §5 sentinel protection (preflight): clamp/reject any range that would + // delete the boundary-0 sentinel newline before mutating the body. + let Some((unicode_index, unicode_len)) = sentinel_protected_delete_range(unicode_index, unicode_len) else { + return Ok(Vec::new()); + }; + let body = body_text(&self.doc); + body + .delete(unicode_index, unicode_len) + .context("deleting text from Loro body flow")?; + // §5: drop object blocks whose U+FFFC placeholder this delete removed, in + // the same transaction, so they never linger as unresolved-anchor records. + prune_orphaned_body_object_blocks(&self.doc, &body)?; + repair_paragraph_metadata_after_text_flow_edit(&self.doc, &body, &[], "semantic_delete_range")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_text(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_index, unicode_len); + }, + SemanticCommand::SplitParagraph { + unicode_index, + inherited_style, + } => { + let body = body_text(&self.doc); + body + .insert(unicode_index, "\n") + .context("splitting Loro body paragraph")?; + body + .mark( + unicode_index..unicode_index + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(inherited_style), + ) + .context("marking split paragraph boundary")?; + repair_paragraph_metadata_after_text_flow_edit(&self.doc, &body, &[unicode_index], "semantic_split_paragraph")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_text(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_index, 1); + }, + SemanticCommand::SetParagraphStyle { + boundary_unicode_index, + style, + } => { + let body = body_text(&self.doc); + body + .mark( + boundary_unicode_index..boundary_unicode_index + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(style), + ) + .context("marking paragraph style in Loro body flow")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_style(from_frontier.encode(), self.doc.state_frontiers().encode(), boundary_unicode_index, 1); + }, + SemanticCommand::SetRunStyles { unicode_range, styles } => { + if unicode_range.is_empty() { + return Ok(Vec::new()); + } + let unicode_start = unicode_range.start; + let unicode_len = unicode_range.end.saturating_sub(unicode_range.start); + mark_run_styles(&body_text(&self.doc), unicode_range, styles).context("marking run styles in Loro body flow")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_style(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_start, unicode_len); + }, + SemanticCommand::InsertImage { + unicode_index, + asset_id, + alt_text, + caption, + sizing, + alignment, + } => { + insert_image_block(&self.doc, unicode_index, asset_id, &alt_text, caption.as_deref(), sizing, alignment) + .context("inserting image block into Loro document")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_object(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_index, "image"); + }, + SemanticCommand::InsertEquation { + unicode_index, + source, + display, + } => { + insert_equation_block(&self.doc, unicode_index, &source, display).context("inserting equation block into Loro document")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_object(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_index, "equation"); + }, + SemanticCommand::InsertTable { + unicode_index, + rows, + columns, + column_widths, + header_row, + } => { + insert_table_block(&self.doc, unicode_index, rows, columns, &column_widths, header_row) + .context("inserting table block into Loro document")?; + self.doc.commit(); + self.record_undo_checkpoint()?; + projection_invalidation = + ProjectionInvalidation::body_object(from_frontier.encode(), self.doc.state_frontiers().encode(), unicode_index, "table"); + }, + SemanticCommand::OpenRevision { revision_id } => { + let document = self.revision_projection(revision_id)?; + return Ok(vec![RuntimeEvent::RevisionOpened { + revision_id, + document: Box::new(document), + }]); + }, + SemanticCommand::ForkRevision { revision_id } => { + let (document, package) = self.fork_revision(revision_id)?; + return Ok(vec![RuntimeEvent::RevisionForked { + revision_id, + document: Box::new(document), + package: Box::new(package), + }]); + }, + SemanticCommand::Undo => { + // §act-three B.1: replay the recorded inverse when the lineage checks + // hold — no checkout, no diff calc, no rebuild. Falls through to the + // UndoManager slow path on any mismatch. + if let Some(events) = hotpath::measure_block!( + "recorded_inverse_undo", + crate::local_write::recorded_inverse::try_fast_step(self, loro::UndoOrRedo::Undo)? + ) { + return Ok(events); + } + let applied = hotpath::measure_block!("loro_undo_manager_undo", self.undo.undo().context("applying Loro undo")?); + // §fidelity: record the undo's frontier transition and assert it only + // introduced local-peer ops (remote-origin ops are excluded from undo). + if fidelity::enabled() { + fidelity::event(FidelityClass::Undo, "undo", || { + format!( + "applied={applied} frontier {:?} -> {:?}", + from_frontier.encode(), + self.doc.state_frontiers().encode() + ) + }); + if applied { + self.fidelity_check_undo_local_only("undo", &from_vv); + } + } + if !applied { + return Ok(Vec::new()); + } + projection_invalidation = ProjectionInvalidation { + frontier_before: from_frontier.encode(), + frontier_after: self.doc.state_frontiers().encode(), + changed_flows: vec![ROOT_BODY_FLOW_ID.to_string()], + ..ProjectionInvalidation::default() + }; + }, + SemanticCommand::Redo => { + // §act-three B.1: symmetric fast path — replays the recorded delete. + if let Some(events) = hotpath::measure_block!( + "recorded_inverse_redo", + crate::local_write::recorded_inverse::try_fast_step(self, loro::UndoOrRedo::Redo)? + ) { + return Ok(events); + } + let applied = hotpath::measure_block!("loro_undo_manager_redo", self.undo.redo().context("applying Loro redo")?); + // §fidelity: record the redo's frontier transition and assert it only + // introduced local-peer ops (remote-origin ops are excluded from redo). + if fidelity::enabled() { + fidelity::event(FidelityClass::Undo, "redo", || { + format!( + "applied={applied} frontier {:?} -> {:?}", + from_frontier.encode(), + self.doc.state_frontiers().encode() + ) + }); + if applied { + self.fidelity_check_undo_local_only("redo", &from_vv); + } + } + if !applied { + return Ok(Vec::new()); + } + projection_invalidation = ProjectionInvalidation { + frontier_before: from_frontier.encode(), + frontier_after: self.doc.state_frontiers().encode(), + changed_flows: vec![ROOT_BODY_FLOW_ID.to_string()], + ..ProjectionInvalidation::default() + }; + }, + } + let mut projection_invalidation = projection_invalidation; + let drained = self.merge_subscription_invalidation(&mut projection_invalidation); + let mut events = self.events_after_local_change(from_frontier, from_vv, projection_invalidation.clone(), false)?; + // §6-R: semantic commands (undo/redo, object inserts) derive their + // projection through THE ladder — the same one remote imports use. No + // whole-body string diff, no O(doc) before/after captures. + let context = if restore_undo_selection { "undo_redo" } else { "semantic_command" }; + self.derive_body_projection_events(projection_invalidation, &drained, context, &mut events)?; + // §act-four M5: the semantic-command path (undo/redo, object inserts, direct + // text edits) commits and derives here — append its root to the version log, + // same as the intent path's `apply_local_intent` hooks. + if restore_undo_selection && let Some(snapshot) = self.take_restored_undo_selection() { + if let Some(selection) = self.resolve_undo_selection(&snapshot) { + events.push(RuntimeEvent::SelectionRestored { selection }); + } else if let Ok(mut state) = self.undo_selection.lock() { + state.restored_selection = Some(snapshot); + } + } + Ok(events) + } + + pub(crate) fn resolve_undo_selection(&mut self, snapshot: &UndoSelectionSnapshot) -> Option { + // §16: restore the stored affinity onto the rebuilt selection. Gravity is + // not persisted in the undo snapshot, so it resolves to neutral. + // §24: `&mut self` so the cursor resolutions can memoize through the index's + // cursor cache without interior mutability (this stays on the actor's + // single-threaded `&mut self` command flow). + Some(EditorSelection { + anchor: self.resolve_undo_cursor(&snapshot.anchor_cursor)?, + head: self.resolve_undo_cursor(&snapshot.head_cursor)?, + anchor_affinity: gpui_affinity_from_undo(snapshot.anchor_affinity), + head_affinity: gpui_affinity_from_undo(snapshot.head_affinity), + anchor_gravity: gpui_flowtext::VisualGravity::Neutral, + head_gravity: gpui_flowtext::VisualGravity::Neutral, + }) + } + + fn resolve_undo_cursor(&mut self, encoded: &[u8]) -> Option { + // §24 cursor resolution cache: memoize the (expensive) `get_cursor_pos` + // resolution keyed by the encoded cursor bytes. The cache is cleared on every + // projection rebuild/incremental update, so a hit always reflects the current + // projection. + if let Some(offset) = self.projection_index.cursor_resolution_cache.get(encoded) { + return Some(*offset); + } + let cursor = Cursor::decode(encoded).ok()?; + let body = body_text(&self.doc); + if cursor.container != body.id() { + return None; + } + let resolved = self.doc.get_cursor_pos(&cursor).ok()?; + let unicode = resolved_cursor_boundary_unicode(&body, &resolved)?; + let offset = self + .projection_index + .offset_for_body_unicode(&self.projection, unicode)?; + self + .projection_index + .cursor_resolution_cache + .insert(encoded.to_vec(), offset); + Some(offset) + } + + pub fn revision_projection(&self, revision_id: u128) -> Result { + let revision_doc = self + .package + .as_ref() + .context("cannot open revision without a package-backed runtime")? + .load_revision_loro_doc(revision_id) + .context("loading revision Loro snapshot")?; + let mut document = document_from_loro(&revision_doc).context("projecting revision document")?; + if let Some(package) = &self.package { + attach_package_assets(&mut document, package); + } + Ok(document) + } + + pub fn fork_revision(&self, revision_id: u128) -> Result<(DocumentProjection, DocumentPackage)> { + let package = self + .package + .as_ref() + .context("cannot fork revision without a package-backed runtime")?; + let revision_doc = package + .load_revision_loro_doc(revision_id) + .context("loading revision Loro snapshot for fork")?; + let forked_doc = revision_doc.fork(); + flowstate_document::fork_document_lineage(&forked_doc).context("assigning forked document lineage")?; + let forked_package = DocumentPackage::from_loro_snapshot_with_assets(&forked_doc, "Forked revision", package.assets.clone()) + .context("creating forked revision package")?; + let mut document = document_from_loro(&forked_doc).context("projecting forked revision")?; + attach_package_assets(&mut document, &forked_package); + Ok((document, forked_package)) + } + + /// Spec §6-R: derive exact structural patches for a remote import by + /// rematerializing ONLY the changed region under the same materializer law + /// as the full rebuild. `None` = take the full-rebuild fallback (loud, + /// counted by the caller). Returns the patches plus the region's projection + /// defects for canonical repair scheduling. + #[hotpath::measure] + /// Loro-first spec §6.3 + §6-R: THE post-commit projection derivation + /// ladder, shared by remote imports and local semantic commands (undo/redo, + /// object inserts). Structural analysis is EXACT, from the composed net + /// delta (pre-change → post-change): structure changed iff (a) an inserted + /// run carries \n/U+FFFC, (b) a deleted pre-change range covers a paragraph + /// boundary or object placeholder, or (c) a structural insert was churned + /// away within the batch (per-op flags can't prove neutrality). Checkout + /// events stay conservative. The ladder: projection-neutral → nothing to + /// derive (the 19:28 churn class ends as a no-op, spec §13.13); + /// non-structural → ranged paragraph patches; structural → regional + /// rematerialization; any bail → full rebuild, loudly, with its cause. + pub(crate) fn derive_body_projection_events( + &mut self, + mut invalidation: ProjectionInvalidation, + drained: &DrainedBodyDelta, + context: &'static str, + events: &mut Vec, + ) -> Result<()> { + invalidation.has_remote_origin |= drained.has_remote_origin; + debug_assert!( + !drained.checkout_seen || invalidation.rebuild_required, + "checkout drains must force a rebuild" + ); + let net = &drained.net; + let body_neutral = net.is_empty(); + let structural = !body_neutral + && (net.structural_churn + || net.inserts_structure() + || net.deletes_any_position(self.projection_index.boundary_positions()) + || net.deletes_any_position(self.projection_index.object_positions())); + // Diagnostic affordance (FLOWSTATE_DERIVE_DEBUG=1): one line per ladder + // decision — which branch a change batch takes and why. Costs one cached + // atomic load when off. + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + "derive[{context}]: neutral={body_neutral} structural={structural} rebuild={} fallback={:?} ranges={:?} net={:?}", + invalidation.rebuild_required, invalidation.fallback_reason, invalidation.changed_text_ranges, net + ); + } + if structural { + invalidation.rebuild_required = true; + if invalidation.fallback_reason.is_none() { + invalidation.fallback_reason = Some("structural_body_text_change"); + } + } else if invalidation.fallback_reason == Some("structural_body_text_change") { + // The union heuristics inside the drain were conservative; the exact + // net-delta analysis supersedes them. + invalidation.rebuild_required = false; + invalidation.fallback_reason = None; + } + // §6-R: structural changes take the REGIONAL rematerializer — the same + // materializer law as the full rebuild, applied to the changed region + // only. Any bail falls through to the rebuild, loudly, with its cause. + let regional = if structural { + let result = self.remote_structural_projection_patches(&invalidation, drained); + if let Err(bail) = &result { + // Mirror of `full-rebuild-after-local-write`: regional bails must be + // visible, not ambient. + tracing::warn!(bail, context, "full-rebuild-after-structural-change"); + } + result.ok() + } else { + None + }; + // §6-R composition: the regional walk covers the structural neighborhood; + // a mixed batch can ALSO carry changes outside it (e.g. an undo restoring + // a split while a distant paragraph is restyled in the same exchange). + // Out-of-region touched paragraphs take the same ranged readback the + // non-structural path uses — same law, region scale + paragraph scale. A + // readback bail falls to the rebuild exactly like a regional bail. + let regional = regional.and_then(|regional| { + let RegionalPatches { + mut patches, + defects, + region_paragraphs, + } = regional; + let live_starts = net.shift_positions(self.projection_index.paragraph_starts()); + let touched = + self + .projection_index + .paragraphs_for_changed_ranges(&invalidation.changed_text_ranges, self.projection.paragraphs.len(), &live_starts); + let outside: Vec = touched + .into_iter() + .filter(|ix| !region_paragraphs.contains(ix)) + .collect(); + if !outside.is_empty() { + match paragraph_projection_patches_ranged(&self.projection, &self.doc, &live_starts, outside) { + Some(extra) => patches.extend(extra), + None => { + tracing::warn!(bail = "outside-region-readback", context, "full-rebuild-after-structural-change"); + return None; + }, + } + } + // Same composition for OBJECT state: a batch can pair a structural body + // change with e.g. a table column reorder on a table outside the region + // (the table fuzz caught the swallowed reorder). The canonical object + // readback covers all object rows; in-region duplicates are benign + // (same derived values). + if !invalidation.changed_blocks.is_empty() + || !invalidation.changed_tables.is_empty() + || invalidation + .changed_flows + .iter() + .any(|flow| flow != ROOT_BODY_FLOW_ID) + { + match projection_patch::remote_object_projection_patches_scoped_or_full(&self.projection, &self.doc, &invalidation) { + Some(extra) => patches.extend(extra), + None => { + tracing::warn!(bail = "outside-region-object-readback", context, "full-rebuild-after-structural-change"); + return None; + }, + } + } + Some((patches, defects)) + }); + if let Some((patches, defects)) = regional { + self.apply_projection_patch_set(&patches); + self.projection.frontier = self.doc.state_frontiers().encode(); + let mut patched_invalidation = invalidation; + patched_invalidation.rebuild_required = false; + patched_invalidation.fallback_reason = None; + events.push(self.projection_patched_event(patches.into(), patched_invalidation)); + match self.schedule_projection_repairs(defects, RepairEmission::Streamed) { + Ok(repair_events) => events.extend(repair_events), + Err(error) => tracing::error!(%error, context, "scheduling regional projection repairs after structural change failed"), + } + } else if invalidation.rebuild_required { + let before_projection = self.projection.clone(); + self.refresh_projection()?; + events.push(self.projection_change_event(&before_projection, invalidation)?); + } else { + // Non-structural: post-change paragraph starts are the pre-change + // starts shifted through the net delta — O(P + ops) arithmetic, no + // body scan. + let live_starts = net.shift_positions(self.projection_index.paragraph_starts()); + let touched_paragraphs = + self + .projection_index + .paragraphs_for_changed_ranges(&invalidation.changed_text_ranges, self.projection.paragraphs.len(), &live_starts); + let nonstructural = remote_nonstructural_projection_patches(&self.projection, &self.doc, &invalidation, &touched_paragraphs, &live_starts); + static DERIVE_DEBUG2: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG2.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + " nonstructural: touched={touched_paragraphs:?} patches={:?}", + nonstructural.as_ref().map(|patches| patches + .iter() + .map(|p| format!("{p:?}").chars().take(90).collect::()) + .collect::>()) + ); + } + if let Some(patches) = nonstructural { + self.apply_projection_patch_set(&patches); + self.projection.frontier = self.doc.state_frontiers().encode(); + events.push(self.projection_patched_event(patches.into(), invalidation)); + } else { + let before_projection = self.projection.clone(); + self.refresh_projection()?; + events.push(self.projection_change_event(&before_projection, invalidation)?); + } + } + Ok(()) + } + + fn remote_structural_projection_patches( + &self, + invalidation: &ProjectionInvalidation, + drained: &DrainedBodyDelta, + ) -> Result { + // Rebuild-only classes: time travel, unclassifiable diffs, section-map + // changes, and non-body flow content (table cells) mixed into the chunk. + // `metadata_record_id_change` is EXPECTED here (splits write records) and + // is exactly what the regional re-derivation handles. + if drained.checkout_seen + || !invalidation.changed_sections.is_empty() + || !invalidation.changed_tables.is_empty() + || invalidation + .changed_flows + .iter() + .any(|flow| flow != ROOT_BODY_FLOW_ID) + || matches!( + invalidation.fallback_reason, + Some("checkout_trigger_projection_rebuild" | "unknown_loro_subscription_diff") + ) + { + return Err("rebuild-only-invalidation"); + } + let (pre_lo, pre_hi) = drained.net.pre_change_span().ok_or("empty-net-span")?; + + // ---- Old affected rows (search spans are block-aligned, PRE space). ---- + let spans = &self.projection_index.search_unit_spans; + if spans.len() != self.projection.blocks.len() { + return Err("search-index-misaligned"); + } + let mut blk_lo = None; + let mut blk_hi = None; + for (ix, span) in spans.iter().enumerate() { + // A row's territory includes its leading boundary sentinel. + let lo = span.unicode_start.saturating_sub(1); + let hi = span.unicode_start + span.unicode_len; + if hi >= pre_lo && lo <= pre_hi { + blk_lo.get_or_insert(ix); + blk_hi = Some(ix); + } + } + let (blk_lo, blk_hi) = (blk_lo.ok_or("no-affected-rows")?, blk_hi.ok_or("no-affected-rows")?); + + // ---- Expand to whole rows + resolve POST-space region bounds. ---------- + // The predecessor paragraph provides the walker's object-coalescing + // context; the successor paragraph's leading sentinel is the exclusive + // region end. Both are unaffected rows, so their durable records resolve + // on the live fast path. + let is_paragraph_row = |ix: usize| matches!(self.projection.blocks.get(ix), Some(flowstate_document::Block::Paragraph(_))); + // §act-ten A10.8: O(log N) tree rank per lookup (was an O(row) block walk, + // called ~4x per regional derivation). + let paragraph_ix_of_row = |row: usize| self.projection.blocks.paragraphs_before_row(row); + let body = body_text(&self.doc); + static REGION_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + let region_debug = *REGION_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()); + let (region_lo, region_sentinel) = match (0..blk_lo).rev().find(|ix| is_paragraph_row(*ix)) { + Some(row) => { + let id = self + .projection + .ids + .paragraph_ids + .get(paragraph_ix_of_row(row)) + .copied() + .ok_or("edge-paragraph-id-missing")?; + let start = paragraph_body_start_in_loro(&self.doc, id); + if start.is_none() && region_debug { + eprintln!("derive[region]: start unresolved — predecessor row {row} paragraph id {id:?} has no resolvable record"); + } + let start = start.ok_or("region-start-unresolved")?; + (row, start.checked_sub(1).ok_or("region-start-underflow")?) + }, + None => (0, 0), + }; + let (region_hi, region_end) = match ((blk_hi + 1)..self.projection.blocks.len()).find(|ix| is_paragraph_row(*ix)) { + Some(row) => { + let id = self + .projection + .ids + .paragraph_ids + .get(paragraph_ix_of_row(row)) + .copied() + .ok_or("edge-paragraph-id-missing")?; + let start = paragraph_body_start_in_loro(&self.doc, id).ok_or("region-end-unresolved")?; + (row, start.checked_sub(1).ok_or("region-end-underflow")?) + }, + None => (self.projection.blocks.len(), body.len_unicode()), + }; + + // ---- Identity candidates (§6-R step 2): the old projection's region rows + // plus every registry record this import touched. Canonical keys make each + // probe O(1); a candidate that no longer resolves is simply dead. --------- + let root = self.doc.get_map(ROOT); + let paragraphs_registry = child_map(&root, PARAGRAPHS_BY_ID).ok_or("paragraph-registry-missing")?; + let blocks_registry = child_map(&root, BLOCKS_BY_ID).ok_or("block-registry-missing")?; + let probe = |registry: &LoroMap, canonical: String, special: &str, id: u128| -> Option<(String, LoroMap)> { + if let Some(record) = child_map(registry, &canonical) { + return Some((canonical, record)); + } + if loro_id_u128(special) == id + && let Some(record) = child_map(registry, special) + { + return Some((special.to_string(), record)); + } + None + }; + + let mut paragraph_candidates: Vec<(String, LoroMap)> = Vec::new(); + let mut pblock_candidates: Vec<(String, LoroMap)> = Vec::new(); + let mut object_candidates: Vec = Vec::new(); + let mut paragraph_pointer = paragraph_ix_of_row(region_lo); + for row in region_lo..region_hi { + let block_id = self + .projection + .ids + .block_ids + .get(row) + .copied() + .ok_or("row-block-id-missing")?; + match self.projection.blocks.get(row).ok_or("row-missing")? { + flowstate_document::Block::Paragraph(_) => { + let id = self + .projection + .ids + .paragraph_ids + .get(paragraph_pointer) + .copied() + .ok_or("row-paragraph-id-missing")?; + paragraph_pointer += 1; + // A missing record is not a bail: the boundary fabricates the same + // stable id the repair writer mints, and defect repair converges it + // (accepted-risk class, spec 6-R). + if let Some(candidate) = probe(¶graphs_registry, format!("paragraph.{}", id.0), ROOT_FIRST_PARAGRAPH_ID, id.0) { + paragraph_candidates.push(candidate); + } + if let Some(candidate) = probe( + &blocks_registry, + format!("paragraph_block.{}", block_id.0), + MAIN_BODY_BLOCK_ID, + block_id.0, + ) { + pblock_candidates.push(candidate); + } + }, + block => { + let kind = match block { + flowstate_document::Block::Image(_) => "image", + flowstate_document::Block::Equation(_) => "equation", + flowstate_document::Block::Table(_) => "table", + flowstate_document::Block::Paragraph(_) => unreachable!(), + }; + object_candidates.push(child_map(&blocks_registry, &format!("{kind}.{}", block_id.0)).ok_or("object-record-missing")?); + }, + } + } + for key in &drained.imported_paragraph_record_keys { + if let Some(record) = child_map(¶graphs_registry, key) { + paragraph_candidates.push((key.clone(), record)); + } + } + for key in &drained.imported_block_record_keys { + let Some(record) = child_map(&blocks_registry, key) else { + continue; + }; + match map_string_opt(&record, "kind").as_deref() { + Some("paragraph") => pblock_candidates.push((key.clone(), record)), + Some(_) => object_candidates.push(record), + None => {}, + } + } + + // ---- ONE batch resolution for every candidate anchor (dead anchors are + // simply absent — never per-id history walks; the ctrl-A lesson). -------- + let container = body.id(); + let decode_cursor_id = |record: &LoroMap, field: &str| -> Option { + let bytes = map_binary_opt(record, field)?; + let cursor = Cursor::decode(&bytes).ok()?; + (cursor.container == container) + .then_some(cursor.id) + .flatten() + }; + let mut query_ids: Vec = Vec::new(); + let paragraph_cursor_ids: Vec<(Option, Option)> = paragraph_candidates + .iter() + .map(|(_, record)| (decode_cursor_id(record, "boundary_cursor"), decode_cursor_id(record, "start_cursor"))) + .collect(); + let pblock_cursor_ids: Vec> = pblock_candidates + .iter() + .map(|(_, record)| decode_cursor_id(record, "anchor_cursor")) + .collect(); + let object_cursor_ids: Vec> = object_candidates + .iter() + .map(|record| decode_cursor_id(record, "anchor_cursor")) + .collect(); + for id in paragraph_cursor_ids + .iter() + .flat_map(|(boundary, start)| [boundary, start]) + .chain(pblock_cursor_ids.iter()) + .chain(object_cursor_ids.iter()) + .flatten() + { + query_ids.push(*id); + } + let mut pos_by_id: FxHashMap = FxHashMap::default(); + if !query_ids.is_empty() { + for (id, pos) in query_ids.iter().copied().zip( + self + .doc + .inner() + .query_text_id_positions(&container, &query_ids), + ) { + if let Some(pos) = pos { + pos_by_id.insert(id, pos); + } + } + } + + // ---- Boundary→key maps, mirroring the full materializer's prefer rule + // (lexicographically smallest key wins; special ids win boundary 0). ----- + let live_newline = |pos: usize| body.char_at(pos) == Ok('\n'); + let mut paragraph_map: FxHashMap = FxHashMap::default(); + let mut paragraph_resolved: Vec<(String, usize)> = paragraph_candidates + .iter() + .zip(¶graph_cursor_ids) + .filter_map(|((key, _), (boundary, start))| { + let pos = boundary + .and_then(|id| pos_by_id.get(&id).copied()) + .or_else(|| start.and_then(|id| pos_by_id.get(&id).copied()))?; + live_newline(pos).then(|| (key.clone(), pos)) + }) + .collect(); + paragraph_resolved.sort(); + for (key, pos) in ¶graph_resolved { + if *pos == 0 && key == ROOT_FIRST_PARAGRAPH_ID { + paragraph_map.insert(0, key.clone()); + } else { + paragraph_map.entry(*pos).or_insert_with(|| key.clone()); + } + } + let mut pblock_map: FxHashMap = FxHashMap::default(); + let mut pblock_resolved: Vec<(String, usize)> = pblock_candidates + .iter() + .zip(&pblock_cursor_ids) + .filter_map(|((key, _), anchor)| { + let pos = anchor.and_then(|id| pos_by_id.get(&id).copied())?; + live_newline(pos).then(|| (key.clone(), pos)) + }) + .collect(); + pblock_resolved.sort(); + for (key, pos) in &pblock_resolved { + if *pos == 0 && key == MAIN_BODY_BLOCK_ID { + pblock_map.insert(0, key.clone()); + } else { + pblock_map.entry(*pos).or_insert_with(|| key.clone()); + } + } + let mut object_map: BTreeMap = BTreeMap::new(); + for (record, anchor) in object_candidates.into_iter().zip(&object_cursor_ids) { + let Some(pos) = anchor.and_then(|id| pos_by_id.get(&id).copied()) else { + continue; + }; + if body.char_at(pos) == Ok(flowstate_document::OBJECT_REPLACEMENT) { + object_map.entry(pos).or_insert(record); + } + } + + // §task #40 (BARE heal): repair-HEALED records are ANCHOR-keyed + // (`paragraph.anchor.op-…`, minted by `stable_boundary_metadata_keys` when + // a record-less boundary is repaired), and the row-id probes above cannot + // reconstruct those keys from the projection's hashed ids (one-way). + // Re-derive each still-unresolved region boundary's anchor keys from the + // live body — the exact derivation the repair writer used — and probe + // them, so healed boundaries resolve regionally instead of re-reporting + // (and re-repairing, and re-projecting) their defects on every later + // structural import touching the region. Agreement with the full law: + // any record the full materializer would prefer over the anchor record + // (cursor-resolved, lexicographically smaller key) already claimed the + // boundary in the maps above, so this only fills genuine holes. + { + let region_text = body + .slice(region_sentinel, region_end) + .map_err(|_| "region-slice-failed")?; + for (offset, ch) in region_text.chars().enumerate() { + if ch != '\n' { + continue; + } + let boundary = region_sentinel + offset; + let paragraph_missing = !paragraph_map.contains_key(&boundary); + let pblock_missing = !pblock_map.contains_key(&boundary); + if !paragraph_missing && !pblock_missing { + continue; + } + let Some((paragraph_key, block_key)) = flowstate_document::loro_projection::stable_boundary_metadata_keys(&body, boundary) else { + continue; + }; + if paragraph_missing && child_map(¶graphs_registry, ¶graph_key).is_some() { + paragraph_map.insert(boundary, paragraph_key); + } + if pblock_missing && child_map(&blocks_registry, &block_key).is_some() { + pblock_map.insert(boundary, block_key); + } + } + } + + // ---- Rematerialize the region under the shared law + splice. ----------- + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + let derive_debug = *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()); + let materialize_start = std::time::Instant::now(); + let region = flowstate_document::materialize_body_region(&self.doc, region_sentinel, region_end, ¶graph_map, &pblock_map, &object_map) + .map_err(|_| "region-materialize-failed")?; + if derive_debug { + eprintln!( + "derive[region]: span={}..{} rows={}..{} objects={} materialize={:?}", + region_sentinel, + region_end, + region_lo, + region_hi, + object_map.len(), + materialize_start.elapsed() + ); + } + let patches = splice_region_patches(&self.projection, region_lo, region_hi, ®ion).ok_or("splice-bailed")?; + // The region's OLD-projection paragraph-index range, so the caller can + // derive out-of-region changes (a mixed batch's distant mark edits) + // through the ranged readback rather than losing them. + let region_paragraphs = paragraph_ix_of_row(region_lo)..paragraph_ix_of_row(region_hi.min(self.projection.blocks.len())); + Ok(RegionalPatches { + patches, + defects: region.defects, + region_paragraphs, + }) + } + + #[hotpath::measure] + pub fn import_remote_update(&mut self, bytes: &[u8]) -> Result> { + let mut replies = self.import_remote_updates(&[bytes])?; + Ok(replies.pop().unwrap_or_default()) + } + + /// Batched sibling of [`Self::import_remote_update`] (§6.4): import EVERY + /// coalesced blob first, then derive projection events ONCE over the + /// composed frontier delta. The former shape ran one full derive (regional + /// walk / readback / editor patch batch) per blob — 41.6% of a receiving + /// peer's runtime in the 2026-07-07 field profile. Returns one event vec + /// per input blob (each carries its own `pending` status for the session's + /// anti-entropy trigger); the composed projection events ride on the LAST + /// blob's vec, preserving commit-order delivery through the ordered stream. + #[hotpath::measure] + pub fn import_remote_updates(&mut self, chunks: &[&[u8]]) -> Result>> { + if chunks.is_empty() { + return Ok(Vec::new()); + } + // §act-eleven C8 observability: batches served (each = one gate hold + one + // derive over N coalesced blobs). The doc_io coalescing bound is asserted + // against this — without it the §6.4 batching could silently degrade to + // one-derive-per-blob (41.6% of a receiving peer's runtime in the field). + self.import_batches_served += 1; + let from_frontier = self.doc.state_frontiers(); + // §fidelity: capture the pre-import version only when tracing so a disabled + // build pays nothing; used to assert the import advanced (never regressed) the + // canonical frontier below. + let fidelity_before_vv = fidelity::enabled().then(|| self.doc.state_vv()); + static IMPORT_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + let import_probe = *IMPORT_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_IMPORT_PROBE").is_some()); + let probe_t = std::time::Instant::now(); + let mut statuses = Vec::with_capacity(chunks.len()); + let mut below_root_chunks: Vec<&[u8]> = Vec::new(); + for bytes in chunks { + let status = hotpath::measure_block!("loro_import_with", { + match self.doc.import_with(bytes, "remote") { + Ok(status) => status, + // §A12.1.3 slice 4: this shallow session cannot hold ops that + // depend on pre-root history (Loro rolled the import back + // cleanly). Absorb them durably into the PACKAGE below instead of + // failing the import. + Err(loro::LoroError::ImportUpdatesThatDependsOnOutdatedVersion) if self.doc.is_shallow() => { + below_root_chunks.push(bytes); + ImportStatus { + success: VersionRange::default(), + pending: None, + } + }, + Err(error) => return Err(error).context("importing remote Loro update"), + } + }); + statuses.push(status); + } + if let Some(before_vv) = &fidelity_before_vv { + self.fidelity_frontier_transition("import", &from_frontier, before_vv); + } + // §6-R: unmarked paragraph boundaries introduced by a merge surface as + // `MissingParagraphStyleMark` defects from whichever projection derivation + // runs below (regional walk or full rebuild) and are repaired canonically + // through `schedule_projection_repairs` — the former pre-import whole-flow + // style scan (O(doc) `to_delta` per structural chunk) is gone. + let frontier_after = self.doc.state_frontiers().encode(); + let version_vector = self.doc.state_vv().encode(); + // §act-ten A10.2: a remote import invalidates the recorded inverse — but + // ONLY if it applied anything (no-op drips leave the frontier — hence the + // doc — untouched). §oom-leads #9 refined this further: an APPLYING import + // no longer clears eagerly either; once the drained net delta is available + // below, `reconcile_recorded_inverse_after_remote` REBASES the stacks + // through hull-disjoint remote changes and only clears when the import + // actually threatens the recorded coordinates. + let probe_loro = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + let import_applied_changes = frontier_after != from_frontier.encode(); + // §22: when an import is missing dependencies, surface the pending version + // range so the UI session can trigger immediate update pull/anti-entropy + // rather than waiting for the periodic digest. The range is both logged here + // and carried on that blob's `RemoteUpdateApplied { pending }` below. + let mut replies: Vec> = statuses + .iter() + .map(|status| { + if let Some(missing) = Self::missing_dependency_request(status) { + tracing::debug!(?missing, "remote Loro import is missing dependencies; requesting anti-entropy pull"); + } + vec![RuntimeEvent::RemoteUpdateApplied { + pending: status.pending.clone(), + frontier: frontier_after.clone(), + version_vector: version_vector.clone(), + }] + }) + .collect(); + // A blob with unresolved dependencies may have been satisfied by a LATER + // blob in the same chunk (Loro buffers and auto-applies), but the recorded + // per-import status cannot see that — treat any pending status as the + // conservative full-rebuild fallback, exactly once for the whole chunk + // (the former shape paid one full rebuild PER pending blob). + let any_pending = statuses.iter().any(|status| status.pending.is_some()); + let frontier_before = from_frontier.encode(); + let last = replies.last_mut().expect("chunks is non-empty"); + if !any_pending { + let mut invalidation = ProjectionInvalidation { + frontier_before, + frontier_after: frontier_after.clone(), + changed_flows: vec![ROOT_BODY_FLOW_ID.to_string()], + ..ProjectionInvalidation::default() + }; + let drained = self.merge_subscription_invalidation(&mut invalidation); + let probe_drain = probe_t.elapsed(); + let probe_t2 = std::time::Instant::now(); + if import_applied_changes { + self.reconcile_recorded_inverse_after_remote(&invalidation, &drained); + } + let probe_reconcile = probe_t2.elapsed(); + let probe_t2 = std::time::Instant::now(); + self.derive_body_projection_events(invalidation, &drained, "remote_import", last)?; + if import_probe { + eprintln!( + "[flowstate-import-probe] loro={probe_loro:?} drain={probe_drain:?} reconcile={probe_reconcile:?} derive={:?}", + probe_t2.elapsed() + ); + } + } else { + if import_applied_changes { + self.clear_recorded_inverse(); + } + let mut invalidation = ProjectionInvalidation::full_rebuild( + frontier_before.clone(), + frontier_after.clone(), + "remote_update_pending_projection_fallback", + ); + // Drain the subscription regardless so no stale events linger. + self.merge_subscription_invalidation(&mut invalidation); + // §act-five P4: a fully-pending import APPLIED NOTHING — Loro buffered the + // out-of-order updates and the canonical frontier is unchanged, so the + // projection is byte-identical (same frontier ⇒ same Loro state). Skip the + // O(doc) full rebuild + event; the pending version range already rode the + // `RemoteUpdateApplied` reply above, which drives the anti-entropy pull. + if frontier_after != frontier_before { + self.refresh_projection()?; + last.push(self.projection_event(invalidation)?); + } + } + if !below_root_chunks.is_empty() { + let event = self.absorb_below_root_import(&below_root_chunks)?; + replies.last_mut().expect("chunks is non-empty").push(event); + } + // §act-four §8 fork-and-share: a remote import advances the canonical + // frontier and forks a new read-model version. Because `blocks`/`paragraphs` + // are persistent trees, the forked root SHARES every subtree the import did + // not touch — the fork costs `O(change)`, not `O(document)`. + if !any_pending { + // The remote updates have already merged into the canonical Loro doc above; + // durability (revision sync + update-segment persistence) is a SECONDARY + // concern and MUST NOT be able to discard a successful merge. Propagating a + // persist error here (`?`) previously made the caller drop the whole import + // (session_io) so the peer never projected the remote edits/presence — a + // one-directional-sync failure. Log and keep the merge in memory instead; + // the segment persistence self-heals (re-snapshots) in `persist_update_segment`. + // One revision sync + one segment persist covers the WHOLE chunk. + let probe_tail = std::time::Instant::now(); + if let Some(package) = &mut self.package + && let Err(error) = package.sync_revisions_from_loro(&self.doc) + { + tracing::error!(%error, "syncing revisions after remote import failed; kept the merged update in memory"); + } + let probe_revisions = probe_tail.elapsed(); + let probe_tail = std::time::Instant::now(); + if let Err(error) = self.persist_update_from_last_frontier() { + tracing::error!(%error, "persisting merged remote update failed; kept the merge in memory (durability degraded until the next successful save)"); + fidelity::event(FidelityClass::Persistence, "remote-persist-failed", || format!("{error:#}")); + } + if import_probe { + eprintln!( + "[flowstate-import-probe] tail: revisions={probe_revisions:?} persist={:?}", + probe_tail.elapsed() + ); + } + } + Ok(replies) + } + + fn projection_event(&mut self, invalidation: ProjectionInvalidation) -> Result { + self.record_projection_fallback(&invalidation); + let document = Box::new(self.projection_snapshot()?); + // Ordered stream (field fix): the editor consumes this replace in commit + // order, never through a racing side channel. + self.push_editor_stream(gpui_flowtext::ProjectionStreamItem::Replace(document.clone())); + Ok(RuntimeEvent::ProjectionUpdated { + document, + invalidation, + frontier: self.doc.state_frontiers().encode(), + version_vector: self.doc.state_vv().encode(), + }) + } + + pub fn export_updates_for(&self, remote_vv: &VersionVector) -> Result> { + // §A12.1.3 slice 4: a shallow live doc's updates export SILENTLY + // truncates below its root — a far-behind puller would import nothing + // usable and spin on its pending-retry loop forever. Detect the + // below-root request and serve it from the reconstructed full-history + // doc (load-serve-drop sidecar) instead. + if self.doc.is_shallow() { + let root = self.doc.shallow_since_vv(); + let below_root = root + .iter() + .any(|(peer, counter)| remote_vv.get(peer).copied().unwrap_or(0) < *counter); + if below_root { + let package = self + .package + .as_ref() + .context("a shallow doc without its package cannot serve below-root anti-entropy")?; + let full = package + .reconstruct_full_doc(&self.doc) + .context("reconstructing full history for below-root anti-entropy")?; + return full + .export(ExportMode::updates(remote_vv)) + .context("exporting full-history Loro updates for anti-entropy"); + } + } + self + .doc + .export(ExportMode::updates(remote_vv)) + .context("exporting Loro updates for anti-entropy") + } + + /// §A12.1.3 slice 4 ("rebase reopen", design §5): merge remote ops that + /// depend on pre-root history into the PACKAGE via the reconstructed + /// full-history doc, persist, and tell the session a reopen is required. + /// The in-memory shallow doc is left untouched — it cannot hold these ops. + fn absorb_below_root_import(&mut self, chunks: &[&[u8]]) -> Result { + let package = self + .package + .as_mut() + .context("a shallow doc without its package cannot absorb below-root history")?; + let full = package + .reconstruct_full_doc(&self.doc) + .context("reconstructing full history to absorb a below-root import")?; + for bytes in chunks { + full + .import_with(bytes, "remote") + .context("importing below-root remote update into the full-history doc")?; + } + package + .compact_to_snapshot(&full) + .map_err(|error| anyhow::anyhow!("persisting the below-root merge: {error}"))?; + if let Some(path) = self.package_path.clone() { + package + .write(&path) + .map_err(|error| anyhow::anyhow!("writing the below-root merge to disk: {error}"))?; + self.package_journal_prepared = true; + } + let merged_frontier = full.state_frontiers().encode(); + tracing::warn!("below-root remote history merged into the package; the session must reopen the document to see it"); + fidelity::event(FidelityClass::Persistence, "below-root-absorb", || { + format!("chunks={} merged_frontier_len={}", chunks.len(), merged_frontier.len()) + }); + Ok(RuntimeEvent::HistoryRebaseRequired { merged_frontier }) + } + + pub fn missing_dependency_request(status: &ImportStatus) -> Option<&VersionRange> { + status.pending.as_ref() + } + + pub fn save_package(&mut self) -> io::Result<()> { + let Some(package) = &self.package else { + return Ok(()); + }; + let Some(path) = &self.package_path else { + return Ok(()); + }; + package.write(path)?; + self.package_journal_prepared = true; + Ok(()) + } + + pub(crate) fn projection_change_event(&mut self, before: &DocumentProjection, invalidation: ProjectionInvalidation) -> Result { + if let Some(patches) = projection_patches_between(before, &self.projection) { + self.record_projection_fallback(&invalidation); + let batch = ProjectionPatchBatch { + transaction_id: uuid::Uuid::new_v4().as_u128(), + base_frontier: before.frontier.clone(), + new_frontier: self.doc.state_frontiers().encode(), + patches: patches.into(), + }; + self.push_editor_stream(gpui_flowtext::ProjectionStreamItem::Patches(batch.clone())); + return Ok(RuntimeEvent::ProjectionPatched { + batch, + invalidation, + version_vector: self.doc.state_vv().encode(), + }); + } + self.projection_event(ProjectionInvalidation::full_rebuild( + invalidation.frontier_before, + invalidation.frontier_after, + "projection_diff_ambiguous", + )) + } + + pub(crate) fn projection_patched_event( + &mut self, + patches: std::sync::Arc<[flowstate_document::ProjectionPatch]>, + invalidation: ProjectionInvalidation, + ) -> RuntimeEvent { + // §fidelity: the single choke point for every incrementally-patched projection + // emission (local semantic commands + remote non-structural imports). Verify + // the maintained projection still matches a fresh full rebuild. + self.fidelity_verify_incremental_projection("projection-patched"); + let batch = ProjectionPatchBatch { + transaction_id: uuid::Uuid::new_v4().as_u128(), + base_frontier: invalidation.frontier_before.clone(), + new_frontier: self.doc.state_frontiers().encode(), + patches, + }; + // Ordered stream (field fix): the editor consumes this batch in commit + // order, never through a racing side channel. + self.push_editor_stream(gpui_flowtext::ProjectionStreamItem::Patches(batch.clone())); + RuntimeEvent::ProjectionPatched { + batch, + invalidation, + version_vector: self.doc.state_vv().encode(), + } + } + + fn record_projection_fallback(&self, invalidation: &ProjectionInvalidation) { + if !invalidation.rebuild_required { + return; + } + let reason = invalidation + .fallback_reason + .unwrap_or("unspecified_projection_fallback"); + if let Ok(mut counts) = self.projection_fallback_counts.lock() { + *counts.entry(reason.to_string()).or_default() += 1; + } + fidelity::event(FidelityClass::Projection, "full-rebuild-fallback", || format!("reason={reason}")); + tracing::warn!(reason, "Flowstate projection used a full rebuild fallback"); + } + + /// §fidelity: when heavy tracing is enabled, verify the incrementally-maintained + /// `self.projection` still equals a fresh full projection built from canonical + /// Loro state via [`document_from_loro`]. A mismatch means an incremental patch + /// diverged from the authoritative materializer (kind + /// `incremental-vs-full-divergence`). Read-only; cheap firehose tracing does not + /// run this full reprojection because it perturbs large-document profiles. + fn fidelity_verify_incremental_projection(&self, context: &str) { + if !fidelity::expensive_checks_enabled() { + return; + } + match document_from_loro(&self.doc) { + Ok(fresh) => { + fidelity::check( + projections_semantically_equal(&self.projection, &fresh), + FidelityClass::Projection, + "incremental-vs-full-divergence", + || { + format!( + "{context}: incremental projection diverged from full rebuild [first_divergence: {}] (incremental_paragraphs={}, full_paragraphs={}, incremental_blocks={}, full_blocks={})", + first_projection_divergence(&self.projection, &fresh), + self.projection.paragraphs.len(), + fresh.paragraphs.len(), + self.projection.blocks.len(), + fresh.blocks.len(), + ) + }, + ); + }, + Err(error) => fidelity::event(FidelityClass::Projection, "full-rebuild-verify-error", || format!("{context}: {error}")), + } + } + + /// §fidelity: log a canonical frontier transition and assert the version only + /// advances (never regresses). Local edits and remote imports are monotone + /// merges, so `before_vv <= after_vv`; a regression (kind `frontier-regressed`) + /// signals canonical-state corruption. No-op (one atomic load) when off. + fn fidelity_frontier_transition(&self, context: &'static str, before_frontier: &Frontiers, before_vv: &VersionVector) { + if !fidelity::enabled() { + return; + } + let after_frontier = self.doc.state_frontiers(); + let after_vv = self.doc.state_vv(); + fidelity::event(FidelityClass::Frontier, context, || { + format!("frontier {:?} -> {:?}", before_frontier.encode(), after_frontier.encode()) + }); + fidelity::check( + matches!( + before_vv.partial_cmp(&after_vv), + Some(std::cmp::Ordering::Less | std::cmp::Ordering::Equal) + ), + FidelityClass::Frontier, + "frontier-regressed", + || { + format!( + "{context}: canonical version regressed (before_vv={:?}, after_vv={:?})", + before_vv.encode(), + after_vv.encode() + ) + }, + ); + } + + /// §fidelity: assert an undo/redo introduced only local-peer operations. Remote- + /// origin changes are excluded from the undo stack + /// (`add_exclude_origin_prefix("remote")`), so an undo/redo must never advance a + /// foreign peer's version. Violation kind `remote-origin-op-in-undo`. No-op off. + fn fidelity_check_undo_local_only(&self, op: &str, before_vv: &VersionVector) { + if !fidelity::enabled() { + return; + } + let after_vv = self.doc.state_vv(); + let local_peer = self.doc.peer_id(); + let foreign: Vec<(u64, i32)> = after_vv + .iter() + .filter_map(|(peer, counter)| { + let before = before_vv.get(peer).copied().unwrap_or(0); + (*peer != local_peer && *counter > before).then_some((*peer, *counter - before)) + }) + .collect(); + fidelity::check(foreign.is_empty(), FidelityClass::Undo, "remote-origin-op-in-undo", || { + format!("{op} advanced non-local peers {foreign:?} (local_peer={local_peer}); remote-origin ops must be excluded from undo") + }); + } + + pub fn projection_fallback_stats(&self) -> ProjectionFallbackStats { + let by_reason = self + .projection_fallback_counts + .lock() + .map(|counts| counts.clone()) + .unwrap_or_default(); + ProjectionFallbackStats { + total: by_reason.values().copied().sum(), + by_reason, + } + } + + /// §P2a: telemetry snapshot of projection-defect repair attempts, keyed by + /// defect `stable_key`. Mirrors [`Self::projection_fallback_stats`]. + pub fn projection_repair_stats(&self) -> ProjectionFallbackStats { + let by_reason = self + .projection_repair_counts + .lock() + .map(|counts| counts.clone()) + .unwrap_or_default(); + ProjectionFallbackStats { + total: by_reason.values().copied().sum(), + by_reason, + } + } + + /// §P2a: record and return the repair-attempt count for `stable_key`. + fn record_projection_repair_attempt(&self, stable_key: &str) -> u64 { + if let Ok(mut counts) = self.projection_repair_counts.lock() { + let entry = counts.entry(stable_key.to_string()).or_default(); + *entry += 1; + *entry + } else { + // A poisoned lock cannot account attempts; treat as over-cap so we quarantine + // rather than risk an unbounded repair loop. + PROJECTION_REPAIR_ATTEMPT_CAP + 1 + } + } + + /// §P2a: apply the idempotent canonical repair for each reported projection + /// defect, then commit the batch under the `repair` origin, re-project, persist + /// the update segment, and return the `LocalUpdate` (+ `ProjectionUpdated`) + /// events so peers receive the repair. + /// + /// `emission` decides whether the repaired projection is ALSO pushed onto + /// the ordered editor stream: `Streamed` when the repair pass runs after a + /// patched emission (the editor is at the pre-repair frontier and must be + /// advanced), `Silent` when an enclosing full-rebuild emission will carry + /// the repaired state itself (a stream item here would be out of order). + /// + /// Defects are deduplicated by `stable_key` and capped at + /// [`PROJECTION_REPAIR_ATTEMPT_CAP`] attempts per key: a defect that persists + /// across repair passes is logged and quarantined (left as the deterministic + /// projection) rather than retried forever. Convergence under concurrent + /// multi-peer repair is guaranteed by the check-before-write, stable-key-keyed + /// mutations in [`projection_repair`] plus Loro map/mark LWW semantics. + /// + /// Re-entrant calls (the inner refresh re-projecting after a repair) are + /// no-ops via the `repairing_projection_defects` guard. + #[hotpath::measure] + pub fn schedule_projection_repairs(&mut self, defects: Vec, emission: RepairEmission) -> Result> { + if self.repairing_projection_defects || defects.is_empty() { + return Ok(Vec::new()); + } + let mut seen = FxHashSet::default(); + let mut actionable = Vec::new(); + for defect in defects { + let stable_key = defect.stable_key(); + if !seen.insert(stable_key.clone()) { + continue; + } + let attempts = self.record_projection_repair_attempt(&stable_key); + if attempts > PROJECTION_REPAIR_ATTEMPT_CAP { + tracing::error!( + stable_key = %stable_key, + class = defect.class(), + attempts, + cap = PROJECTION_REPAIR_ATTEMPT_CAP, + "projection defect exceeded repair attempt cap; quarantining (leaving deterministic projection)" + ); + continue; + } + // §fidelity: record each defect queued for canonical repair, with its class, + // stable key, and this-pass attempt count (bounded by the attempt cap). + fidelity::event(fidelity_class_for_defect(&defect), "repair-scheduled", || { + format!("class={} key={stable_key} attempt={attempts}", defect.class()) + }); + actionable.push(defect); + } + if actionable.is_empty() { + return Ok(Vec::new()); + } + + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + let streamed = matches!(emission, RepairEmission::Streamed); + self.repairing_projection_defects = true; + let mut applied = 0_usize; + for defect in &actionable { + match projection_repair::apply_projection_repair(&self.doc, defect) { + Ok(true) => applied += 1, + Ok(false) => {}, + Err(error) => { + tracing::error!(%error, stable_key = %defect.stable_key(), class = defect.class(), "applying projection defect repair failed"); + }, + } + } + if applied == 0 { + self.repairing_projection_defects = false; + return Ok(Vec::new()); + } + + // §registry hygiene (field, 2026-07-07 act two): a MASS repair pass — an + // undo restoring a whole-document deletion fabricates a fresh + // anchor-keyed record per boundary — leaves the previous generation of + // records with permanently dead cursors in the registry. They project as + // nothing but every registry scan (`boundary_cursor_positions`, the + // fallback key scans) pays for them FOREVER: repeated mass undos + // compounded the registry to hundreds of thousands of records and pushed + // scan P95 to seconds. Fold the existing convergent stale/duplicate prune + // into the same repair-origin commit whenever a pass writes many records. + const REPAIR_PRUNE_THRESHOLD: usize = 32; + if applied >= REPAIR_PRUNE_THRESHOLD { + let body = body_text(&self.doc); + match prune_stale_paragraph_metadata(&self.doc, &body) { + Ok(pruned) if pruned.changed() => tracing::info!( + stale_paragraphs = pruned.stale_paragraphs, + duplicate_paragraphs = pruned.duplicate_paragraphs, + stale_blocks = pruned.stale_blocks, + duplicate_blocks = pruned.duplicate_blocks, + "pruned stale metadata records alongside a mass repair pass" + ), + Ok(_) => {}, + Err(error) => tracing::error!(%error, "pruning stale metadata during repair pass failed"), + } + } + + // Commit the whole repair batch atomically under the dedicated origin so it is + // excluded from undo and identifiable by peers. + self.doc.set_next_commit_origin(REPAIR_ORIGIN); + self.doc.commit(); + + // §fidelity: a repair pass must make progress — re-projecting the repaired + // canonical state must not surface MORE defects than it started with. A + // genuinely repairable defect count drops; an unrepairable one is bounded by + // the per-key attempt cap (`PROJECTION_REPAIR_ATTEMPT_CAP`) rather than + // growing (kind `repair-not-converging`). Gated so it costs nothing when off. + if fidelity::enabled() { + let scheduled = actionable.len(); + let remaining = document_from_loro_with_defects(&self.doc) + .map(|(_, defects)| defects.len()) + .unwrap_or(0); + fidelity::check(remaining <= scheduled, FidelityClass::Structure, "repair-not-converging", || { + format!("repair pass scheduled {scheduled} defect(s) but {remaining} remain after re-projection (cap={PROJECTION_REPAIR_ATTEMPT_CAP})") + }); + } + + // Encode the pre-repair frontier before `from_frontier` is consumed by + // `persist_update_segment` below. + let repair_frontier_before = from_frontier.encode(); + let mut events = Vec::new(); + match self.local_update_bytes(&from_vv) { + Ok(update) if !update.is_empty() => { + if let Err(error) = self.persist_update_segment_lazy(from_frontier, from_vv, update.clone()) { + tracing::error!(%error, "persisting projection repair update segment failed"); + } + events.push(RuntimeEvent::LocalUpdate { + bytes: update, + frontier: self.doc.state_frontiers().encode(), + version_vector: self.doc.state_vv().encode(), + }); + }, + Ok(_) => {}, + Err(error) => tracing::error!(%error, "exporting projection repair update failed; later synchronization must recover it"), + } + // §task #40 (BARE heal): `Missing{ParagraphMetadata,ParagraphBlock, + // ParagraphStyleMark}` repairs are PROJECTION-NEUTRAL BY CONSTRUCTION — + // the repaired record carries exactly the id the projection already + // fabricated (the shared `stable_boundary_metadata_keys` + + // `loro_id_u128` law) and the repaired style mark writes the same + // `Normal` the projector already defaulted to, so re-deriving the + // projection reproduces it bit-identically. Skip the whole-doc refresh + // for such batches: retire the repair commit's own subscription summary + // via the same epoch bump a refresh performs, advance the maintained + // projection's frontier, and stream a frontier-advancing (empty-diff) + // batch so the editor never falls behind the authority. Every + // content-touching class (anchor/placeholder deletions, table topology, + // asset ids) and every mass pass (prune threshold) keeps the flat + // rebuild below. + // Boundary-0 Missing* repairs are the ONE non-neutral case in the family: + // the shared key law special-cases boundary 0 to `paragraph.initial`, so + // when a remote sentinel lands at the doc head the repair REASSIGNS the + // existing record there — displacing whichever boundary it anchored + // before (healed by the follow-up pass below). Anchor-keyed repairs can + // never displace (the key derives from the boundary's own char op). + let neutral_batch = streamed + && applied < REPAIR_PRUNE_THRESHOLD + && actionable.iter().all(|defect| match defect { + ProjectionDefect::MissingParagraphMetadata { boundary_unicode, .. } + | ProjectionDefect::MissingParagraphBlock { boundary_unicode, .. } => *boundary_unicode != Some(0), + ProjectionDefect::MissingParagraphStyleMark { .. } => true, + _ => false, + }); + if neutral_batch { + self.bump_runtime_epoch(); + self.repairing_projection_defects = false; + self.projection.frontier = self.doc.state_frontiers().encode(); + let invalidation = ProjectionInvalidation { + frontier_before: repair_frontier_before, + frontier_after: self.projection.frontier.clone(), + ..Default::default() + }; + // An EMPTY patch batch: content unchanged, base→new frontier advanced. + // (`projection_patched_event`'s fidelity hook re-verifies maintained == + // full rebuild when tracing is on — the neutrality claim's own net.) + let empty: std::sync::Arc<[ProjectionPatch]> = std::sync::Arc::from(Vec::new()); + events.push(self.projection_patched_event(empty, invalidation)); + // §act-twelve: the repair commit advanced the frontier AFTER the import + // reconcile re-armed the recorded fast-path tops — re-arm again or the + // next undo clears a perfectly valid stack (the bench tail undo fell to + // the slow path through every BARE heal). Repairs are position-neutral + // to recorded coordinates (registry records + boundary marks); the + // excluded-origin compose is covered by `survives_pending_remote` + + // vendor #22's buffered-diff transform. + crate::local_write::recorded_inverse::rearm_recorded_inverse_frontier(self); + return Ok(events); + } + // Re-project wholesale onto the repaired canonical state. (Measured: a + // ladder-based derivation here LOST to the flat rebuild on large repair + // batches — a mass-undo repair pass went 26s → 35s — and repair passes on + // small batches are rare enough that the ~300ms rebuild is acceptable.) + let before_projection = streamed.then(|| self.projection.clone()); + let refresh_result = self.refresh_projection(); + self.repairing_projection_defects = false; + let follow_up_defects = refresh_result?; + let invalidation = + ProjectionInvalidation::full_rebuild(repair_frontier_before, self.doc.state_frontiers().encode(), "projection_defect_repair"); + match before_projection { + // Ordered stream: after a PATCHED emission the repair commit advanced + // the canonical frontier past what the editor received, so the editor + // MUST get the repaired projection through the stream — constructing + // the event without streaming it leaves the editor one repair-commit + // behind the authority (frontier mismatch class). + Some(before_projection) => events.push(self.projection_change_event(&before_projection, invalidation)?), + // Silent: the caller (a full rebuild in flight, or runtime construction + // before any editor attaches) emits AFTER this repair pass with the + // final frontier, so a stream item here would land out of causal order + // (StaleFrontier class). The repair is still committed + persisted. + None => events.push(RuntimeEvent::ProjectionUpdated { + document: Box::new(self.projection.clone()), + invalidation, + frontier: self.projection.frontier.clone(), + version_vector: self.doc.state_vv().encode(), + }), + } + // §task #40: a repair can DISPLACE a record (the boundary-0 + // `paragraph.initial` reassignment orphans the boundary it anchored + // before), and the refresh above is where the displaced boundary's defect + // surfaces. The old re-entrancy guard silently DROPPED it — the boundary + // stayed un-healed until an arbitrarily-later import, and every peer that + // imported the stranded state then paid its OWN heal commit (unsynced + // repair ops that stall dripped deltas behind Loro's pending buffer until + // anti-entropy). Follow up in the same call instead; termination is + // guaranteed by the per-key attempt cap (a persistent defect quarantines + // rather than looping). + if !follow_up_defects.is_empty() { + match self.schedule_projection_repairs(follow_up_defects, emission) { + Ok(more) => events.extend(more), + Err(error) => tracing::error!(%error, "follow-up projection repair pass failed"), + } + } + // Same re-arm as the neutral-skip exit (see above): the repair commits + // advanced the frontier past the import reconcile's re-arm. + crate::local_write::recorded_inverse::rearm_recorded_inverse_frontier(self); + Ok(events) + } + + /// Returns the defects the rebuild surfaced WHEN a repair pass is in flight + /// (the re-entrancy guard suppresses scheduling them here; the repair pass + /// follows them up itself — §task #40); otherwise schedules them internally + /// and returns empty. + #[hotpath::measure] + pub(crate) fn refresh_projection(&mut self) -> Result> { + let current_assets = self.projection.assets.clone(); + // §P2a: a full rebuild is where malformed canonical state surfaces; collect + // the projection defects so we can schedule their canonical repair. + let (mut projection, defects) = document_from_loro_with_defects(&self.doc).context("refreshing projection from canonical Loro state")?; + if let Some(package) = &self.package { + attach_package_assets(&mut projection, package); + } + for (id, record) in current_assets.assets { + projection.assets.assets.insert(id, record); + } + projection.theme = self.projection.theme.clone(); + self.projection = projection; + self.projection_index = ProjectionRuntimeIndex::from_projection(&self.projection); + // §23: a full rebuild discards the meaning of any incremental summary buffered + // before this point. Every full-rebuild path (local structural fallback, remote + // non-structural fallback, and the pending/again-changed remote import that + // forces a rebuild) routes through here, so bumping once here covers them all. + self.bump_runtime_epoch(); + // §P2a: schedule canonical repair for any defects. The re-entrancy guard + // (`repairing_projection_defects`) stops the inner re-projection that + // `schedule_projection_repairs` performs from recursing back into a repair + // pass — those defects are RETURNED for the repair pass's follow-up + // instead (§task #40). Repairs committed here are persisted (durable + + // anti-entropy), so peers converge even though this low-level helper + // cannot surface the repair's `LocalUpdate` event to the caller. + if self.repairing_projection_defects { + return Ok(defects); + } + if !defects.is_empty() + && let Err(error) = self.schedule_projection_repairs(defects, RepairEmission::Silent) + { + tracing::error!(%error, "scheduling projection repairs after projection refresh failed"); + } + Ok(Vec::new()) + } + + /// §23: advance the runtime epoch after a full projection reset/rebuild. + /// + /// `merge_subscription_invalidation` reads the live epoch and discards buffered + /// summaries stamped at an earlier epoch, so the permanent subscription stays + /// correct without depending on synchronous drain timing around import/checkout. + fn bump_runtime_epoch(&self) { + let previous = self.runtime_epoch.fetch_add(1, AtomicOrdering::SeqCst); + tracing::trace!( + previous_epoch = previous, + new_epoch = previous.wrapping_add(1), + "Flowstate runtime epoch bumped after full projection rebuild" + ); + } + + #[hotpath::measure] + pub(crate) fn apply_projection_patch_set(&mut self, patches: &[ProjectionPatch]) { + let (rebuild_index, table_refresh) = self + .projection_index + .update_for_patches(&self.projection, patches); + let batch = ProjectionPatchBatch { + transaction_id: uuid::Uuid::new_v4().as_u128(), + base_frontier: self.projection.frontier.clone(), + new_frontier: self.doc.state_frontiers().encode(), + patches: std::sync::Arc::from(patches), + }; + if let Err(error) = apply_projection_patch_batch(&mut self.projection, &batch) { + tracing::warn!(%error, "incremental runtime projection patch failed; refreshing projection"); + if let Err(error) = self.refresh_projection() { + tracing::error!(%error, "refreshing projection after patch failure failed"); + } + return; + } + if rebuild_index { + self.projection_index = ProjectionRuntimeIndex::from_projection(&self.projection); + } else { + self + .projection_index + .refresh_object_entries(&self.projection, &table_refresh); + // §act-ten A10.3 oracle (debug builds only): the incrementally-updated + // index must equal a fresh full rebuild — the fuzz suites run this on + // every spliced Enter/join/delete shape. + #[cfg(debug_assertions)] + self + .projection_index + .debug_assert_matches_fresh(&self.projection); + } + } + + pub fn save_package_to(&mut self, path: impl AsRef) -> io::Result<()> { + self.package_path = Some(path.as_ref().to_path_buf()); + self.package_journal_prepared = false; + self.save_package() + } + + /// Field fix 2026-07-07: phase 1 of a split checkpoint — everything that + /// must see the LIVE doc (metadata + revision commit, update export/persist) + /// plus a fork and the package CHECKOUT, all under one short gate hold. The + /// heavy assembly then runs off-gate/off-loop via [`CheckpointJob::run`], + /// and [`Self::finish_checkpoint`] restores the package. Returns `None` when + /// no package exists yet (first save) — callers fall back to the in-place + /// path, which is rare and user-initiated. + /// + /// While the package is checked out, import-side revision syncs and segment + /// persistence skip + self-heal on the next pass (their existing behavior + /// for package-less runtimes). + pub fn begin_checkpoint(&mut self, title: &str, path: Option) -> io::Result)>> { + if self.package.is_none() { + return Ok(None); + } + let revision_id = Uuid::new_v4().as_u128(); + let revision_frontiers = self.doc.state_frontiers(); + let revision_frontier = revision_frontiers.encode(); + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + // §A13.4.1: was the maintained projection current BEFORE this function's + // own commits? The commits below (metadata touch + revision record) are + // meta-only — they never touch body flows — so a projection current here + // is still current at the post-commit frontier (restamped at job build). + let projection_was_current = self.projection.frontier == from_frontier.encode(); + static BEGIN_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + let begin_probe = *BEGIN_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_CHECKPOINT_PROBE").is_some()); + let probe_t = std::time::Instant::now(); + flowstate_document::touch_document_metadata(&self.doc).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + flowstate_document::record_revision(&self.doc, revision_id, revision_frontier, title, "Explicit save", self.author_user_id) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + // The metadata/revision commit is undo-inert ("meta" origin): keep the + // recorded fast-undo timelines alive across it (§act-ten A10.2). + crate::local_write::recorded_inverse::rearm_recorded_inverse_frontier(self); + let mut revision_invalidation = ProjectionInvalidation::default(); + self.merge_subscription_invalidation(&mut revision_invalidation); + let probe_commits = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + let update = self + .local_update_bytes(&from_vv) + .map_err(|error| io::Error::other(error.to_string()))?; + let probe_export = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + let mut events = Vec::new(); + if !update.is_empty() { + let event_update = update.clone(); + self + .persist_update_segment(from_frontier, from_vv, update) + .map_err(|error| io::Error::other(error.to_string()))?; + events.push(RuntimeEvent::LocalUpdate { + bytes: event_update, + frontier: self.doc.state_frontiers().encode(), + version_vector: self.doc.state_vv().encode(), + }); + } + let probe_persist = probe_t.elapsed(); + if let Some(path) = path { + self.package_path = Some(path); + self.package_journal_prepared = false; + } + let package = self.package.take().expect("checked above"); + let mut projection = self.projection.clone(); + if projection_was_current { + // Meta-only commits above: the projection content is unchanged, so it + // is current at the post-commit frontier too (§A13.4.1). + projection.frontier = self.doc.state_frontiers().encode(); + } + if begin_probe { + eprintln!( + "[flowstate-begin-probe] commits={probe_commits:?} export={probe_export:?} persist={probe_persist:?} fork=skipped(package-derived)" + ); + } + let job = CheckpointJob { + // §A13.4.4: no gate-held fork — the job reconstructs from the package. + fork: None, + projection, + package, + title: title.to_string(), + revision_id, + revision_frontiers, + author_user_id: self.author_user_id, + peer_id: self.doc.peer_id(), + write_path: self.package_path.clone(), + record_revision: true, + }; + Ok(Some((job, events))) + } + + /// §P3 (act two): a revision-LESS split checkpoint for document close/idle. + /// Rebuilds the projection + search caches and compacts the package + /// off-gate so the NEXT preview/open of this file takes the cache fast path + /// instead of a multi-second full Loro materialization + /// (`append_update_segment` nulls the cache on every edit, so an edited + /// document otherwise never previews fast again until an explicit save). + /// Returns `None` when there is no package or the cache is already + /// frontier-current. + pub fn begin_cache_flush(&mut self) -> Option { + let package = self.package.as_ref()?; + let latest = self.doc.state_frontiers().encode(); + if package.manifest.projection_cache_frontier.as_deref() == Some(latest.as_slice()) && package.manifest.latest_frontier == latest { + return None; + } + let package = self.package.take().expect("checked above"); + Some(CheckpointJob { + // The cache flush persists no trailing segment, so the package may lag + // the doc — a gate-held fork stays correct here (close/idle path). + fork: Some(self.doc.fork()), + projection: self.projection.clone(), + package, + title: String::new(), + revision_id: 0, + revision_frontiers: self.doc.state_frontiers(), + author_user_id: self.author_user_id, + peer_id: self.doc.peer_id(), + write_path: self.package_path.clone(), + record_revision: false, + }) + } + + /// Phase 3 of a split checkpoint: restore the (now compacted + saved) + /// package under the gate. + pub fn finish_checkpoint(&mut self, package: DocumentPackage, wrote_to_disk: bool) { + self.package = Some(package); + if wrote_to_disk { + self.package_journal_prepared = true; + self.last_persisted_frontier = self.doc.state_frontiers(); + self.last_persisted_vv = self.doc.state_vv(); + } + } + + pub fn checkpoint_package(&mut self, title: &str, path: Option) -> io::Result> { + let revision_id = Uuid::new_v4().as_u128(); + let revision_frontiers = self.doc.state_frontiers(); + let revision_frontier = revision_frontiers.encode(); + let from_frontier = self.doc.state_frontiers(); + let from_vv = self.doc.state_vv(); + flowstate_document::touch_document_metadata(&self.doc).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + flowstate_document::record_revision(&self.doc, revision_id, revision_frontier, title, "Explicit save", self.author_user_id) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + let mut revision_invalidation = ProjectionInvalidation::default(); + self.merge_subscription_invalidation(&mut revision_invalidation); + let update = self + .local_update_bytes(&from_vv) + .map_err(|error| io::Error::other(error.to_string()))?; + let mut events = Vec::new(); + if !update.is_empty() { + let event_update = update.clone(); + self + .persist_update_segment(from_frontier, from_vv, update) + .map_err(|error| io::Error::other(error.to_string()))?; + events.push(RuntimeEvent::LocalUpdate { + bytes: event_update, + frontier: self.doc.state_frontiers().encode(), + version_vector: self.doc.state_vv().encode(), + }); + } + if self.package.is_none() { + let package_creation_vv = self.doc.state_vv(); + let mut created = DocumentPackage::from_loro_snapshot_with_assets(&self.doc, title, assets_from_document(&self.projection))?; + // §act-twelve A12.1.1: the runtime maintains paragraph-style marks, so + // a package IT creates is marks-clean at its own frontier — later opens + // skip the whole-corpus verification scan (the raw + // `from_loro_snapshot*` constructors deliberately do not stamp). + created.manifest.style_marks_clean_frontier = Some(created.manifest.latest_frontier.clone()); + self.package = Some(created); + let package_creation_update = self + .local_update_bytes(&package_creation_vv) + .map_err(|error| io::Error::other(error.to_string()))?; + if !package_creation_update.is_empty() { + events.push(RuntimeEvent::LocalUpdate { + bytes: package_creation_update, + frontier: self.doc.state_frontiers().encode(), + version_vector: self.doc.state_vv().encode(), + }); + } + self.last_persisted_frontier = self.doc.state_frontiers(); + self.last_persisted_vv = self.doc.state_vv(); + } + let Some(package) = &mut self.package else { + return Ok(events); + }; + package.replace_assets_from_document(&self.projection)?; + // §act-ten A10.5: ONE materialization feeds both the projection cache and + // the search units (the separate rebuild calls each ran a full projection + // walk — the checkpoint path got this fused in §act-five P9, this in-place + // save path kept paying double). + package.rebuild_caches_from_loro(&self.doc)?; + package.compact_to_snapshot_any(&self.doc)?; + package.create_named_revision_at_with_id( + &self.doc, + revision_id, + &revision_frontiers, + title, + "Explicit save", + self.author_user_id, + Some(self.doc.peer_id() as u128), + )?; + if let Some(path) = path { + self.package_path = Some(path); + self.package_journal_prepared = false; + } + self.save_package()?; + // ALL the metadata/revision commits above ("meta" origin — the explicit + // record_revision plus the ones package creation and the named-revision + // write perform) are undo-inert: re-arm the recorded fast-undo timelines + // to the final frontier so the next Ctrl-Z stays fast (§act-ten A10.2, + // vendor patch #14). Must run LAST — any later meta commit would re-break + // the expected-frontier equality. + crate::local_write::recorded_inverse::rearm_recorded_inverse_frontier(self); + Ok(events) + } + + /// Field fix 2026-07-07 (I-9a completion): capture everything a package + /// assembly needs under a SHORT gate hold — a doc fork plus a shallow + /// projection clone. Assembly then runs off-gate and off the I/O loop; the + /// 18-second `package-bytes` gate holds in the field logs (typing frozen, + /// imports starved, peer pulls timing out) were the whole-assembly-under- + /// gate shape. + /// Read access to the runtime's package (diagnostics + gate tests). + pub fn package(&self) -> Option<&DocumentPackage> { + self.package.as_ref() + } + + pub fn package_export_context(&self) -> (LoroDoc, DocumentProjection) { + (self.doc.fork(), self.projection.clone()) + } + + /// Legacy in-place variant retained for gate-held callers that already own + /// the core exclusively (tests, seeding). Production package exports go + /// through `package_export_context` + `assemble_package_bytes`. + pub fn package_bytes(&mut self, title: &str) -> io::Result> { + let (fork, projection) = self.package_export_context(); + assemble_package_bytes(&fork, &projection, title) + } + + #[hotpath::measure] + pub(crate) fn events_after_local_change( + &mut self, + from_frontier: Frontiers, + from_vv: VersionVector, + invalidation: ProjectionInvalidation, + emit_projection: bool, + ) -> Result> { + let update = self.local_update_bytes(&from_vv)?; + let mut events = Vec::new(); + if !update.is_empty() { + self.persist_update_segment(from_frontier, from_vv.clone(), update.clone())?; + let frontier = self.doc.state_frontiers().encode(); + let version_vector = self.doc.state_vv().encode(); + for bytes in self.chunk_local_update(&from_vv, update) { + events.push(RuntimeEvent::LocalUpdate { + bytes, + frontier: frontier.clone(), + version_vector: version_vector.clone(), + }); + } + } + if emit_projection { + events.push(self.projection_event(invalidation)?); + } + Ok(events) + } + + /// §A14.1.1: split a MASS local update into bounded sub-updates so a + /// receiver imports them as SEPARATE gate holds — the A13.2 priority lane + /// admits its user's keystrokes between slices instead of stalling ~240ms + /// behind one mass hold (the alpha complaint). Each slice is a + /// counter-window of THIS peer's new ops (`updates_in_range`), so slices + /// arriving in emission order are causally self-contained. Updates that + /// carry OTHER peers' ops (re-broadcast merges — interleavable causality) + /// and small updates pass through unsplit. `FLOWSTATE_DISABLE_UPDATE_ + /// CHUNKING` reverts. + fn chunk_local_update(&self, from_vv: &VersionVector, update: Vec) -> Vec> { + /// Op atoms per slice: a restyle is ~2 atoms/paragraph, a keystroke 1 — + /// 2048 keeps a slice's apply+derive in the low single-digit ms. + const CHUNK_ATOMS: i32 = 4096; + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DISABLED.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_UPDATE_CHUNKING").is_some()) { + return vec![update]; + } + let now_vv = self.doc.oplog_vv(); + let peer = self.doc.peer_id(); + let start = from_vv.get(&peer).copied().unwrap_or(0); + let end = now_vv.get(&peer).copied().unwrap_or(start); + let only_our_ops = now_vv + .iter() + .all(|(other, counter)| *other == peer || from_vv.get(other).copied().unwrap_or(0) >= *counter); + if !only_our_ops || end.saturating_sub(start) <= CHUNK_ATOMS { + return vec![update]; + } + let mut chunks = Vec::new(); + let mut window_start = start; + while window_start < end { + let window_end = (window_start + CHUNK_ATOMS).min(end); + match self + .doc + .export(ExportMode::updates_in_range(&[loro::IdSpan::new(peer, window_start, window_end)])) + { + Ok(bytes) if !bytes.is_empty() => chunks.push(bytes), + Ok(_) => {}, + Err(error) => { + // Fail SAFE: one slice failing means publish the original whole + // update instead of a gappy sequence. + tracing::warn!(%error, "chunking a mass local update failed; publishing unsplit"); + return vec![update]; + }, + } + window_start = window_end; + } + if chunks.is_empty() { vec![update] } else { chunks } + } + + #[hotpath::measure] + pub(crate) fn local_update_bytes(&self, from_vv: &VersionVector) -> Result> { + let mut subscribed = self + .local_subscription_updates + .lock() + .map(|mut updates| std::mem::take(&mut *updates)) + .unwrap_or_default(); + if subscribed.len() == 1 { + return Ok(subscribed.pop().unwrap_or_default()); + } + self + .doc + .export(ExportMode::updates(from_vv)) + .context("exporting local Loro update fallback") + } + + /// §23: drain the permanent subscription buffer and fold the in-epoch, in-frontier + /// summaries into `invalidation`, filtering/processing by runtime epoch, emit-time + /// frontier, origin, and trigger. + /// + /// * Epoch — summaries stamped before the most recent full rebuild are discarded. + /// * Frontier — summaries stamped strictly ahead of `frontier_after` belong to a + /// later batch and are returned to the buffer for the next drain. + /// * Origin — a remote-origin summary sets `has_remote_origin` (telemetry/bias) + /// without forcing a rebuild, so the incremental remote fast paths still apply. + /// * Trigger — a checkout-triggered event forces a conservative full rebuild; + /// import/local triggers are left to the existing structural detection. + #[hotpath::measure] + pub(crate) fn merge_subscription_invalidation(&self, invalidation: &mut ProjectionInvalidation) -> DrainedBodyDelta { + let summaries = self + .subscription_events + .lock() + .map(|mut events| std::mem::take(&mut *events)) + .unwrap_or_default(); + let body_target = body_text(&self.doc).id().to_string(); + let current_epoch = self.runtime_epoch.load(AtomicOrdering::SeqCst); + // Loro-first spec §6.3: harvest per-summary body deltas (copied-out data, + // I-9b) so the import path can compose them into ONE exact pre→post + // delta instead of relying on the lossy union heuristics below. + let mut body_batches: Vec> = Vec::new(); + let mut checkout_seen = false; + // §23 FRONTIER: decode the drain target once so each summary can be compared + // causally. Empty/undecodable targets (e.g. revision bookkeeping drains) disable + // the frontier filter and fall back to draining everything in-epoch. + let target_frontier = if invalidation.frontier_after.is_empty() { + None + } else { + Frontiers::decode(&invalidation.frontier_after).ok() + }; + let mut deferred: Vec = Vec::new(); + let mut has_remote_origin = false; + let mut imported_paragraph_record_keys: Vec = Vec::new(); + let mut imported_block_record_keys: Vec = Vec::new(); + for summary in summaries { + // §23 EPOCH: drop summaries emitted before the latest full projection + // rebuild/reset; their incremental meaning no longer maps onto the projection. + if summary.epoch != current_epoch { + tracing::trace!( + summary_epoch = summary.epoch, + current_epoch, + origin = %summary.origin, + trigger = %summary.triggered_by, + "Flowstate discarding stale pre-reset Loro subscription summary", + ); + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + " drain: DISCARDED stale-epoch summary (epoch {} != {current_epoch}, origin {}, trigger {}, changes {})", + summary.epoch, + summary.origin, + summary.triggered_by, + summary.changes.len() + ); + } + continue; + } + // §23 FRONTIER: a summary stamped strictly ahead of the drain target is from a + // later batch. Re-buffer it instead of misattributing it to this invalidation. + if let Some(target) = target_frontier.as_ref() + && !summary.frontier.is_empty() + && let Ok(summary_frontier) = Frontiers::decode(&summary.frontier) + && matches!(self.doc.cmp_frontiers(&summary_frontier, target), Ok(Some(std::cmp::Ordering::Greater))) + { + tracing::trace!( + origin = %summary.origin, + trigger = %summary.triggered_by, + "Flowstate deferring Loro subscription summary stamped ahead of the drain frontier", + ); + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + " drain: DEFERRED ahead-of-frontier summary (origin {}, trigger {}, changes {})", + summary.origin, + summary.triggered_by, + summary.changes.len() + ); + } + deferred.push(summary); + continue; + } + // §23 ORIGIN: record remote-origin processing. The UndoManager keeps its + // `add_exclude_origin_prefix("remote")` exclusion unchanged. + has_remote_origin |= summary.origin.starts_with("remote"); + // §23 TRIGGER: checkout events reflect time-travel/detached state that cannot be + // expressed as an incremental projection patch, so force a conservative rebuild. + // Ordinary local and import triggers preserve the incremental fast paths. + if summary.triggered_by.eq_ignore_ascii_case("checkout") { + checkout_seen = true; + invalidation.rebuild_required = true; + invalidation.fallback_reason = Some("checkout_trigger_projection_rebuild"); + tracing::debug!(origin = %summary.origin, "Flowstate forcing projection rebuild for checkout-triggered Loro event"); + } + static DERIVE_DEBUG_PROCESS: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG_PROCESS.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + " drain: PROCESSING summary (origin {}, trigger {}, changes {:?})", + summary.origin, + summary.triggered_by, + summary + .changes + .iter() + .map(|change| format!("{change:?}")) + .collect::>() + ); + } + let mut batch_ops: Vec = Vec::new(); + let mut implied_cursor = 0usize; + for change in &summary.changes { + if let SubscriptionChange::Text { + target, + kind, + unicode_start, + unicode_len, + deleted_len, + inserted_structure, + } = change + && *target == body_target + { + if *unicode_start > implied_cursor { + batch_ops.push(import_delta::NetOp::Retain(unicode_start - implied_cursor)); + implied_cursor = *unicode_start; + } + match kind { + TextChangeKind::StyledRetain => { + batch_ops.push(import_delta::NetOp::Retain(*unicode_len)); + implied_cursor += unicode_len; + }, + TextChangeKind::Insert => { + batch_ops.push(import_delta::NetOp::Insert { + len: *unicode_len, + structure: *inserted_structure, + }); + implied_cursor += unicode_len; + }, + TextChangeKind::Delete => { + batch_ops.push(import_delta::NetOp::Delete(*deleted_len)); + }, + } + } + } + if !batch_ops.is_empty() { + body_batches.push(batch_ops); + } + for change in summary.changes { + match change { + SubscriptionChange::Text { + target, + unicode_start, + unicode_len, + deleted_len, + inserted_structure, + .. + } if target == body_target => { + if inserted_structure + || self + .projection_index + .deleted_range_contains_structure(unicode_start, deleted_len) + { + invalidation.rebuild_required = true; + invalidation.fallback_reason = Some("structural_body_text_change"); + } + invalidation + .changed_flows + .push(ROOT_BODY_FLOW_ID.to_string()); + invalidation.changed_text_ranges.push(ProjectionTextRange { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + unicode_start, + unicode_len, + }); + }, + SubscriptionChange::Text { target, .. } => invalidation.changed_flows.push(target), + SubscriptionChange::Map { target, keys } => { + // §6-R: harvest the registry record keys this batch touched — the + // regional rematerializer's identity candidates beyond the old + // projection's own rows. + if target == self.paragraph_registry_container_id || target == self.block_registry_container_id { + if target == self.paragraph_registry_container_id { + imported_paragraph_record_keys.extend(keys.iter().cloned()); + } else { + imported_block_record_keys.extend(keys.iter().cloned()); + } + // A key change ON the registry parent is a record ADDED or + // DELETED — the projection's row SET derives from these records, + // so this is structural even with zero body-text ops. The class + // that slipped through: an undo whose text component transforms + // away (the char was deleted and re-inserted with fresh ids in + // between) but whose record deletion survives — canonical drops + // the row (orphan placeholder) while a neutral drain left the + // maintained row standing (object-fuzz undo arm). Imports still + // take the regional path (this reason is not in its rebuild-only + // guard); local undo/redo takes the rebuild. + invalidation.rebuild_required = true; + if invalidation.fallback_reason.is_none() { + invalidation.fallback_reason = Some("registry_record_set_change"); + } + } + classify_map_invalidation(invalidation, &target, &keys); + // Every Map change may sit inside an object record's subtree + // (attrs, caption/alt/source flow records, nested table maps) — + // including opaque dynamically-created container ids that no path + // rule can recognize. Marking the batch object-dirty makes the + // ladder run the canonical object readback + // (`remote_object_projection_patches`: O(objects), batched, pure + // read, empty when nothing actually changed). The class that + // slipped through: undo of an image alignment change was + // body-neutral and left the maintained object row stale + // (object-fuzz undo arm). + invalidation.changed_blocks.push(target); + }, + SubscriptionChange::List { target } => invalidation.changed_blocks.push(target), + SubscriptionChange::Unknown { target } => { + invalidation.rebuild_required = true; + invalidation.fallback_reason = Some("unknown_loro_subscription_diff"); + invalidation.changed_blocks.push(target); + }, + } + } + } + // §23 ORIGIN: surface remote-origin processing for downstream consumers/telemetry. + invalidation.has_remote_origin |= has_remote_origin; + // §24: fold the auxiliary projection indexes into the accumulated invalidation + // before the sort/dedup below so any enrichment is normalized with the rest. + self.consume_projection_indexes(invalidation); + invalidation.changed_flows.sort(); + invalidation.changed_flows.dedup(); + invalidation.changed_blocks.sort(); + invalidation.changed_blocks.dedup(); + invalidation.changed_tables.sort(); + invalidation.changed_tables.dedup(); + invalidation.changed_assets.sort(); + invalidation.changed_assets.dedup(); + invalidation.changed_sections.sort(); + invalidation.changed_sections.dedup(); + // §23 FRONTIER: return later-batch summaries to the front of the buffer (ahead of + // anything emitted after this drain) so ordering is preserved on the next drain. + if !deferred.is_empty() + && let Ok(mut events) = self.subscription_events.lock() + { + let newly_buffered = std::mem::replace(&mut *events, deferred); + events.extend(newly_buffered); + } + DrainedBodyDelta { + net: import_delta::compose_batches(&body_batches), + has_remote_origin, + checkout_seen, + imported_paragraph_record_keys, + imported_block_record_keys, + } + } + + /// §24: consume the auxiliary projection indexes while merging an invalidation. + /// + /// Two functional, additive, conservative enrichments and several behavior- + /// neutral diagnostic reads: + /// * Asset reference index — a changed asset id (the `merge_asset_records` + /// path records ids in their `u128` form) is mapped to the image blocks that + /// embed it, whose ids are appended to `changed_blocks`. + /// * Table row/column/cell index — a table-map diff names the Loro container, + /// not the projected block id, so when any table changed every indexed table + /// block id is surfaced into `changed_blocks`. + /// + /// Downstream code only ever tests `changed_blocks`/`changed_tables` for + /// *emptiness* (never their contents), so this can only widen coverage and + /// never drops a needed invalidation. The section/style/search reads do not + /// mutate the invalidation; they map it onto the remaining indexes so those + /// materialized structures have live consumers for the incremental work §24 + /// builds toward. + fn consume_projection_indexes(&self, invalidation: &mut ProjectionInvalidation) { + let index = &self.projection_index; + + // Asset reference index → changed_blocks. + if !invalidation.changed_assets.is_empty() && !index.asset_refs_by_id.is_empty() { + let mut referenced = Vec::new(); + for asset in &invalidation.changed_assets { + if let Ok(asset_id) = asset.parse::() + && let Some(blocks) = index.asset_refs_by_id.get(&AssetId(asset_id)) + { + referenced.extend(blocks.iter().map(|block| block.0.to_string())); + } + } + invalidation.changed_blocks.extend(referenced); + } + + // Table row/column/cell index → changed_blocks (conservative). + if !invalidation.changed_tables.is_empty() && !index.table_cells_by_block.is_empty() { + let mut total_cells = 0usize; + let mut dense_cells = 0usize; + let mut table_blocks = Vec::new(); + for (block_id, entry) in &index.table_cells_by_block { + table_blocks.push(block_id.0.to_string()); + total_cells = total_cells.saturating_add(entry.cells.len()); + dense_cells = dense_cells.saturating_add(entry.row_ids.len().saturating_mul(entry.column_ids.len())); + } + tracing::trace!( + tables = index.table_cells_by_block.len(), + total_cells = total_cells, + dense_cells = dense_cells, + "Flowstate §24 table change surfaced indexed table blocks", + ); + invalidation.changed_blocks.extend(table_blocks); + } + + // Search-unit and style-interval indexes (diagnostic, behavior-neutral). Gated + // on the trace level so the O(spans) mapping never runs on the hot text-edit + // path in production; the field reads stay compiled so the indexes are live. + if !invalidation.changed_text_ranges.is_empty() && tracing::enabled!(tracing::Level::TRACE) { + let touched_units = index.search_units_for_changed_ranges(&invalidation.changed_text_ranges); + let mut paragraph_units = 0usize; + let mut object_units = 0usize; + let mut styled_units = 0usize; + for unit_ix in &touched_units { + match index + .search_unit_spans + .get(*unit_ix) + .and_then(|span| span.paragraph) + { + Some(paragraph_ix) => { + paragraph_units += 1; + if index + .run_styles_at(paragraph_ix, 0) + .is_some_and(|styles| styles != RunStyles::default()) + { + styled_units += 1; + } + }, + None => object_units += 1, + } + } + tracing::trace!( + touched_units = touched_units.len(), + paragraph_units = paragraph_units, + object_units = object_units, + styled_units = styled_units, + "Flowstate §24 changed body ranges mapped onto search-unit / style-interval indexes", + ); + } + + // Section anchor index (diagnostic, behavior-neutral). + if !invalidation.changed_sections.is_empty() { + let anchored = index + .section_anchor_by_id + .values() + .filter(|start| index.paragraph_metadata_by_id.contains_key(*start)) + .count(); + tracing::trace!( + changed_sections = invalidation.changed_sections.len(), + anchored_sections = anchored, + sections = index.section_anchor_by_id.len(), + "Flowstate §24 section change mapped onto section-anchor index", + ); + } + } + + fn persist_update_from_last_frontier(&mut self) -> Result<()> { + let from_frontier = self.last_persisted_frontier.clone(); + let from_vv = self.last_persisted_vv.clone(); + let update = self + .doc + .export(ExportMode::updates(&from_vv)) + .context("exporting accepted remote Loro update for persistence")?; + if update.is_empty() { + return Ok(()); + } + self.persist_update_segment(from_frontier, from_vv, update) + } + + #[hotpath::measure] + pub(crate) fn persist_update_segment(&mut self, from_frontier: Frontiers, from_vv: VersionVector, update: Vec) -> Result<()> { + self.persist_update_segment_with_durability(from_frontier, from_vv, update, true) + } + + /// §act-twelve A12.5.1: repair persists take the LAZY journal lane (no + /// per-transaction fsync) — repairs are re-derivable, torn tails drop at + /// replay, and any later synced append covers them. Everything else keeps + /// the synchronous lane. + pub(crate) fn persist_update_segment_lazy(&mut self, from_frontier: Frontiers, from_vv: VersionVector, update: Vec) -> Result<()> { + self.persist_update_segment_with_durability(from_frontier, from_vv, update, false) + } + + fn persist_update_segment_with_durability( + &mut self, + from_frontier: Frontiers, + from_vv: VersionVector, + update: Vec, + sync: bool, + ) -> Result<()> { + if let Some(package) = &mut self.package { + match package.append_update_segment(&from_frontier, &from_vv, &self.doc.state_frontiers(), &self.doc.state_vv(), update) { + Ok(_) => { + // §act-four M2: refresh the tiny preview header from the live + // projection (O(header)) so cold preview of this edited-but-unflushed + // document stays O(viewport). The header rides the journal delta to + // disk with the segment. A failure here must not fail the persist. + if let Err(error) = package.refresh_preview_header(&self.projection) { + tracing::warn!(%error, "refreshing preview header after edit failed; cold preview may fall back to the full read"); + } + // §act-eleven A11.3: inline compaction is a BACKSTOP, not the routine + // path — the debounced autosave checkpoint compacts OFF-thread every + // cycle (`CheckpointJob::run`), so paying a full O(doc+history) + // snapshot export inside an interactive op (keystroke/undo commit) + // is only justified when segments have grown far past the point a + // checkpoint should have handled (headless handles with no autosave + // wiring). 4x the routine threshold keeps the durability invariant + // with a bound while removing the periodic latency spike from the + // measured op (the recorded_inverse rows' hidden component). + let compacted = package.compact_update_segments_if_needed(&self.doc, DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD * 4)?; + if let Some(path) = &self.package_path { + if compacted.is_some() { + package.write(path)?; + self.package_journal_prepared = true; + } else if self.package_journal_prepared { + if sync { + package.append_latest_update_to_prepared_path(path)?; + } else { + package.append_latest_update_to_prepared_path_lazy(path)?; + } + } else { + package.append_latest_update_to_path(path)?; + self.package_journal_prepared = true; + } + } + }, + Err(error) => { + // The linear update-segment chain cannot represent this frontier + // transition — most often a concurrent multi-head merge whose + // `from_frontier` does not chain from the last persisted head. Rather + // than fail the persist (and, before the caller was hardened, lose the + // already-merged remote update), re-base the whole package onto a fresh + // snapshot of the current doc: a snapshot is always a valid, complete + // save. Fine-grained update-segment history folds into the snapshot; + // document data and convergence are preserved. + tracing::warn!(%error, "update-segment chain rejected a merge; re-basing the package onto a fresh snapshot"); + fidelity::event(FidelityClass::Persistence, "segment-chain-resnapshot", || format!("{error:#}")); + package.compact_to_snapshot_any(&self.doc)?; + if let Some(path) = &self.package_path { + package.write(path)?; + self.package_journal_prepared = true; + } + }, + } + } + self.last_persisted_frontier = self.doc.state_frontiers(); + self.last_persisted_vv = self.doc.state_vv(); + Ok(()) + } +} + +fn summarize_subscription_event(event: &DiffEvent<'_>) -> SubscriptionEventSummary { + let mut changes = Vec::new(); + for container in &event.events { + let target = container.target.to_string(); + match &container.diff { + Diff::Text(delta) => { + let mut cursor = 0usize; + for item in delta { + match item { + loro::TextDelta::Retain { retain, attributes } => { + if attributes.is_some() { + changes.push(SubscriptionChange::Text { + target: target.clone(), + kind: TextChangeKind::StyledRetain, + unicode_start: cursor, + unicode_len: *retain, + deleted_len: 0, + inserted_structure: false, + }); + } + cursor = cursor.saturating_add(*retain); + }, + loro::TextDelta::Insert { insert, .. } => { + let len = insert.chars().count(); + changes.push(SubscriptionChange::Text { + target: target.clone(), + kind: TextChangeKind::Insert, + unicode_start: cursor, + unicode_len: len, + deleted_len: 0, + inserted_structure: insert + .chars() + .any(|ch| ch == '\n' || ch == OBJECT_REPLACEMENT), + }); + cursor = cursor.saturating_add(len); + }, + loro::TextDelta::Delete { delete } => { + changes.push(SubscriptionChange::Text { + target: target.clone(), + kind: TextChangeKind::Delete, + unicode_start: cursor, + unicode_len: *delete, + deleted_len: *delete, + inserted_structure: false, + }); + }, + } + } + }, + Diff::Map(delta) => changes.push(SubscriptionChange::Map { + target, + keys: delta.updated.keys().map(|key| key.to_string()).collect(), + }), + Diff::List(_) => changes.push(SubscriptionChange::List { target }), + Diff::Tree(_) | Diff::Unknown => changes.push(SubscriptionChange::Unknown { target }), + Diff::Counter(_) => changes.push(SubscriptionChange::Unknown { target }), + } + } + SubscriptionEventSummary { + origin: event.origin.to_string(), + triggered_by: format!("{:?}", event.triggered_by), + // §23: `summarize_subscription_event` stays pure — it cannot read the runtime + // epoch or the live doc frontier. These are stamped by the root callback after + // summarization. The unstamped defaults (epoch 0 / empty frontier) are treated + // as "not stamped" by `merge_subscription_invalidation`. + epoch: 0, + frontier: Vec::new(), + changes, + } +} + +fn classify_map_invalidation(invalidation: &mut ProjectionInvalidation, target: &str, keys: &[String]) { + // §divergence: a change to a paragraph/block metadata record's durable identity + // or anchor alters which id a boundary resolves to. The incremental remote patch + // path applies content only and NEVER re-derives the paragraph_ids/block_ids + // arrays, so on such a change the incremental projection would freeze stale ids + // that diverge from the authoritative full rebuild (and clobber peers). Force a + // full rebuild — which re-resolves the ids — for any id-affecting record change. + // (Text-structural changes already force a rebuild above; this covers the + // non-structural record writes, e.g. a peer's repaired metadata record syncing.) + // Exemption: user/replica IDENTITY records also write an `"id"` key but never + // feed positional id resolution — rebuilding on them put a full O(doc) rebuild + // on every fresh peer's first update (field: "second peer's first op stalled"). + // Identity records are recognizable by their exclusive key set. + let identity_record = keys + .iter() + .any(|key| matches!(key.as_str(), "user_id" | "display_name" | "last_seen_at" | "created_at")); + if !identity_record + && keys + .iter() + .any(|key| matches!(key.as_str(), "id" | "boundary_cursor" | "start_cursor" | "anchor_cursor")) + { + invalidation.rebuild_required = true; + invalidation.fallback_reason = Some("metadata_record_id_change"); + } + if keys.iter().any(|key| { + matches!( + key.as_str(), + "asset_id" | "content_hash" | "mime_type" | "byte_length" | "dimensions" | "original_name" + ) + }) { + invalidation.changed_assets.push(target.to_string()); + } + if keys.iter().any(|key| { + matches!( + key.as_str(), + "row_order" | "rows_by_id" | "column_order" | "columns_by_id" | "cells_by_id" | "row_span" | "column_span" + ) + }) { + invalidation.changed_tables.push(target.to_string()); + } + if keys.iter().any(|key| { + matches!( + key.as_str(), + "kind" | "flow_id" | "anchor_cursor" | "attrs" | "nested_refs" | "external_url" + ) + }) { + invalidation.changed_blocks.push(target.to_string()); + } + if keys + .iter() + .any(|key| key == "section_id" || key == "sections_by_id") + { + invalidation.changed_sections.push(target.to_string()); + } +} + +/// §5 sentinel protection: the boundary-0 [`SENTINEL_NEWLINE`] anchors the first +/// paragraph and must never be deleted. Given a requested body delete of `len` +/// unicode chars starting at `start`, return the largest sub-range that leaves +/// position 0 intact, or `None` when nothing outside the sentinel remains to +/// delete. Clamping here in *preflight* keeps a malformed delete from +/// half-applying. A well-formed editor command never targets position 0 (the +/// first paragraph's text starts at unicode index 1), so the clamp only fires on +/// corruption or an explicit whole-document delete — which correctly keeps the +/// lone sentinel and drops everything after it. +pub(crate) fn sentinel_protected_delete_range(start: usize, len: usize) -> Option<(usize, usize)> { + if len == 0 { + return None; + } + let end = start.saturating_add(len); + let protected_start = start.max(1); + (end > protected_start).then_some((protected_start, end - protected_start)) +} + +/// §5 sentinel/object coupling: after a body text delete, an object block whose +/// U+FFFC placeholder was removed must not linger as a dangling record — it would +/// otherwise project as an `UnresolvedObjectAnchor` quarantine on every future +/// projection. Remove such body object blocks in the **same transaction** as the +/// delete so canonical state stays coherent (the paired paragraph metadata is +/// handled separately by [`repair_paragraph_metadata_after_text_flow_edit`]). +/// +/// Convergent: deletion is keyed on the block's stable map key, so two peers that +/// concurrently delete the same placeholder converge on the same removed record. +/// Returns the number of blocks pruned. +#[hotpath::measure] +pub(crate) fn prune_orphaned_body_object_blocks(doc: &LoroDoc, body: &loro::LoroText) -> loro::LoroResult { + let body_snapshot = body.to_string(); + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return Ok(0); + }; + // ONE batched resolver pass for every anchor cursor. A DELETED anchor simply + // fails to resolve (absent from the map) — which is exactly the orphan + // signal. The previous per-record `doc.get_cursor_pos` walked update history + // for each dead anchor, turning a select-all delete into an + // O(objects × history) multi-minute freeze (the 2026-07-07 ctrl-A field bug). + let anchor_pos = boundary_cursor_positions(doc, body, &blocks, &["anchor_cursor"]); + let placeholder_positions: Vec = body_snapshot + .chars() + .enumerate() + .filter_map(|(pos, ch)| (ch == OBJECT_REPLACEMENT).then_some(pos)) + .collect(); + let mut pruned = 0_usize; + for key in map_keys(&blocks) { + let Some(block) = child_map(&blocks, &key) else { + continue; + }; + if map_string_opt(&block, "flow_id").as_deref() != Some(ROOT_BODY_FLOW_ID) { + continue; + } + // Only object blocks (image/equation/table) anchor to a U+FFFC placeholder; + // paragraph blocks are pruned by the paragraph-metadata repair path. + match map_string_opt(&block, "kind").as_deref() { + Some("paragraph") | None => continue, + Some(_) => {}, + } + let live = map_binary_opt(&block, "anchor_cursor") + .and_then(|bytes| Cursor::decode(&bytes).ok()) + .and_then(|cursor| cursor.id) + .and_then(|id| anchor_pos.get(&id).copied()) + .is_some_and(|pos| placeholder_positions.binary_search(&pos).is_ok()); + if !live { + blocks.delete(&key)?; + pruned += 1; + } + } + Ok(pruned) +} + +/// §fidelity: boolean mirror of the test helper `assert_semantic_projection_eq` +/// — whether two projections are +/// semantically equal across identity, sections, frontier, per-paragraph +/// style/runs/text, and object (non-paragraph) blocks. Paragraph `Block`s are +/// covered by the per-paragraph comparison, so only object blocks are compared +/// structurally. Used to detect incremental-vs-full projection divergence without +/// panicking; assets and theme are intentionally excluded (mirrors the helper). +fn projections_semantically_equal(left: &DocumentProjection, right: &DocumentProjection) -> bool { + if left.ids != right.ids || left.sections != right.sections || left.frontier != right.frontier { + return false; + } + if left.paragraphs.len() != right.paragraphs.len() { + return false; + } + for paragraph_ix in 0..left.paragraphs.len() { + let left_paragraph = &left.paragraphs[paragraph_ix]; + let right_paragraph = &right.paragraphs[paragraph_ix]; + if left_paragraph.style != right_paragraph.style + || left_paragraph.runs != right_paragraph.runs + || flowstate_document::paragraph_text(left, paragraph_ix) != flowstate_document::paragraph_text(right, paragraph_ix) + { + return false; + } + } + if left.blocks.len() != right.blocks.len() { + return false; + } + left + .blocks + .iter() + .zip(right.blocks.iter()) + .all(|(left_block, right_block)| match (left_block, right_block) { + (Block::Paragraph(_), Block::Paragraph(_)) => true, + _ => left_block == right_block, + }) +} + +/// §divergence-diagnostic: name the FIRST concrete field where `left` (the +/// incremental projection) differs from `right` (the authoritative full rebuild), +/// so the `incremental-vs-full-divergence` event pinpoints the exact id/text +/// instead of only counts. Called only once a divergence is already known. +fn first_projection_divergence(left: &DocumentProjection, right: &DocumentProjection) -> String { + if left.ids.document_id != right.ids.document_id { + return format!("document_id {} != {}", left.ids.document_id, right.ids.document_id); + } + if left.ids.paragraph_ids.len() != right.ids.paragraph_ids.len() { + return format!("paragraph_ids len {} != {}", left.ids.paragraph_ids.len(), right.ids.paragraph_ids.len()); + } + for (ix, (l, r)) in left + .ids + .paragraph_ids + .iter() + .zip(right.ids.paragraph_ids.iter()) + .enumerate() + { + if l != r { + return format!( + "paragraph_ids[{ix}] incremental={} full={} text={:?}", + l.0, + r.0, + flowstate_document::paragraph_text(right, ix) + ); + } + } + if left.ids.block_ids.len() != right.ids.block_ids.len() { + return format!("block_ids len {} != {}", left.ids.block_ids.len(), right.ids.block_ids.len()); + } + for (ix, (l, r)) in left + .ids + .block_ids + .iter() + .zip(right.ids.block_ids.iter()) + .enumerate() + { + if l != r { + return format!("block_ids[{ix}] incremental={} full={}", l.0, r.0); + } + } + if left.sections != right.sections { + return "sections differ".to_string(); + } + for ix in 0..left.paragraphs.len().min(right.paragraphs.len()) { + if flowstate_document::paragraph_text(left, ix) != flowstate_document::paragraph_text(right, ix) { + return format!( + "paragraph[{ix}] text incremental={:?} full={:?}", + flowstate_document::paragraph_text(left, ix), + flowstate_document::paragraph_text(right, ix) + ); + } + if left.paragraphs[ix].style != right.paragraphs[ix].style || left.paragraphs[ix].runs != right.paragraphs[ix].runs { + return format!("paragraph[{ix}] style/runs differ"); + } + } + "blocks or other non-id content differ".to_string() +} + +/// §fidelity: classify a projection defect into a fidelity event class. Paragraph +/// / block / asset identity defects carry durable ids (Identity); object-anchor +/// and paragraph-style-mark defects are structural (Structure). +fn fidelity_class_for_defect(defect: &ProjectionDefect) -> FidelityClass { + match defect { + ProjectionDefect::MissingParagraphMetadata { .. } + | ProjectionDefect::MissingParagraphBlock { .. } + | ProjectionDefect::InvalidAssetId { .. } => FidelityClass::Identity, + ProjectionDefect::MissingParagraphStyleMark { .. } + | ProjectionDefect::UnresolvedObjectAnchor { .. } + | ProjectionDefect::CollidingObjectAnchors { .. } + | ProjectionDefect::OrphanObjectPlaceholder { .. } + | ProjectionDefect::TableTopology { .. } => FidelityClass::Structure, + } +} + +#[derive(Clone, Debug)] +struct SubscriptionEventSummary { + origin: String, + triggered_by: String, + // §23: runtime epoch read when the event was emitted. `summarize_subscription_event` + // leaves this at 0 (it stays a pure function of the diff); the permanent root + // callback stamps the live epoch before buffering. + epoch: u64, + // §23: doc state frontier captured at emit time (`doc.state_frontiers().encode()`). + // Left empty by `summarize_subscription_event` and stamped by the root callback, + // which holds a reference clone of the document. + frontier: Vec, + changes: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TextChangeKind { + /// Style-only retain (mark/unmark over existing text). + StyledRetain, + Insert, + Delete, +} + +#[derive(Clone, Debug)] +enum SubscriptionChange { + Text { + target: String, + kind: TextChangeKind, + unicode_start: usize, + unicode_len: usize, + deleted_len: usize, + inserted_structure: bool, + }, + Map { + target: String, + keys: Vec, + }, + List { + target: String, + }, + Unknown { + target: String, + }, +} + +pub(crate) fn object_replacement_patch(projection: &DocumentProjection, block_ix: usize, block: InputBlock) -> Option> { + Some(vec![ProjectionPatch::ReplaceObjectBlock { + block_id: *projection.ids.block_ids.get(block_ix)?, + row_hint: block_ix, + block: ProjectionStructuralBlock { + block_id: *projection.ids.block_ids.get(block_ix)?, + paragraph_id: None, + block, + }, + }]) +} + +pub(crate) fn projection_text_delta( + prefix_retain: usize, + delete_len: usize, + insert_len: usize, + trailing_retain: usize, +) -> Vec { + let mut delta = Vec::new(); + if prefix_retain > 0 { + delta.push(flowstate_document::ProjectionTextDelta::Retain(prefix_retain)); + } + if delete_len > 0 { + delta.push(flowstate_document::ProjectionTextDelta::Delete(delete_len)); + } + if insert_len > 0 { + delta.push(flowstate_document::ProjectionTextDelta::Insert(insert_len)); + } + if trailing_retain > 0 { + delta.push(flowstate_document::ProjectionTextDelta::Retain(trailing_retain)); + } + delta +} + +pub(crate) fn insert_projection_object_block( + doc: &LoroDoc, + projection: &DocumentProjection, + block_id: flowstate_document::BlockId, + block_ix: usize, + input: &InputBlock, +) -> Result { + if matches!(input, InputBlock::Paragraph(_)) { + tracing::warn!( + block_ix, + ?block_id, + "skipping InsertBlock for paragraph payload; paragraph edits must use text/paragraph semantic commands" + ); + return Ok(false); + } + + let body = body_text(doc); + if object_loro_block_by_projected_id(doc, &body, block_id).is_some() { + tracing::warn!(block_ix, ?block_id, "skipping InsertBlock because the Loro object block already exists"); + return Ok(false); + } + let unicode_index = projection_block_lead_pos_in_loro(doc, projection, &body, block_ix); + insert_input_object_block(doc, unicode_index, block_id, input)?; + Ok(true) +} + +pub(crate) fn insert_input_object_block( + doc: &LoroDoc, + unicode_index: usize, + block_id: flowstate_document::BlockId, + input: &InputBlock, +) -> Result<()> { + match input { + InputBlock::Image(image) => insert_image_block_with_id(doc, unicode_index, block_id, image), + InputBlock::Equation(equation) => insert_equation_block_with_id(doc, unicode_index, block_id, equation), + InputBlock::Table(table) => insert_table_block_with_id(doc, unicode_index, block_id, table), + InputBlock::Paragraph(_) => Ok(()), + } +} + +/// §act-three B.1: container-only sibling of [`insert_input_object_block`] for +/// the recorded-inverse undo restore. The U+FFFC placeholder is already back +/// in the body (it rode the verbatim bulk-text restore), so only the block +/// registry record + nested containers are recreated — keyed by the ORIGINAL +/// block id, which reproduces the exact durable key the delete pruned. +/// Tables are gated out of the fast path (their durable row/column/cell ids +/// cannot be re-minted losslessly here). +pub(crate) fn restore_input_object_block_containers( + doc: &LoroDoc, + unicode_index: usize, + block_id: flowstate_document::BlockId, + input: &InputBlock, +) -> Result<()> { + let body = body_text(doc); + match input { + InputBlock::Image(image) => { + let block = ensure_block_with_id( + doc, + &object_block_key("image", block_id), + "image", + ROOT_BODY_FLOW_ID, + &body, + unicode_index, + )?; + replace_image_block_from_input(doc, &block, image) + }, + InputBlock::Equation(equation) => { + let block = ensure_block_with_id( + doc, + &object_block_key("equation", block_id), + "equation", + ROOT_BODY_FLOW_ID, + &body, + unicode_index, + )?; + replace_equation_block_from_input(doc, &block, equation) + }, + InputBlock::Table(_) => anyhow::bail!("tables are excluded from the recorded-inverse fast path"), + InputBlock::Paragraph(_) => Ok(()), + } +} + +pub(crate) fn replace_projection_object_block( + doc: &LoroDoc, + projection: &DocumentProjection, + block_id: Option, + block_ix: usize, + after: &InputBlock, +) -> Result { + if matches!(after, InputBlock::Paragraph(_)) { + tracing::warn!( + block_ix, + "skipping ReplaceBlock for paragraph payload; paragraph edits must use text/paragraph semantic commands" + ); + return Ok(false); + } + if block_id.is_none() && projection.blocks.get(block_ix).is_none() { + tracing::warn!(block_ix, "skipping ReplaceBlock because the projection block index is out of range"); + return Ok(false); + } + + let body = body_text(doc); + let block = block_id + .and_then(|block_id| object_loro_block_by_projected_id(doc, &body, block_id).map(|(_, block, _)| block)) + .or_else(|| { + projection + .ids + .block_ids + .get(block_ix) + .and_then(|block_id| object_loro_block_by_projected_id(doc, &body, *block_id).map(|(_, block, _)| block)) + }) + .or_else(|| { + let anchor_pos = object_unicode_pos_for_projection_block(&body, block_ix)?; + object_loro_block_at_unicode_pos(doc, &body, anchor_pos) + }); + let Some(block) = block else { + tracing::warn!(block_ix, "skipping ReplaceBlock because no Loro object block maps to the projected block"); + return Ok(false); + }; + + match after { + InputBlock::Image(image) => replace_image_block_from_input(doc, &block, image)?, + InputBlock::Equation(equation) => replace_equation_block_from_input(doc, &block, equation)?, + InputBlock::Table(table) => { + tracing::warn!( + block_ix, + "applying coarse structured table ReplaceBlock; editor should emit finer table operations later" + ); + replace_table_block_from_input(doc, &block, table)?; + }, + InputBlock::Paragraph(_) => unreachable!("paragraph payload was handled above"), + } + Ok(true) +} + +pub(crate) fn replace_projection_equation_source_range( + doc: &LoroDoc, + equation_block_id: flowstate_document::BlockId, + range: &std::ops::Range, + replacement: &str, +) -> Result { + let body = body_text(doc); + let Some((_, block, _)) = object_loro_block_by_projected_id(doc, &body, equation_block_id) else { + tracing::warn!( + ?equation_block_id, + ?range, + "skipping equation source edit because no Loro equation maps to the projected block id" + ); + return Ok(false); + }; + if map_string_opt(&block, "kind").as_deref() != Some("equation") { + tracing::warn!( + ?equation_block_id, + ?range, + "skipping equation source edit because the projected block is not an equation" + ); + return Ok(false); + } + let source_flow_id = map_string_opt(&block, "source_flow_id").unwrap_or_else(|| nested_flow_id("equation_source")); + block.insert("source_flow_id", source_flow_id.as_str())?; + let source_flow = ensure_flow(doc, &source_flow_id, "equation_source")?; + // §28: resolve the source flow's text via its stored `text_container_id`. + let source_text = flow_text(doc, &source_flow)?; + let before = source_text.to_string(); + let Some(start) = byte_index_to_unicode_index(&before, range.start) else { + tracing::warn!( + ?equation_block_id, + ?range, + "skipping equation source edit because the start byte is not a source boundary" + ); + return Ok(false); + }; + let Some(end) = byte_index_to_unicode_index(&before, range.end) else { + tracing::warn!( + ?equation_block_id, + ?range, + "skipping equation source edit because the end byte is not a source boundary" + ); + return Ok(false); + }; + if end < start { + tracing::warn!(?equation_block_id, ?range, "skipping equation source edit because the range is inverted"); + return Ok(false); + } + if end > start { + source_text.delete(start, end - start)?; + } + if !replacement.is_empty() { + source_text.insert(start, replacement)?; + } + Ok(true) +} + +pub(crate) fn replace_projection_image_alt_text(doc: &LoroDoc, image_block_id: flowstate_document::BlockId, text: &str) -> Result { + let body = body_text(doc); + let Some((_, block, _)) = object_loro_block_by_projected_id(doc, &body, image_block_id) else { + tracing::warn!( + ?image_block_id, + "skipping image alt text edit because no Loro image maps to the projected block id" + ); + return Ok(false); + }; + if map_string_opt(&block, "kind").as_deref() != Some("image") { + tracing::warn!( + ?image_block_id, + "skipping image alt text edit because the projected block is not an image" + ); + return Ok(false); + } + let alt_flow_id = map_string_opt(&block, "alt_text_flow_id").unwrap_or_else(|| nested_flow_id("image_alt")); + block.insert("alt_text_flow_id", alt_flow_id.as_str())?; + let alt_flow = ensure_flow(doc, &alt_flow_id, "alt_text")?; + // §28: resolve the alt-text flow's text via its stored `text_container_id`. + replace_text_incrementally(&flow_text(doc, &alt_flow)?, text)?; + Ok(true) +} + +pub(crate) fn replace_projection_image_caption( + doc: &LoroDoc, + image_block_id: flowstate_document::BlockId, + caption: Option<&InputParagraph>, +) -> Result { + let body = body_text(doc); + let Some((_, block, _)) = object_loro_block_by_projected_id(doc, &body, image_block_id) else { + tracing::warn!( + ?image_block_id, + "skipping image caption edit because no Loro image maps to the projected block id" + ); + return Ok(false); + }; + if map_string_opt(&block, "kind").as_deref() != Some("image") { + return Ok(false); + } + if let Some(caption) = caption { + let caption_flow_id = map_string_opt(&block, "caption_flow_id").unwrap_or_else(|| nested_flow_id("image_caption")); + block.insert("caption_flow_id", caption_flow_id.as_str())?; + let caption_flow = ensure_flow(doc, &caption_flow_id, "caption")?; + let text = caption_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + let desired = format!( + "{SENTINEL_NEWLINE}{}", + caption + .runs + .iter() + .map(|run| run.text.as_str()) + .collect::() + ); + replace_text_incrementally(&text, &desired)?; + let len = text.len_unicode(); + for key in [ + MARK_PARAGRAPH_STYLE, + MARK_RUN_SEMANTIC_STYLE, + MARK_HIGHLIGHT_STYLE, + MARK_DIRECT_UNDERLINE, + MARK_STRIKETHROUGH, + ] { + text.unmark(0..len, key)?; + } + text.mark(0..1, MARK_PARAGRAPH_STYLE, paragraph_style_value(caption.style))?; + let mut cursor = 1usize; + for run in &caption.runs { + let run_len = run.text.chars().count(); + if run_len > 0 { + mark_run_styles(&text, cursor..cursor + run_len, run.styles)?; + } + cursor += run_len; + } + } else { + block.delete("caption_flow_id")?; + } + Ok(true) +} + +pub(crate) fn set_projection_image_layout( + doc: &LoroDoc, + image_block_id: flowstate_document::BlockId, + sizing: &InputImageSizing, + alignment: InputBlockAlignment, +) -> Result { + let body = body_text(doc); + let Some((_, block, _)) = object_loro_block_by_projected_id(doc, &body, image_block_id) else { + tracing::warn!( + ?image_block_id, + "skipping image layout edit because no Loro image maps to the projected block id" + ); + return Ok(false); + }; + if map_string_opt(&block, "kind").as_deref() != Some("image") { + tracing::warn!(?image_block_id, "skipping image layout edit because the projected block is not an image"); + return Ok(false); + } + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("alignment", alignment_name(alignment))?; + write_image_sizing_attrs(&attrs, sizing)?; + Ok(true) +} + +fn byte_index_to_unicode_index(value: &str, byte: usize) -> Option { + (byte <= value.len() && value.is_char_boundary(byte)).then(|| value[..byte].chars().count()) +} + +/// §P2b: the deterministic empty cell for the `(row_id, column_id)` coordinate. +/// Its [`CellId`] is derived from the coordinate so a peer that fills a +/// grid-gap cell addresses the identical Loro container (LWW merge, never a +/// duplicate). Used as the fallback when a structural command's carried cell +/// list is shorter than the grid. +pub(crate) fn empty_input_table_cell(row_id: RowId, column_id: ColumnId) -> InputTableCell { + InputTableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + })], + row_span: 1, + col_span: 1, + } +} + +fn projection_table_map_by_block_id(doc: &LoroDoc, table_block_id: flowstate_document::BlockId) -> Option { + let body = body_text(doc); + let (_, block, _) = object_loro_block_by_projected_id(doc, &body, table_block_id)?; + if map_string_opt(&block, "kind").as_deref() != Some("table") { + return None; + } + child_map(&block, "table") +} + +pub(crate) fn delete_projection_object_block(doc: &LoroDoc, block_id: flowstate_document::BlockId) -> Result { + let body = body_text(doc); + let Some((key, _, anchor_pos)) = object_loro_block_by_projected_id(doc, &body, block_id) else { + tracing::warn!( + ?block_id, + "skipping DeleteBlock because no Loro object block maps to the projected block id" + ); + return Ok(false); + }; + if body.to_string().chars().nth(anchor_pos) != Some(OBJECT_REPLACEMENT) { + tracing::warn!( + ?block_id, + anchor_pos, + "skipping DeleteBlock because the Loro object anchor is no longer live" + ); + return Ok(false); + } + body + .delete(anchor_pos, 1) + .context("deleting object placeholder from body flow")?; + doc + .get_map(ROOT) + .ensure_mergeable_map(BLOCKS_BY_ID)? + .delete(&key) + .context("deleting object block metadata")?; + Ok(true) +} + +pub(crate) fn move_projection_object_block( + doc: &LoroDoc, + projection: &DocumentProjection, + block_id: flowstate_document::BlockId, + new_block_ix: usize, +) -> Result { + let body = body_text(doc); + let Some((_, block, anchor_pos)) = object_loro_block_by_projected_id(doc, &body, block_id) else { + tracing::warn!( + ?block_id, + new_block_ix, + "skipping MoveBlock because no Loro object block maps to the projected block id" + ); + return Ok(false); + }; + if body.to_string().chars().nth(anchor_pos) != Some(OBJECT_REPLACEMENT) { + tracing::warn!( + ?block_id, + anchor_pos, + "skipping MoveBlock because the Loro object anchor is no longer live" + ); + return Ok(false); + } + // `new_block_ix` is the target index in the POST-removal projection block list (the + // incremental replay removes the source, then inserts at `new_block_ix`). Map it back + // to the pre-removal block whose lead position it lands before, skipping the source's + // own slot, then resolve that lead position on the post-delete body via durable cursors + // (which survive the char deletion). A target at/after the tail appends. + let source_ix = projection + .ids + .block_ids + .iter() + .position(|id| *id == block_id); + body + .delete(anchor_pos, 1) + .context("deleting object placeholder before move")?; + let remaining = projection.blocks.len().saturating_sub(1); + let insert_pos = if new_block_ix >= remaining { + body.len_unicode() + } else { + let original_ix = match source_ix { + Some(source_ix) if new_block_ix >= source_ix => new_block_ix + 1, + _ => new_block_ix, + }; + projection_block_lead_pos_in_loro(doc, projection, &body, original_ix) + }; + body + .insert(insert_pos, &OBJECT_REPLACEMENT.to_string()) + .context("reinserting object placeholder after move")?; + if let Some(cursor) = body.get_cursor(insert_pos, Side::Left) { + block.insert("anchor_cursor", cursor.encode())?; + } + Ok(true) +} + +/// Body-unicode position at which projection block `block_ix` begins — inserting an +/// `OBJECT_REPLACEMENT` char here makes it the new block `block_ix` (append when +/// `block_ix >= blocks.len()`). Coalescing-aware BY CONSTRUCTION: it resolves the lead +/// position from the projection's own (already-coalesced) block list plus each block's +/// durable cursor — an object's `anchor_cursor`, or a paragraph's boundary `\n` — so it +/// never miscounts the record-less phantom empties a raw body walk stumbled over (the +/// off-by-N that diverged the incremental replay from the canonical Loro rebuild). Insert +/// before a paragraph lands on its boundary `\n` (clamped past the sentinel), which +/// attaches the object to the previous block's tail exactly as `push_flow_blocks` +/// re-segments it. +fn projection_block_lead_pos_in_loro(doc: &LoroDoc, projection: &DocumentProjection, body: &LoroText, block_ix: usize) -> usize { + let Some(block) = projection.blocks.get(block_ix) else { + return body.len_unicode(); + }; + match block { + flowstate_document::Block::Paragraph(_) => { + let paragraph_ix = projection + .blocks + .iter() + .take(block_ix) + .filter(|block| matches!(block, flowstate_document::Block::Paragraph(_))) + .count(); + // The paragraph's leading `\n` (sentinel-clamped: never insert before body pos 1). + paragraph_boundary_loro_unicode_index(doc, projection, paragraph_ix).max(1) + }, + _ => projection + .ids + .block_ids + .get(block_ix) + .and_then(|block_id| object_loro_block_by_projected_id(doc, body, *block_id)) + .map_or_else(|| body.len_unicode(), |(_, _, anchor_pos)| anchor_pos), + } +} + +fn object_unicode_pos_for_projection_block(body: &LoroText, target_block_ix: usize) -> Option { + let mut block_ix = 0_usize; + let mut current_paragraph_has_text = false; + let mut seen_sentinel = false; + + for (unicode_pos, ch) in body.to_string().chars().enumerate() { + match ch { + '\n' => { + if seen_sentinel { + if block_ix == target_block_ix { + return None; + } + block_ix += 1; + } else { + seen_sentinel = true; + } + current_paragraph_has_text = false; + }, + OBJECT_REPLACEMENT => { + if current_paragraph_has_text { + if block_ix == target_block_ix { + return None; + } + block_ix += 1; + current_paragraph_has_text = false; + } + if block_ix == target_block_ix { + return Some(unicode_pos); + } + block_ix += 1; + }, + _ => current_paragraph_has_text = true, + } + } + None +} + +/// Live start (unicode) of the paragraph identified by `paragraph_id` in the actual +/// Loro body flow, resolved from its durable boundary cursor — the paragraph's text +/// begins just past its boundary newline. Coalescing-agnostic: unlike the +/// projection-derived body-unicode index, this reflects boundary newlines that are +/// physically present in the body even when the projection has coalesced that +/// paragraph out of view (an object-adjacent empty paragraph). Returns `None` when +/// the durable record or its cursor cannot be resolved, so callers fall back to the +/// projection-space start. +#[hotpath::measure] +pub(crate) fn paragraph_body_start_in_loro(doc: &LoroDoc, paragraph_id: ParagraphId) -> Option { + let root = doc.get_map(ROOT); + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID).ok()?; + // Fast path: canonical record keys are directly constructible from the id + // (`paragraph.{u128}`, plus the seeded first-paragraph key), so the common + // case is an O(1) map probe instead of an O(records) key scan. + let canonical_key = format!("paragraph.{}", paragraph_id.0); + for key in [canonical_key.as_str(), ROOT_FIRST_PARAGRAPH_ID] { + if loro_id_u128(key) == paragraph_id.0 + && let Some(paragraph) = child_map(¶graphs, key) + { + return paragraph_record_body_start(doc, ¶graph); + } + } + for key in map_keys(¶graphs) { + if loro_id_u128(&key) != paragraph_id.0 { + continue; + } + let paragraph = child_map(¶graphs, &key)?; + return paragraph_record_body_start(doc, ¶graph); + } + None +} + +fn paragraph_record_body_start(doc: &LoroDoc, paragraph: &LoroMap) -> Option { + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + let debug = *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()); + for field in ["boundary_cursor", "start_cursor"] { + let Some(bytes) = map_binary_opt(paragraph, field) else { + if debug { + eprintln!("derive[body-start]: field {field} missing on record"); + } + continue; + }; + let cursor = match Cursor::decode(&bytes) { + Ok(cursor) => cursor, + Err(error) => { + if debug { + eprintln!("derive[body-start]: field {field} decode failed: {error:?}"); + } + continue; + }, + }; + match doc.get_cursor_pos(&cursor) { + Ok(resolved) => return Some(resolved.current.pos.saturating_add(1)), + Err(error) => { + if debug { + eprintln!("derive[body-start]: field {field} get_cursor_pos failed: {error:?}"); + } + }, + } + } + None +} + +/// Batched sibling of [`paragraph_body_start_in_loro`]: resolve MANY +/// paragraphs' body starts through ONE `query_text_id_positions` pass. +/// Per-paragraph `get_cursor_pos` is a linear chunk scan per call (the +/// batch-resolver lesson), so a caller resolving thousands of paragraph +/// starts — a replace-all storm — was quadratic in document size. A +/// paragraph whose record or cursor cannot be batch-resolved falls back to +/// the exact single-paragraph path (rare: degraded cursors, seeded docs). +pub(crate) fn paragraph_body_starts_in_loro(doc: &LoroDoc, paragraph_ids: &[ParagraphId]) -> Vec> { + let root = doc.get_map(ROOT); + let Ok(paragraphs) = root.ensure_mergeable_map(PARAGRAPHS_BY_ID) else { + return vec![None; paragraph_ids.len()]; + }; + let body = body_text(doc); + let container = body.id(); + let mut cursor_ids: Vec> = Vec::with_capacity(paragraph_ids.len()); + for paragraph_id in paragraph_ids { + let canonical_key = format!("paragraph.{}", paragraph_id.0); + let record = [canonical_key.as_str(), ROOT_FIRST_PARAGRAPH_ID] + .into_iter() + .find(|key| loro_id_u128(key) == paragraph_id.0) + .and_then(|key| child_map(¶graphs, key)); + let id = record.and_then(|record| { + ["boundary_cursor", "start_cursor"] + .into_iter() + .find_map(|field| { + let bytes = map_binary_opt(&record, field)?; + let cursor = Cursor::decode(&bytes).ok()?; + (cursor.container == container) + .then_some(cursor.id) + .flatten() + }) + }); + cursor_ids.push(id); + } + let query: Vec = cursor_ids.iter().copied().flatten().collect(); + let mut resolved = if query.is_empty() { + Vec::new() + } else { + doc.inner().query_text_id_positions(&container, &query) + } + .into_iter(); + let batched: Vec> = paragraph_ids + .iter() + .zip(&cursor_ids) + .map(|(_, cursor_id)| { + cursor_id + .and_then(|_| resolved.next().flatten()) + .map(|pos| pos.saturating_add(1)) + }) + .collect(); + // §hang fix (field, 2026-07-07 peer 2): the per-id fallback runs + // `get_cursor_pos`, and a DEAD cursor there triggers a full history-trace + // (commit barrier + DiffCalculator checkout) PER ID. After an undo restores + // a whole-document deletion, EVERY restored record's cursor is dead — the + // receiving peer ran thousands of history traces and hung until ctrl-C. + // The fallback is for the rare degraded-cursor/seeded-doc case; a MASS-dead + // state must return `None`s instead, sending callers to the O(doc) full + // rebuild + canonical re-anchor repair (~1s) rather than a multi-minute + // trace storm. + const DEAD_CURSOR_FALLBACK_CAP: usize = 8; + let misses = batched.iter().filter(|start| start.is_none()).count(); + if misses > DEAD_CURSOR_FALLBACK_CAP { + tracing::warn!( + misses, + total = paragraph_ids.len(), + "mass dead-cursor state in batched paragraph-start resolution; skipping per-id history-trace fallbacks" + ); + return batched; + } + batched + .into_iter() + .zip(paragraph_ids) + .map(|(start, paragraph_id)| start.or_else(|| paragraph_body_start_in_loro(doc, *paragraph_id))) + .collect() +} + +/// Loro-space boundary position (the paragraph's leading `\n`) for projection +/// paragraph `paragraph_ix`, resolved from its durable cursor so coalesced empties +/// don't shift it. Loro-space counterpart of [`paragraph_boundary_unicode_index`]; +/// use at Loro body-mutation sites (join delete, style mark). The paragraph's text +/// begins one unicode past its boundary. Falls back to the projection-space boundary +/// when the durable record can't be resolved (e.g. the boundary-0 sentinel). +pub(crate) fn paragraph_boundary_loro_unicode_index(doc: &LoroDoc, projection: &DocumentProjection, paragraph_ix: usize) -> usize { + if paragraph_ix == 0 { + return 0; + } + let record = projection + .ids + .paragraph_ids + .get(paragraph_ix) + .and_then(|paragraph_id| paragraph_body_start_in_loro(doc, *paragraph_id)) + .map(|start| start.saturating_sub(1)); + paragraph_boundary_from_record(doc, projection, paragraph_ix, record) +} + +/// Batched sibling of [`paragraph_boundary_loro_unicode_index`]: resolves all +/// requested paragraphs' durable starts through ONE `query_text_id_positions` +/// pass (per-paragraph `get_cursor_pos` is a linear chunk scan — the +/// batch-resolver lesson), then applies the same rot-check + fallback law per +/// paragraph. Order-preserving with the input. +pub(crate) fn paragraph_boundaries_loro_unicode_indices(doc: &LoroDoc, projection: &DocumentProjection, paragraph_ixs: &[usize]) -> Vec { + let query_ids: Vec = paragraph_ixs + .iter() + .filter(|&&ix| ix != 0) + .filter_map(|&ix| projection.ids.paragraph_ids.get(ix).copied()) + .collect(); + let mut starts = paragraph_body_starts_in_loro(doc, &query_ids).into_iter(); + paragraph_ixs + .iter() + .map(|&ix| { + if ix == 0 { + return 0; + } + let record = if projection.ids.paragraph_ids.get(ix).is_some() { + starts.next().flatten().map(|start| start.saturating_sub(1)) + } else { + None + }; + paragraph_boundary_from_record(doc, projection, ix, record) + }) + .collect() +} + +/// The shared boundary law: trust the durable-record boundary only when the +/// body text right after it matches the paragraph's projected prefix. +/// +/// The durable cursor survives coalesced empties (projection-row math does +/// not), but it can ROT under undo churn: the record's anchor re-resolves +/// onto a NEIGHBORING newline after its original op ids die, so a style +/// mark or join lands on the wrong paragraph (object-fuzz divergence). +/// Content check: the boundary is only trusted when the text right after it +/// matches the paragraph's projected prefix; otherwise fall back to +/// projection-space math (in-sync with the doc inside the gate). +fn paragraph_boundary_from_record(doc: &LoroDoc, projection: &DocumentProjection, paragraph_ix: usize, record: Option) -> usize { + let body = body_text(doc); + match record { + Some(boundary) if boundary_matches_paragraph(&body, projection, paragraph_ix, boundary) => boundary, + _ => { + let fallback = paragraph_boundary_unicode_index(projection, paragraph_ix); + if boundary_matches_paragraph(&body, projection, paragraph_ix, fallback) { + fallback + } else { + record.unwrap_or(fallback) + } + }, + } +} + +/// The shared boundary trust check: `boundary` is this paragraph's leading +/// newline only when the live body text right after it matches the +/// paragraph's projected prefix. +fn boundary_matches_paragraph(body: &loro::LoroText, projection: &DocumentProjection, paragraph_ix: usize, boundary: usize) -> bool { + if body.char_at(boundary) != Ok('\n') { + return false; + } + let text = flowstate_document::paragraph_text(projection, paragraph_ix); + if text.is_empty() { + // An empty paragraph's boundary is followed by a terminator (next + // boundary, an object placeholder, or end of body) — anything else is a + // different paragraph's newline. Without this, EVERY newline vacuously + // matched an empty paragraph and a rotten cursor sailed through. + return match body.char_at(boundary + 1) { + Ok(next) => next == '\n' || next == flowstate_document::OBJECT_REPLACEMENT, + Err(_) => true, + }; + } + text + .chars() + .take(4) + .enumerate() + .all(|(offset, expected)| body.char_at(boundary + 1 + offset) == Ok(expected)) +} + +/// Whether the MAINTAINED body-unicode `start` for `paragraph_ix` matches the +/// live body (its leading boundary newline plus the paragraph's projected +/// prefix) — the cheap gate that lets keystroke anchor resolution skip the +/// per-call durable-cursor chunk scan. +fn body_start_matches_paragraph(doc: &LoroDoc, projection: &DocumentProjection, paragraph_ix: usize, start: usize) -> bool { + let Some(boundary) = start.checked_sub(1) else { + return false; + }; + boundary_matches_paragraph(&body_text(doc), projection, paragraph_ix, boundary) +} + +fn object_loro_block_by_projected_id(doc: &LoroDoc, body: &LoroText, block_id: flowstate_document::BlockId) -> Option<(String, LoroMap, usize)> { + let root = doc.get_map(ROOT); + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID).ok()?; + let body_snapshot = body.to_string(); + for key in map_keys(&blocks) { + if loro_id_u128(&key) != block_id.0 { + continue; + } + let block = child_map(&blocks, &key)?; + if map_string_opt(&block, "kind").as_deref() == Some("paragraph") { + return None; + } + let anchor_pos = live_object_cursor_pos(doc, &body_snapshot, &block, "anchor_cursor")?; + return Some((key, block, anchor_pos)); + } + for key in map_keys(&blocks) { + let block = child_map(&blocks, &key)?; + if map_string_opt(&block, "kind").as_deref() == Some("paragraph") { + continue; + } + if map_string_opt(&block, "id").is_some_and(|id| loro_id_u128(&id) == block_id.0) { + let anchor_pos = live_object_cursor_pos(doc, &body_snapshot, &block, "anchor_cursor")?; + return Some((key, block, anchor_pos)); + } + } + None +} + +fn object_loro_block_at_unicode_pos(doc: &LoroDoc, body: &LoroText, unicode_pos: usize) -> Option { + let root = doc.get_map(ROOT); + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID).ok()?; + let body_snapshot = body.to_string(); + for key in map_keys(&blocks) { + let block = child_map(&blocks, &key)?; + if map_string_opt(&block, "kind").as_deref() == Some("paragraph") { + continue; + } + if live_object_cursor_pos(doc, &body_snapshot, &block, "anchor_cursor") == Some(unicode_pos) { + return Some(block); + } + } + None +} + +fn loro_id_u128(id: &str) -> u128 { + if let Some(value) = id + .rsplit('.') + .next() + .and_then(|suffix| suffix.parse::().ok()) + { + return value; + } + // MUST match `flowstate_document::loro_projection::loro_id_u128` exactly — + // the projector mints every projection id with this law, and every runtime + // lookup compares against those ids. A uuid-v5 variant here once silently + // disagreed on every NON-decimal key (`paragraph.anchor.op-…` repair keys, + // `paragraph.initial`), which made repaired record-less boundaries + // permanently unresolvable to the runtime: the §6-R region resolver bailed + // to a full rebuild on every later structural import whose region touched a + // healed paragraph (the REMOTE-STRUCT-BARE compounding heal cost), and the + // same paragraphs were re-reported as defects and re-repaired every round. + let hash = blake3::hash(id.as_bytes()); + let mut bytes = [0_u8; 16]; + bytes.copy_from_slice(&hash.as_bytes()[..16]); + u128::from_le_bytes(bytes) +} + +fn replace_image_block_from_input(doc: &LoroDoc, block: &LoroMap, image: &flowstate_document::InputImageBlock) -> Result<()> { + block.insert("kind", "image")?; + block.insert("asset_id", image.asset_id.0.to_string())?; + copy_asset_metadata_to_image_block(doc, block, image.asset_id.0)?; + // §A11.9: linked images carry their external URL; written only when a + // non-empty URL exists, deleted otherwise (a replace may swap a linked image + // for an embedded one) — mirrors `loro_import::import_image_block` and the + // projector's `external_url` readback. The presence guard avoids minting a + // tombstone op per replace on the common embedded path. + match image.external_url.as_deref().filter(|url| !url.is_empty()) { + Some(url) => block.insert("external_url", url)?, + None => { + if block.get("external_url").is_some() { + block.delete("external_url")?; + } + }, + } + + let alt_flow_id = map_string_opt(block, "alt_text_flow_id").unwrap_or_else(|| nested_flow_id("image_alt")); + block.insert("alt_text_flow_id", alt_flow_id.as_str())?; + let alt_flow = ensure_flow(doc, &alt_flow_id, "alt_text")?; + replace_text(&alt_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, image.alt_text.as_ref())?; + + if let Some(caption) = &image.caption { + let caption_flow_id = map_string_opt(block, "caption_flow_id").unwrap_or_else(|| nested_flow_id("image_caption")); + block.insert("caption_flow_id", caption_flow_id.as_str())?; + let caption_flow = ensure_flow(doc, &caption_flow_id, "caption")?; + let caption_text = caption_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + replace_text(&caption_text, SENTINEL_NEWLINE)?; + append_input_paragraph_text_only(&caption_text, caption)?; + } else { + block.delete("caption_flow_id")?; + } + + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("alignment", alignment_name(image.alignment))?; + write_image_sizing_attrs(&attrs, &image.sizing)?; + Ok(()) +} + +fn copy_asset_metadata_to_image_block(doc: &LoroDoc, block: &LoroMap, asset_id: u128) -> Result<()> { + let root = doc.get_map(ROOT); + let Some(assets) = child_map(&root, flowstate_document::loro_schema::ASSETS_BY_ID) else { + return Ok(()); + }; + let Some(asset) = child_map(&assets, &asset_id.to_string()) else { + return Ok(()); + }; + for field in ["content_hash", "mime_type", "byte_length"] { + if let Some(ValueOrContainer::Value(value)) = asset.get(field) { + block.insert(field, value)?; + } + } + Ok(()) +} + +fn refresh_image_asset_metadata(doc: &LoroDoc) -> Result<()> { + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return Ok(()); + }; + for key in map_keys(&blocks) { + let Some(block) = child_map(&blocks, &key) else { + continue; + }; + if map_string_opt(&block, "kind").as_deref() != Some("image") { + continue; + } + let Some(asset_id) = map_string_opt(&block, "asset_id").and_then(|id| id.parse().ok()) else { + continue; + }; + copy_asset_metadata_to_image_block(doc, &block, asset_id)?; + } + Ok(()) +} + +fn replace_equation_block_from_input(doc: &LoroDoc, block: &LoroMap, equation: &flowstate_document::InputEquationBlock) -> Result<()> { + block.insert("kind", "equation")?; + let source_flow_id = map_string_opt(block, "source_flow_id").unwrap_or_else(|| nested_flow_id("equation_source")); + block.insert("source_flow_id", source_flow_id.as_str())?; + let source_flow = ensure_flow(doc, &source_flow_id, "equation_source")?; + replace_text(&source_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, &equation.source)?; + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("syntax", "latex")?; + attrs.insert("display", equation_display_name(equation.display))?; + Ok(()) +} + +fn replace_table_block_from_input(doc: &LoroDoc, block: &LoroMap, table: &InputTableBlock) -> Result<()> { + block.insert("kind", "table")?; + let table_map = block.ensure_mergeable_map("table")?; + // §P2b: the create-only writer reuses the carried durable ids and writes into + // the same durable-block-keyed container, so it no longer rekeys the table. + write_table_map_from_input(doc, &table_map, table) +} + +fn write_image_sizing_attrs(attrs: &LoroMap, sizing: &InputImageSizing) -> Result<()> { + attrs.delete("width_px")?; + attrs.delete("height_px")?; + match sizing { + InputImageSizing::Intrinsic => attrs.insert("sizing", "intrinsic")?, + InputImageSizing::FitWidth => attrs.insert("sizing", "fit_width")?, + InputImageSizing::Fixed { width_px, height_px } => { + attrs.insert("sizing", "fixed")?; + attrs.insert("width_px", i64::from(*width_px))?; + if let Some(height_px) = *height_px { + attrs.insert("height_px", i64::from(height_px))?; + } + }, + }; + Ok(()) +} + +/// §P2b create-only whole-table writer, mirroring `loro_import::import_table`. +/// +/// Every column / row / cell is addressed by its carried durable id via `ensure_*` +/// and its `*_loro_id` string, and an id is only pushed into an order list when it +/// is not already present. There is no positional `{prefix}.row.{ix}` scheme and +/// no clear + repopulate, so concurrent creation of the same id merges (Loro LWW) +/// instead of duplicating, and re-applying the same input rewrites in place +/// (which is what lets `replace_table_block_from_input` reuse existing ids). +fn write_table_map_from_input(doc: &LoroDoc, table_map: &LoroMap, table: &InputTableBlock) -> Result<()> { + table_map.insert("header_row", table.style.header_row)?; + let row_order = table_map.ensure_mergeable_movable_list("row_order")?; + let column_order = table_map.ensure_mergeable_movable_list("column_order")?; + let rows_by_id = table_map.ensure_mergeable_map("rows_by_id")?; + let columns_by_id = table_map.ensure_mergeable_map("columns_by_id")?; + let cells_by_id = table_map.ensure_mergeable_map("cells_by_id")?; + table_map.insert("columns_container_id", columns_by_id.id().to_string())?; + table_map.insert("cells_container_id", cells_by_id.id().to_string())?; + + let existing_columns = movable_list_strings(&column_order); + for column in &table.columns { + let column_id = column_loro_id(column.id); + if !existing_columns.iter().any(|id| id == &column_id) { + column_order.push(column_id.as_str())?; + } + let column_map = columns_by_id.ensure_mergeable_map(&column_id)?; + column_map.insert("id", column_id.as_str())?; + let _attrs = column_map.ensure_mergeable_map("attrs")?; + write_table_column_width(&column_map, &column.width)?; + } + + let existing_rows = movable_list_strings(&row_order); + for row in &table.rows { + let row_id = row_loro_id(row.id); + if !existing_rows.iter().any(|id| id == &row_id) { + row_order.push(row_id.as_str())?; + } + let row_map = rows_by_id.ensure_mergeable_map(&row_id)?; + row_map.insert("id", row_id.as_str())?; + let _attrs = row_map.ensure_mergeable_map("attrs")?; + for cell in &row.cells { + // §P2b law hardening: a cell's canonical identity IS its coordinate mix. + // Derive it here rather than trusting the payload's `cell.id` — a + // payload-minted random id would double-key the coordinate (writer by + // coordinate, reader by record) and split the cell across replicas. + let cell_id = cell_loro_id(CellId::from_coordinate(cell.row_id, cell.column_id)); + let cell_row_id = row_loro_id(cell.row_id); + let cell_column_id = column_loro_id(cell.column_id); + let cell_map = cells_by_id.ensure_mergeable_map(&cell_id)?; + write_table_cell_map_from_input(doc, &cell_map, &cell_id, &cell_row_id, &cell_column_id, cell, true)?; + } + } + Ok(()) +} + +fn write_table_cell_map_from_input( + doc: &LoroDoc, + cell_map: &LoroMap, + cell_id: &str, + row_id: &str, + column_id: &str, + cell: &InputTableCell, + // §P2b: when `false`, ensure the (empty) cell flow container but write no + // sentinel `\n` or block content. The topology-repair pass uses this so two + // peers concurrently materializing the SAME missing coordinate converge to an + // empty flow (one empty paragraph) instead of racing two `\n` inserts into the + // same deterministic flow (which would merge to `\n\n` = two empty paragraphs). + seed_flow: bool, +) -> Result<()> { + cell_map.insert("id", cell_id)?; + cell_map.insert("row_id", row_id)?; + cell_map.insert("column_id", column_id)?; + cell_map.insert("row_span", i64::from(cell.row_span))?; + cell_map.insert("column_span", i64::from(cell.col_span))?; + let _attrs = cell_map.ensure_mergeable_map("attrs")?; + let nested_table_ids = cell_map.ensure_mergeable_movable_list("nested_table_ids")?; + let nested_tables_by_id = cell_map.ensure_mergeable_map("nested_tables_by_id")?; + clear_movable_list(&nested_table_ids)?; + clear_map(&nested_tables_by_id)?; + let flow_id = format!("{cell_id}.flow"); + cell_map.insert("flow_id", flow_id.as_str())?; + let flow = ensure_flow(doc, &flow_id, "table_cell")?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + cell_map.insert("text_container_id", text.id().to_string())?; + if !seed_flow { + // Empty cell flow: the projector reads it back as a single empty paragraph, + // and doing no text insert keeps concurrent coordinate-cell repair idempotent. + return Ok(()); + } + replace_text(&text, SENTINEL_NEWLINE)?; + text.mark(0..1, MARK_PARAGRAPH_STYLE, 0_i64)?; + for (block_ix, cell_block) in cell.blocks.iter().enumerate() { + match cell_block { + InputTableCellBlock::Paragraph(paragraph) => append_input_paragraph_text_only(&text, paragraph)?, + InputTableCellBlock::Table(nested) => { + let pos = text.len_unicode(); + text.insert(pos, &OBJECT_REPLACEMENT.to_string())?; + let nested_table_id = format!("{cell_id}.nested_table.{block_ix}"); + nested_table_ids.push(nested_table_id.as_str())?; + let nested_map = nested_tables_by_id.ensure_mergeable_map(&nested_table_id)?; + nested_map.insert("id", nested_table_id.as_str())?; + nested_map.insert("kind", "table")?; + if let Some(cursor) = text.get_cursor(pos, Side::Left) { + nested_map.insert("anchor_cursor", cursor.encode())?; + } + nested_map.ensure_mergeable_map("attrs")?; + write_table_map_from_input(doc, &nested_map.ensure_mergeable_map("table")?, nested)?; + }, + } + } + Ok(()) +} + +fn update_table_cell_map_from_input( + doc: &LoroDoc, + cell_map: &LoroMap, + cell_id: &str, + row_id: &str, + column_id: &str, + cell: &InputTableCell, +) -> Result<()> { + if cell + .blocks + .iter() + .any(|block| matches!(block, InputTableCellBlock::Table(_))) + { + tracing::warn!(cell_id, "using full table-cell rebuild fallback for nested table structure"); + return write_table_cell_map_from_input(doc, cell_map, cell_id, row_id, column_id, cell, true); + } + cell_map.insert("id", cell_id)?; + cell_map.insert("row_id", row_id)?; + cell_map.insert("column_id", column_id)?; + cell_map.insert("row_span", i64::from(cell.row_span))?; + cell_map.insert("column_span", i64::from(cell.col_span))?; + let _attrs = cell_map.ensure_mergeable_map("attrs")?; + let flow_id = map_string_opt(cell_map, "flow_id").unwrap_or_else(|| format!("{cell_id}.flow")); + cell_map.insert("flow_id", flow_id.as_str())?; + let flow = ensure_flow(doc, &flow_id, "table_cell")?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + cell_map.insert("text_container_id", text.id().to_string())?; + + let paragraphs = cell + .blocks + .iter() + .filter_map(|block| match block { + InputTableCellBlock::Paragraph(paragraph) => Some(paragraph), + InputTableCellBlock::Table(_) => None, + }) + .collect::>(); + let desired = if paragraphs.is_empty() { + SENTINEL_NEWLINE.to_string() + } else { + let mut desired = String::from(SENTINEL_NEWLINE); + for (paragraph_ix, paragraph) in paragraphs.iter().enumerate() { + if paragraph_ix > 0 { + desired.push('\n'); + } + for run in ¶graph.runs { + desired.push_str(&run.text); + } + } + desired + }; + replace_text_incrementally(&text, &desired)?; + let len = text.len_unicode(); + for key in [ + MARK_PARAGRAPH_STYLE, + MARK_RUN_SEMANTIC_STYLE, + MARK_HIGHLIGHT_STYLE, + MARK_DIRECT_UNDERLINE, + MARK_STRIKETHROUGH, + ] { + text.unmark(0..len, key)?; + } + if paragraphs.is_empty() { + text.mark(0..1, MARK_PARAGRAPH_STYLE, paragraph_style_value(ParagraphStyle::Normal))?; + return Ok(()); + } + let mut cursor = 0usize; + for paragraph in paragraphs { + text.mark(cursor..cursor + 1, MARK_PARAGRAPH_STYLE, paragraph_style_value(paragraph.style))?; + cursor += 1; + for run in ¶graph.runs { + let run_len = run.text.chars().count(); + if run_len > 0 { + mark_run_styles(&text, cursor..cursor + run_len, run.styles)?; + } + cursor += run_len; + } + } + Ok(()) +} + +fn replace_text_incrementally(text: &LoroText, desired: &str) -> loro::LoroResult<()> { + let current = text.to_string(); + if current == desired { + return Ok(()); + } + let current_chars = current.chars().collect::>(); + let desired_chars = desired.chars().collect::>(); + let prefix = current_chars + .iter() + .zip(&desired_chars) + .take_while(|(left, right)| left == right) + .count(); + let suffix = current_chars + .iter() + .skip(prefix) + .rev() + .zip(desired_chars.iter().skip(prefix).rev()) + .take_while(|(left, right)| left == right) + .count(); + let delete_len = current_chars.len().saturating_sub(prefix + suffix); + if delete_len > 0 { + text.delete(prefix, delete_len)?; + } + let insert_end = desired_chars.len().saturating_sub(suffix); + if insert_end > prefix { + let insert = desired_chars[prefix..insert_end].iter().collect::(); + text.insert(prefix, &insert)?; + } + Ok(()) +} + +fn write_table_column_width(column: &LoroMap, width: &InputTableColumnWidth) -> Result<()> { + column.delete("width_px")?; + column.delete("fraction")?; + match width { + InputTableColumnWidth::Auto => column.insert("width_kind", "auto")?, + InputTableColumnWidth::FixedPx(px) => { + column.insert("width_kind", "fixed_px")?; + column.insert("width_px", i64::from(*px))?; + }, + InputTableColumnWidth::Fraction(fraction) => { + column.insert("width_kind", "fraction")?; + column.insert("fraction", i64::from(*fraction))?; + }, + }; + Ok(()) +} + +fn append_input_paragraph_text_only(text: &LoroText, paragraph: &InputParagraph) -> Result<()> { + let use_existing_sentinel = text.len_unicode() == 1 && text.to_string() == SENTINEL_NEWLINE; + let boundary_pos = if use_existing_sentinel { + 0 + } else { + let pos = text.len_unicode(); + text.insert(pos, "\n")?; + pos + }; + text.mark( + boundary_pos..boundary_pos + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(paragraph.style), + )?; + for run in ¶graph.runs { + if run.text.is_empty() { + continue; + } + let start = text.len_unicode(); + text.insert(start, &run.text)?; + let len = run.text.chars().count(); + mark_run_styles(text, start..start + len, run.styles)?; + } + Ok(()) +} + +fn clear_map(map: &LoroMap) -> loro::LoroResult<()> { + // §29: rebuild/fallback paths use Loro's native container clear instead of a + // manual per-key delete loop. Behavior is identical, just native. + map.clear() +} + +fn clear_movable_list(list: &LoroMovableList) -> loro::LoroResult<()> { + // §29: native clear instead of `delete(0, len)`; identical effect. + list.clear() +} + +fn projection_offset_to_body_unicode_index(projection: &DocumentProjection, offset: flowstate_document::DocumentOffset) -> usize { + ProjectionRuntimeIndex::from_projection(projection) + .body_unicode_for_offset(projection, offset) + .unwrap_or(1) +} + +fn clamp_projection_offset(projection: &DocumentProjection, offset: DocumentOffset) -> DocumentOffset { + let paragraph = offset + .paragraph + .min(projection.paragraphs.len().saturating_sub(1)); + let byte = projection + .paragraphs + .get(paragraph) + .map(flowstate_document::paragraph_text_len) + .unwrap_or_default() + .min(offset.byte); + DocumentOffset { paragraph, byte } +} + +fn paragraph_boundary_unicode_index(projection: &DocumentProjection, paragraph_ix: usize) -> usize { + if paragraph_ix == 0 { + return 0; + } + projection_offset_to_body_unicode_index( + projection, + flowstate_document::DocumentOffset { + paragraph: paragraph_ix, + byte: 0, + }, + ) - 1 +} + +pub(crate) fn join_projection_paragraphs( + doc: &LoroDoc, + projection: &DocumentProjection, + first: flowstate_document::ParagraphId, + second: flowstate_document::ParagraphId, +) -> Result { + let Some(first_ix) = projection + .ids + .paragraph_ids + .iter() + .position(|id| *id == first) + else { + tracing::warn!( + ?first, + ?second, + "skipping JoinParagraphs because the first paragraph id is absent from the supplied projection" + ); + return Ok(false); + }; + let Some(second_ix) = projection + .ids + .paragraph_ids + .iter() + .position(|id| *id == second) + else { + tracing::warn!( + ?first, + ?second, + "skipping JoinParagraphs because the second paragraph id is absent from the supplied projection" + ); + return Ok(false); + }; + if first_ix + 1 != second_ix { + tracing::warn!( + ?first, + ?second, + first_ix, + second_ix, + "skipping JoinParagraphs for non-adjacent paragraphs" + ); + return Ok(false); + } + if !projection_paragraph_blocks_are_adjacent(projection, first_ix, second_ix) { + tracing::warn!( + ?first, + ?second, + first_ix, + second_ix, + "skipping JoinParagraphs because an object block separates the paragraphs and projection offsets are not object-aware", + ); + return Ok(false); + } + + let Some(second_block_ix) = flowstate_document::block_ix_for_paragraph(projection, second_ix) else { + return Ok(false); + }; + let Some(second_block) = projection.ids.block_ids.get(second_block_ix).copied() else { + return Ok(false); + }; + + let boundary = paragraph_boundary_loro_unicode_index(doc, projection, second_ix); + let body = body_text(doc); + if body.char_at(boundary) != Ok('\n') { + tracing::warn!( + ?first, + ?second, + boundary, + "skipping JoinParagraphs because the computed Loro boundary is not a live paragraph newline", + ); + return Ok(false); + } + // Atomicity: only drop the second paragraph's durable metadata once we have + // committed to deleting its boundary. Deleting the record before the liveness + // check could orphan the boundary (record gone, '\n' still live) when the join + // then skips — a full reprojection fabricates a fresh id there and diverges from + // the incremental projection, which still holds the record's original id. + delete_projection_paragraph_metadata(doc, second, second_block)?; + body + .delete(boundary, 1) + .context("deleting joined paragraph boundary from Loro body flow")?; + // The joined-away paragraph's records were retired explicitly above; no + // record can go stale here, so the O(records) prune sweep is not run + // (§11 — it cost ~100ms per Backspace-join on a 6k-paragraph doc). + Ok(true) +} + +pub(crate) fn delete_projection_paragraph_metadata(doc: &LoroDoc, paragraph_id: ParagraphId, block_id: BlockId) -> loro::LoroResult<()> { + let root = doc.get_map(ROOT); + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; + // Canonical keys are directly constructible, making the common case an + // O(1) delete — the key scan (which also sweeps non-canonical duplicate + // keys) would go quadratic on multi-paragraph deletes (select-all class). + // A missed legacy-keyed duplicate surfaces as a projection defect on the + // next full rebuild and is repaired there. + let canonical_paragraph = format!("paragraph.{}", paragraph_id.0); + if child_map(¶graphs, &canonical_paragraph).is_some() { + paragraphs.delete(&canonical_paragraph)?; + } else { + for key in map_keys(¶graphs) { + let id = child_map(¶graphs, &key) + .and_then(|paragraph| map_string_opt(¶graph, "id")) + .unwrap_or_else(|| key.clone()); + if loro_id_u128(&id) == paragraph_id.0 { + paragraphs.delete(&key)?; + } + } + } + + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + let canonical_block = format!("paragraph_block.{}", block_id.0); + if child_map(&blocks, &canonical_block).is_some() { + blocks.delete(&canonical_block)?; + } else { + for key in map_keys(&blocks) { + let id = child_map(&blocks, &key) + .and_then(|block| map_string_opt(&block, "id")) + .unwrap_or_else(|| key.clone()); + if loro_id_u128(&id) == block_id.0 { + blocks.delete(&key)?; + } + } + } + Ok(()) +} + +fn projection_paragraph_blocks_are_adjacent(projection: &DocumentProjection, first_ix: usize, second_ix: usize) -> bool { + let Some(first_block_ix) = flowstate_document::block_ix_for_paragraph(projection, first_ix) else { + return false; + }; + let Some(second_block_ix) = flowstate_document::block_ix_for_paragraph(projection, second_ix) else { + return false; + }; + second_block_ix == first_block_ix + 1 +} + +pub(crate) fn inserted_newline_boundaries(start_unicode: usize, text: &str) -> Vec { + text + .chars() + .enumerate() + .filter_map(|(offset, ch)| (ch == '\n').then_some(start_unicode + offset)) + .collect() +} + +fn persist_body_paragraph_style_mark_repair(doc: &LoroDoc, package: Option<&mut DocumentPackage>, package_path: Option<&Path>) -> Result { + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + // §act-twelve A12.1.1: two open-cost kills, measured on the impact doc. + // (a) `register_replica` unconditionally committed a `last_seen_at` touch, + // advancing the frontier on EVERY open and permanently invalidating the + // package's frontier-stamped projection/search caches for later opens + // (the probe showed hit=true only on a package's first-ever open). The + // if-absent variant commits only for genuinely new replicas/versions; + // the touch rides the first real edit (`register_pending_author_identity`). + // (b) The whole-corpus style-mark verification scan cost 1.1-1.2s per open; + // packages written by live runtimes stamp `style_marks_clean_frontier`, + // and a frontier match skips the scan (legacy/foreign packages still + // scan once — their next checkpoint stamps). + // Registration is FULLY deferred to the first local edit (the runtime's + // `pending_replica_touch`): every open mints a fresh Loro peer id, so an + // open-time "if absent" check still registered (and committed) every time. + // A replica that never edits needs no record at all. + let replica_registered = false; + let marks_known_clean = package + .as_ref() + .is_some_and(|package| package.manifest.style_marks_clean_frontier.as_deref() == Some(from_frontier.encode().as_slice())); + { + static OPEN_PROBE3: std::sync::OnceLock = std::sync::OnceLock::new(); + if *OPEN_PROBE3.get_or_init(|| std::env::var_os("FLOWSTATE_OPEN_PROBE").is_some()) { + eprintln!( + "[flowstate-open-probe] style-stamp: known_clean={marks_known_clean} stamp_present={} frontier_len={}", + package + .as_ref() + .is_some_and(|p| p.manifest.style_marks_clean_frontier.is_some()), + from_frontier.encode().len() + ); + } + } + let paragraph_marks_repaired = if marks_known_clean { + false + } else { + repair_missing_paragraph_style_marks(doc)? + }; + if !replica_registered && !paragraph_marks_repaired { + return Ok(false); + } + let Some(package) = package else { + return Ok(paragraph_marks_repaired); + }; + package.sync_revisions_from_loro(doc)?; + let update = doc + .export(ExportMode::updates(&from_vv)) + .context("exporting paragraph style repair update")?; + if !update.is_empty() { + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + package.compact_update_segments_if_needed(doc, DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD)?; + } + package.rebuild_search_units_from_loro(doc)?; + if let Some(path) = package_path { + package.write(path)?; + } + Ok(paragraph_marks_repaired) +} + +fn repair_missing_paragraph_style_marks(doc: &LoroDoc) -> Result { + let root = doc.get_map(ROOT); + let Some(flows) = child_map(&root, FLOWS_BY_ID) else { + return Ok(false); + }; + let mut repaired = false; + for flow_id in map_keys(&flows) { + let Some(flow) = child_map(&flows, &flow_id) else { + continue; + }; + if !matches!( + map_string_opt(&flow, FLOW_KIND_KEY).as_deref(), + Some("body" | "table_cell" | "caption" | "header" | "footer") + ) { + continue; + } + let Some(ValueOrContainer::Container(Container::Text(text))) = flow.get(FLOW_TEXT_KEY) else { + continue; + }; + for boundary in body_paragraph_boundaries_missing_style_mark(&text) { + text + .mark( + boundary..boundary + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(ParagraphStyle::Normal), + ) + .context("repairing missing paragraph style mark")?; + repaired = true; + } + } + if repaired { + doc.commit(); + } + Ok(repaired) +} + +fn body_paragraph_boundaries_missing_style_mark(body: &loro::LoroText) -> Vec { + let mut missing = Vec::new(); + let mut unicode_pos = 0_usize; + for item in flowstate_document::streaming_delta::streaming_to_delta(body) { + let loro::TextDelta::Insert { insert, attributes } = item else { + continue; + }; + let has_paragraph_style = paragraph_style_from_attrs(attributes.as_ref()).is_some(); + for ch in insert.chars() { + if ch == '\n' && !has_paragraph_style { + missing.push(unicode_pos); + } + unicode_pos += 1; + } + } + missing +} + +pub(crate) fn repair_paragraph_metadata_after_text_flow_edit( + doc: &LoroDoc, + body: &loro::LoroText, + live_boundaries: &[usize], + reason: &'static str, +) -> loro::LoroResult<()> { + for boundary in live_boundaries { + ensure_paragraph_metadata_at_boundary(doc, body, *boundary)?; + } + let pruned = prune_stale_paragraph_metadata(doc, body)?; + if pruned.changed() { + tracing::warn!( + reason, + stale_paragraphs = pruned.stale_paragraphs, + duplicate_paragraphs = pruned.duplicate_paragraphs, + stale_blocks = pruned.stale_blocks, + duplicate_blocks = pruned.duplicate_blocks, + "pruned stale Loro paragraph metadata after text-flow edit", + ); + } + Ok(()) +} + +#[hotpath::measure] +pub(crate) fn repair_paragraph_metadata_after_stable_split( + doc: &LoroDoc, + body: &loro::LoroText, + boundary: usize, + paragraph_id: flowstate_document::ParagraphId, + block_id: flowstate_document::BlockId, + reason: &'static str, +) -> loro::LoroResult<()> { + let _ = reason; + // §11 complexity contract: a local split retires no records (fresh unique + // ids at a fresh boundary), so the O(records) stale-metadata prune does NOT + // run here (it cost ~100ms per Enter on a 6k-paragraph doc). Stale and + // duplicate records only arise from concurrent-merge imports, and every + // full rebuild collects them as projection defects and schedules canonical + // repair (`refresh_projection` §P2a). + ensure_paragraph_metadata_at_boundary_with_ids(doc, body, boundary, paragraph_id, block_id) +} + +fn ensure_paragraph_metadata_at_boundary(doc: &LoroDoc, body: &loro::LoroText, boundary: usize) -> loro::LoroResult<()> { + ensure_paragraph_metadata_at_boundary_with_keys(doc, body, boundary, None, None) +} + +fn ensure_paragraph_metadata_at_boundary_with_ids( + doc: &LoroDoc, + body: &loro::LoroText, + boundary: usize, + paragraph_id: flowstate_document::ParagraphId, + block_id: flowstate_document::BlockId, +) -> loro::LoroResult<()> { + ensure_paragraph_metadata_at_boundary_with_keys( + doc, + body, + boundary, + Some(format!("paragraph.{}", paragraph_id.0)), + Some(format!("paragraph_block.{}", block_id.0)), + ) +} + +#[hotpath::measure] +fn ensure_paragraph_metadata_at_boundary_with_keys( + doc: &LoroDoc, + body: &loro::LoroText, + boundary: usize, + forced_paragraph_id: Option, + forced_block_id: Option, +) -> loro::LoroResult<()> { + if body.char_at(boundary) != Ok('\n') { + tracing::warn!( + boundary, + "cannot create paragraph metadata because boundary is not a live paragraph newline" + ); + return Ok(()); + } + + let root = doc.get_map(ROOT); + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + // ONE cursor resolution for all three anchor fields — `get_cursor` walks + // text chunks per call, and this writer runs per split and per boundary in + // a mass-repair pass; the three identical calls tripled that cost. + let boundary_cursor = body + .get_cursor(boundary, Side::Left) + .map(|cursor| cursor.encode()); + let paragraph_id = forced_paragraph_id + .or_else(|| paragraph_metadata_key_at_boundary(doc, body, ¶graphs, boundary)) + .unwrap_or_else(|| new_paragraph_metadata_id(boundary)); + let paragraph = paragraphs.ensure_mergeable_map(¶graph_id)?; + paragraph.insert("id", paragraph_id.as_str())?; + paragraph.insert("flow_id", ROOT_BODY_FLOW_ID)?; + if let Some(encoded) = &boundary_cursor { + paragraph.insert("start_cursor", encoded.clone())?; + paragraph.insert("boundary_cursor", encoded.clone())?; + } + let _paragraph_attrs = paragraph.ensure_mergeable_map("attrs")?; + + let block_id = forced_block_id + .or_else(|| paragraph_block_key_at_boundary(doc, body, &blocks, boundary)) + .unwrap_or_else(|| new_paragraph_block_id(boundary)); + let block = blocks.ensure_mergeable_map(&block_id)?; + block.insert("id", block_id.as_str())?; + block.insert("kind", "paragraph")?; + block.insert("flow_id", ROOT_BODY_FLOW_ID)?; + block.insert("paragraph_id", paragraph_id.as_str())?; + if let Some(encoded) = boundary_cursor { + block.insert("anchor_cursor", encoded)?; + } + let _block_attrs = block.ensure_mergeable_map("attrs")?; + let _nested_refs = block.ensure_mergeable_map("nested_refs")?; + Ok(()) +} + +fn paragraph_metadata_key_at_boundary(doc: &LoroDoc, body: &loro::LoroText, paragraphs: &LoroMap, boundary: usize) -> Option { + let mut keys = metadata_keys_at_boundary(doc, body, paragraphs, "boundary_cursor", boundary); + if boundary == 0 + && let Some(root_ix) = keys.iter().position(|key| key == ROOT_FIRST_PARAGRAPH_ID) + { + return Some(keys.swap_remove(root_ix)); + } + keys.into_iter().next() +} + +fn paragraph_block_key_at_boundary(doc: &LoroDoc, body: &loro::LoroText, blocks: &LoroMap, boundary: usize) -> Option { + // `boundary` is already validated live by every caller, so the resolved position + // only has to equal it — a single-element live set gives that test in O(1). + let pos_by_id = boundary_cursor_positions(doc, body, blocks, &["anchor_cursor"]); + let live = [boundary]; + let mut keys = Vec::new(); + for key in map_keys(blocks) { + let Some(block) = child_map(blocks, &key) else { + continue; + }; + if map_string_opt(&block, "kind").as_deref() != Some("paragraph") { + continue; + } + if live_cursor_pos(doc, &live, &pos_by_id, &block, "anchor_cursor") == Some(boundary) { + keys.push(key); + } + } + if boundary == 0 + && let Some(main_ix) = keys.iter().position(|key| key == MAIN_BODY_BLOCK_ID) + { + return Some(keys.swap_remove(main_ix)); + } + keys.into_iter().next() +} + +fn metadata_keys_at_boundary(doc: &LoroDoc, body: &loro::LoroText, maps: &LoroMap, cursor_key: &str, boundary: usize) -> Vec { + // `boundary` is already validated live by callers, so a single-element live set + // reduces the per-record check to `resolved position == boundary` in O(1). + let pos_by_id = boundary_cursor_positions(doc, body, maps, &[cursor_key]); + let live = [boundary]; + map_keys(maps) + .into_iter() + .filter(|key| { + child_map(maps, key) + .as_ref() + .and_then(|map| live_cursor_pos(doc, &live, &pos_by_id, map, cursor_key)) + == Some(boundary) + }) + .collect() +} + +#[derive(Default)] +struct ParagraphMetadataPrune { + stale_paragraphs: usize, + duplicate_paragraphs: usize, + stale_blocks: usize, + duplicate_blocks: usize, +} + +impl ParagraphMetadataPrune { + fn changed(&self) -> bool { + self.stale_paragraphs > 0 || self.duplicate_paragraphs > 0 || self.stale_blocks > 0 || self.duplicate_blocks > 0 + } +} + +#[hotpath::measure] +fn prune_stale_paragraph_metadata(doc: &LoroDoc, body: &loro::LoroText) -> loro::LoroResult { + let body_snapshot = body.to_string(); + let root = doc.get_map(ROOT); + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + let mut pruned = ParagraphMetadataPrune::default(); + + // Resolve every record's boundary in one batched pass instead of an O(records) + // `get_cursor_pos` per record: `live_boundaries` validates liveness in O(log N), + // and the two `*_pos` indexes give O(1) position lookups. `block_pos` is reused by + // both block loops (the block registry is untouched between them); `paragraph_pos` + // covers both the `boundary_cursor` and `start_cursor` fields. + let live_boundaries = live_boundary_positions(&body_snapshot); + let block_pos = boundary_cursor_positions(doc, body, &blocks, &["anchor_cursor"]); + let paragraph_pos = boundary_cursor_positions(doc, body, ¶graphs, &["boundary_cursor", "start_cursor"]); + + let mut block_boundary_by_paragraph = FxHashMap::::default(); + for key in map_keys(&blocks) { + let Some(block) = child_map(&blocks, &key) else { + continue; + }; + if map_string_opt(&block, "kind").as_deref() != Some("paragraph") { + continue; + } + let Some(paragraph_id) = map_string_opt(&block, "paragraph_id") else { + continue; + }; + let Some(boundary) = live_cursor_pos(doc, &live_boundaries, &block_pos, &block, "anchor_cursor") else { + continue; + }; + block_boundary_by_paragraph + .entry(paragraph_id) + .or_insert(boundary); + } + + let mut paragraph_by_boundary = BTreeMap::::new(); + let mut paragraphs_to_delete = Vec::new(); + for key in map_keys(¶graphs) { + let Some(paragraph) = child_map(¶graphs, &key) else { + paragraphs_to_delete.push(key); + pruned.stale_paragraphs += 1; + continue; + }; + let boundary = live_cursor_pos(doc, &live_boundaries, ¶graph_pos, ¶graph, "boundary_cursor") + .or_else(|| live_cursor_pos(doc, &live_boundaries, ¶graph_pos, ¶graph, "start_cursor")) + .or_else(|| { + let boundary = block_boundary_by_paragraph.get(&key).copied()?; + repair_paragraph_boundary_cursors(body, ¶graph, boundary).ok()?; + Some(boundary) + }); + let Some(boundary) = boundary else { + paragraphs_to_delete.push(key); + pruned.stale_paragraphs += 1; + continue; + }; + if let Some(existing) = paragraph_by_boundary.get(&boundary) { + if prefer_paragraph_metadata_key(boundary, existing, &key) { + paragraphs_to_delete.push(existing.clone()); + paragraph_by_boundary.insert(boundary, key); + } else { + paragraphs_to_delete.push(key); + } + pruned.duplicate_paragraphs += 1; + } else { + paragraph_by_boundary.insert(boundary, key); + } + } + for key in paragraphs_to_delete { + paragraphs.delete(&key)?; + } + + let mut block_by_boundary = BTreeMap::::new(); + let mut blocks_to_delete = Vec::new(); + for key in map_keys(&blocks) { + let Some(block) = child_map(&blocks, &key) else { + continue; + }; + if map_string_opt(&block, "kind").as_deref() != Some("paragraph") { + continue; + } + let Some(boundary) = live_cursor_pos(doc, &live_boundaries, &block_pos, &block, "anchor_cursor") else { + blocks_to_delete.push(key); + pruned.stale_blocks += 1; + continue; + }; + if let Some(existing) = block_by_boundary.get(&boundary) { + if prefer_paragraph_block_key(boundary, existing, &key) { + blocks_to_delete.push(existing.clone()); + block_by_boundary.insert(boundary, key); + } else { + blocks_to_delete.push(key); + } + pruned.duplicate_blocks += 1; + } else { + block_by_boundary.insert(boundary, key); + } + } + for key in blocks_to_delete { + blocks.delete(&key)?; + } + + Ok(pruned) +} + +fn repair_paragraph_boundary_cursors(body: &loro::LoroText, paragraph: &LoroMap, boundary: usize) -> loro::LoroResult<()> { + if let Some(cursor) = body.get_cursor(boundary, Side::Left) { + paragraph.insert("boundary_cursor", cursor.encode())?; + } + if let Some(cursor) = body.get_cursor(boundary, Side::Left) { + paragraph.insert("start_cursor", cursor.encode())?; + } + Ok(()) +} + +fn prefer_paragraph_metadata_key(boundary: usize, existing: &str, candidate: &str) -> bool { + boundary == 0 && candidate == ROOT_FIRST_PARAGRAPH_ID && existing != ROOT_FIRST_PARAGRAPH_ID +} + +fn prefer_paragraph_block_key(boundary: usize, existing: &str, candidate: &str) -> bool { + boundary == 0 && candidate == MAIN_BODY_BLOCK_ID && existing != MAIN_BODY_BLOCK_ID +} + +/// Resolve every live boundary cursor stored under `cursor_fields` across `records` +/// in a SINGLE pass, returning an `id → position` map. Mirrors the batch resolver in +/// `flowstate_document::loro_projection` (`boundary_cursor_positions`): each record +/// contributes an O(1) cursor decode, and the whole set of positions is resolved by +/// one vendored-Loro `query_text_id_positions` chunk scan instead of an O(records) +/// history-traced `get_cursor_pos` per record. That is what removes the O(records²) +/// scan which pinned the CRDT actor at 100% CPU — and drove the unbounded allocation +/// that OOM-killed the host — while editing a large document. Ids not present +/// (deleted) are simply absent; [`live_cursor_pos`] falls back to per-id +/// `get_cursor_pos` for those, preserving exact parity with the old scan. +fn boundary_cursor_positions(doc: &LoroDoc, body: &loro::LoroText, records: &LoroMap, cursor_fields: &[&str]) -> FxHashMap { + let container = body.id(); + let mut ids: Vec = Vec::new(); + for key in map_keys(records) { + let Some(record) = child_map(records, &key) else { + continue; + }; + for field in cursor_fields { + if let Some(bytes) = map_binary_opt(&record, field) + && let Ok(cursor) = Cursor::decode(&bytes) + && cursor.container == container + && let Some(id) = cursor.id + { + ids.push(id); + } + } + } + let mut positions = FxHashMap::default(); + if ids.is_empty() { + return positions; + } + for (id, pos) in ids + .iter() + .copied() + .zip(doc.inner().query_text_id_positions(&container, &ids)) + { + if let Some(pos) = pos { + positions.insert(id, pos); + } + } + positions +} + +/// Sorted Unicode-code-point indices of every paragraph-boundary newline in +/// `body_snapshot`, built in one O(N) pass so boundary-liveness can be tested with an +/// O(log N) `binary_search` instead of a per-record O(N) `chars().nth(pos)` — the +/// second quadratic factor (alongside `get_cursor_pos`) in the old per-record scan. +fn live_boundary_positions(body_snapshot: &str) -> Vec { + body_snapshot + .chars() + .enumerate() + .filter_map(|(i, c)| (c == '\n').then_some(i)) + .collect() +} + +/// Resolve one record's boundary cursor to its live position. `pos_by_id` (built by +/// [`boundary_cursor_positions`]) gives an O(1) hit for every live cursor; only a +/// cursor whose id is no longer live falls back to the history-traced `get_cursor_pos` +/// (exact parity with the old path). The resolved position must land on a member of +/// the sorted `live_boundaries` set to count — pass the full newline set to validate +/// against the whole document, or a single-element slice to test one already-validated +/// live boundary. +fn live_cursor_pos( + doc: &LoroDoc, + live_boundaries: &[usize], + pos_by_id: &FxHashMap, + map: &LoroMap, + cursor_key: &str, +) -> Option { + let cursor = Cursor::decode(&map_binary_opt(map, cursor_key)?).ok()?; + let pos = match cursor.id { + // The batched resolver covered every id-carrying cursor: absence means the + // anchor is DELETED. Falling back to `get_cursor_pos` here would walk + // update history per dead anchor — O(records × history) after a mass + // delete (the ctrl-A freeze class). Dead ⇒ not live, directly. + Some(id) => pos_by_id.get(&id).copied()?, + // Id-less cursors (F7: whole-body-end resolution) are rare and can't be + // batch-resolved; the per-cursor resolution stays for them alone. + None => doc.get_cursor_pos(&cursor).ok()?.current.pos, + }; + live_boundaries.binary_search(&pos).is_ok().then_some(pos) +} + +fn live_object_cursor_pos(doc: &LoroDoc, body_snapshot: &str, map: &LoroMap, cursor_key: &str) -> Option { + let cursor = Cursor::decode(&map_binary_opt(map, cursor_key)?).ok()?; + let pos = doc.get_cursor_pos(&cursor).ok()?.current.pos; + (body_snapshot.chars().nth(pos) == Some(OBJECT_REPLACEMENT)).then_some(pos) +} + +fn new_paragraph_metadata_id(boundary: usize) -> String { + if boundary == 0 { + ROOT_FIRST_PARAGRAPH_ID.to_string() + } else { + format!("paragraph.{}", Uuid::new_v4().as_u128()) + } +} + +fn new_paragraph_block_id(boundary: usize) -> String { + if boundary == 0 { + MAIN_BODY_BLOCK_ID.to_string() + } else { + format!("paragraph_block.{}", Uuid::new_v4().as_u128()) + } +} + +fn map_keys(map: &LoroMap) -> Vec { + let mut keys = map.keys().map(|key| key.to_string()).collect::>(); + keys.sort(); + keys +} + +fn child_map(parent: &LoroMap, key: &str) -> Option { + parent.get(key).and_then(|value| match value { + ValueOrContainer::Container(container) => container.into_map().ok(), + ValueOrContainer::Value(_) => None, + }) +} + +/// §28: centralized resolution of a stored raw Loro container id string. +/// +/// Parses the durable `*_container_id` string into a [`ContainerID`] and fetches +/// the live container directly from the document for efficient runtime access. +/// Returns `None` when the id is missing/unparseable or the container is +/// absent/detached/deleted, so callers can fall back to map-key traversal. +fn container_by_id(doc: &LoroDoc, container_id: &str) -> Option { + let container = doc.get_container(ContainerID::try_from(container_id).ok()?)?; + (container.is_attached() && !container.is_deleted()).then_some(container) +} + +fn container_text_by_id(doc: &LoroDoc, container_id: &str) -> Option { + container_by_id(doc, container_id)?.into_text().ok() +} + +/// §28: resolve a flow's canonical `LoroText`, preferring direct resolution via +/// the flow's stored `text_container_id` and falling back to key traversal when +/// the id is missing/unresolvable. +fn flow_text(doc: &LoroDoc, flow: &LoroMap) -> loro::LoroResult { + if let Some(container_id) = map_string_opt(flow, "text_container_id") + && let Some(text) = container_text_by_id(doc, &container_id) + { + return Ok(text); + } + flow.ensure_mergeable_text(FLOW_TEXT_KEY) +} + +fn child_movable_list(parent: &LoroMap, key: &str) -> Option { + parent.get(key).and_then(|value| match value { + ValueOrContainer::Container(Container::MovableList(list)) => Some(list), + _ => None, + }) +} + +fn movable_list_strings(list: &LoroMovableList) -> Vec { + (0..list.len()) + .filter_map(|ix| match list.get(ix) { + Some(ValueOrContainer::Value(LoroValue::String(value))) => Some(value.to_string()), + _ => None, + }) + .collect() +} + +fn map_string_opt(map: &LoroMap, key: &str) -> Option { + map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + }) +} + +fn map_binary_opt(map: &LoroMap, key: &str) -> Option> { + map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::Binary(value)) => Some(value.to_vec()), + _ => None, + }) +} + +fn attach_package_assets(document: &mut DocumentProjection, package: &DocumentPackage) { + flowstate_document::attach_package_assets(document, &package.assets); +} + +#[derive(Debug, Default)] +struct AssetMergeSummary { + any_changed: bool, + metadata_changed: bool, + changed_asset_ids: Vec, +} + +fn merge_asset_records_into_projection(projection: &mut DocumentProjection, records: &[AssetRecord]) -> AssetMergeSummary { + let mut summary = AssetMergeSummary::default(); + for record in records { + let existing = projection.assets.assets.get(&record.id); + let metadata_changed = existing.is_none_or(|existing| asset_record_metadata_changed(existing, record)); + let bytes_changed = existing.is_none_or(|existing| existing.bytes.as_ref() != record.bytes.as_ref()); + if !metadata_changed && !bytes_changed { + continue; + } + summary.any_changed = true; + if metadata_changed { + summary.metadata_changed = true; + let id = record.id.0.to_string(); + if !summary.changed_asset_ids.contains(&id) { + summary.changed_asset_ids.push(id); + } + } + projection.assets.assets.insert(record.id, record.clone()); + } + summary +} + +fn asset_record_metadata_changed(existing: &AssetRecord, next: &AssetRecord) -> bool { + existing.mime_type != next.mime_type || existing.original_name != next.original_name || existing.content_hash != next.content_hash +} + +fn install_undo_selection_callbacks(undo: &UndoManager, state: &Arc>) { + let push_state = Arc::clone(state); + undo.set_on_push(Some(Box::new(move |_, _, _| { + let mut meta = UndoItemMeta::new(); + if let Ok(state) = push_state.lock() + && let Some(selection) = &state.pending_selection + { + // Loro-first spec §10: selection positions ride in the first-class + // `cursors` field, which Loro actively TRANSFORMS through remote diffs + // and container remaps while the item sits on the undo stack. The + // opaque `value` blob keeps only affinity/direction sidecar data (plus + // the capture-time cursor bytes as a decode fallback). + if let Ok(snapshot) = postcard::from_bytes::(selection.as_ref()) { + if let Ok(anchor) = Cursor::decode(&snapshot.anchor_cursor) { + meta.add_cursor(&anchor); + } + if let Ok(head) = Cursor::decode(&snapshot.head_cursor) { + meta.add_cursor(&head); + } + } + meta.set_value(LoroValue::Binary(selection.clone().into())); + } + meta + }))); + + let pop_state = Arc::clone(state); + undo.set_on_pop(Some(Box::new(move |_, _, meta| { + let transformed: Vec> = meta + .cursors + .iter() + .map(|entry| entry.cursor.encode()) + .collect(); + let LoroValue::Binary(bytes) = meta.value else { + return; + }; + match postcard::from_bytes::(bytes.as_ref()) { + Ok(mut selection) => { + // Prefer the stack-transformed cursors over the capture-time bytes: + // concurrent remote edits between capture and undo have already been + // applied to them by the UndoManager (spec §10, test #9). + if transformed.len() == 2 { + selection.anchor_cursor = transformed[0].clone(); + selection.head_cursor = transformed[1].clone(); + } + if let Ok(mut state) = pop_state.lock() { + state.restored_selection = Some(selection); + } + }, + Err(error) => { + tracing::warn!(error = %error, "decoding Loro undo selection metadata failed"); + }, + } + }))); +} + +fn map_i64_opt(map: &LoroMap, key: &str) -> Option { + map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::I64(value)) => Some(value), + _ => None, + }) +} + +fn map_bool_opt(map: &LoroMap, key: &str) -> Option { + map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::Bool(value)) => Some(value), + _ => None, + }) +} + +fn existing_child_map(parent: &LoroMap, key: &str) -> Result { + match parent.get(key) { + Some(ValueOrContainer::Container(Container::Map(map))) => Ok(map), + _ => anyhow::bail!("Comment record `{key}` does not exist"), + } +} + +fn existing_comment_thread(doc: &LoroDoc, comment_id: u128) -> Result { + let root = flowstate_document::loro_schema::root_map(doc); + let comments = existing_child_map(&root, flowstate_document::COMMENTS_BY_ID)?; + existing_child_map(&comments, &comment_id.to_string()) +} + +fn validated_comment_body(body: &str) -> Result { + // Strip control characters (a pasted `\r\n` becomes `\n`) so a comment can + // never smuggle escape sequences or zero-width control bytes into peers' UIs. + let sanitized: String = body + .chars() + .filter(|ch| *ch == '\n' || *ch == '\t' || !ch.is_control()) + .collect(); + let body = sanitized.trim(); + anyhow::ensure!(!body.is_empty(), "Comment text cannot be empty"); + anyhow::ensure!(body.len() <= 32 * 1024, "Comment text is too long"); + Ok(body.to_owned()) +} + +fn write_comment_message( + messages: &LoroMap, + message_id: u128, + body: &str, + author_user_id: u128, + author_display_name: &str, + now: i64, +) -> Result<()> { + let message = messages.ensure_mergeable_map(&message_id.to_string())?; + message.insert("id", message_id.to_string())?; + message.insert("author_user_id", author_user_id.to_string())?; + message.insert("author_display_name", author_display_name)?; + message.insert("body", body)?; + message.insert("created_at", now)?; + message.insert("updated_at", now)?; + Ok(()) +} + +fn unix_time_secs() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .try_into() + .unwrap_or(i64::MAX) +} + +fn parse_blake3_hex(value: &str) -> Option<[u8; 32]> { + if value.len() != 64 { + return None; + } + let mut bytes = [0u8; 32]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).ok()?; + } + Some(bytes) +} + +fn selection_direction(anchor: DocumentOffset, head: DocumentOffset) -> SelectionDirection { + match anchor.cmp(&head) { + std::cmp::Ordering::Less => SelectionDirection::Forward, + std::cmp::Ordering::Greater => SelectionDirection::Backward, + std::cmp::Ordering::Equal => SelectionDirection::None, + } +} + +std::thread_local! { + /// §caret-anchor: count of O(doc) `fork_at` caret rebases on this thread. The + /// FAST path (stored cursors) does NOT touch it; a test asserts the delta is 0 + /// when the caret was captured, so a regression to always-fork fails loud. + static CARET_REBASE_FORKS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Read this thread's caret fork-rebase count (see `CARET_REBASE_FORKS`). +pub fn caret_rebase_fork_count() -> u64 { + CARET_REBASE_FORKS.with(std::cell::Cell::get) +} + +pub(crate) fn cursor_for_boundary(text: &LoroText, unicode_pos: usize, affinity: SelectionAffinity) -> Option { + let len = text.len_unicode(); + let pos = unicode_pos.min(len); + match affinity { + SelectionAffinity::Before if pos > 0 => text.get_cursor(pos - 1, Side::Right), + SelectionAffinity::Before => text.get_cursor(0, Side::Left), + SelectionAffinity::After if pos < len => text.get_cursor(pos, Side::Left), + SelectionAffinity::After => text.get_cursor(len, Side::Right), + SelectionAffinity::Neutral => text.get_cursor(pos, Side::Middle), + } +} + +fn resolved_cursor_boundary_unicode(text: &LoroText, resolved: &loro::cursor::PosQueryResult) -> Option { + let event_len = text.convert_pos(text.len_utf8(), PosType::Bytes, PosType::Event)?; + let event_pos = resolved + .current + .pos + .saturating_add(usize::from(resolved.current.side == Side::Right)) + .min(event_len); + text.convert_pos(event_pos, PosType::Event, PosType::Unicode) +} + +fn gpui_gravity_from_presence(gravity: VisualGravity) -> gpui_flowtext::VisualGravity { + match gravity { + VisualGravity::Upstream => gpui_flowtext::VisualGravity::Upstream, + VisualGravity::Downstream => gpui_flowtext::VisualGravity::Downstream, + VisualGravity::Neutral => gpui_flowtext::VisualGravity::Neutral, + } +} + +/// §16: map the persisted undo-snapshot affinity back onto the editor's +/// `gpui_flowtext::SelectionAffinity` when rebuilding a restored selection. +fn gpui_affinity_from_undo(affinity: UndoSelectionAffinity) -> gpui_flowtext::SelectionAffinity { + match affinity { + UndoSelectionAffinity::Before => gpui_flowtext::SelectionAffinity::Before, + UndoSelectionAffinity::After => gpui_flowtext::SelectionAffinity::After, + UndoSelectionAffinity::Neutral => gpui_flowtext::SelectionAffinity::Neutral, + } +} + +pub(super) fn paragraph_style_from_attrs(attrs: Option<&FxHashMap>) -> Option { + let value = attrs?.get(MARK_PARAGRAPH_STYLE)?; + match value { + LoroValue::I64(0) => Some(ParagraphStyle::Normal), + LoroValue::I64(slot) if *slot > 0 => u8::try_from(*slot - 1).ok().map(ParagraphStyle::Custom), + _ => None, + } +} + +pub(crate) fn paragraph_style_value(style: ParagraphStyle) -> i64 { + match style { + ParagraphStyle::Normal => 0, + ParagraphStyle::Custom(slot) => i64::from(slot) + 1, + } +} + +pub(crate) fn mark_run_styles(text: &loro::LoroText, range: std::ops::Range, styles: RunStyles) -> loro::LoroResult<()> { + for key in [MARK_RUN_SEMANTIC_STYLE, MARK_HIGHLIGHT_STYLE, MARK_DIRECT_UNDERLINE, MARK_STRIKETHROUGH] { + text.unmark(range.clone(), key)?; + } + if let RunSemanticStyle::Custom(slot) = styles.semantic { + text.mark(range.clone(), MARK_RUN_SEMANTIC_STYLE, i64::from(slot))?; + } + if let Some(flowstate_document::HighlightStyle::Custom(slot)) = styles.highlight { + text.mark(range.clone(), MARK_HIGHLIGHT_STYLE, i64::from(slot))?; + } + if styles.direct_underline { + text.mark(range.clone(), MARK_DIRECT_UNDERLINE, true)?; + } + if styles.strikethrough { + text.mark(range, MARK_STRIKETHROUGH, true)?; + } + Ok(()) +} + +fn insert_image_block( + doc: &LoroDoc, + unicode_index: usize, + asset_id: u128, + alt_text: &str, + caption: Option<&str>, + sizing: InputImageSizing, + alignment: InputBlockAlignment, +) -> Result<()> { + let body = body_text(doc); + body.insert(unicode_index, &OBJECT_REPLACEMENT.to_string())?; + let block = ensure_block(doc, "image", ROOT_BODY_FLOW_ID, &body, unicode_index)?; + block.insert("asset_id", asset_id.to_string())?; + copy_asset_metadata_to_image_block(doc, &block, asset_id)?; + + let alt_flow_id = nested_flow_id("image_alt"); + block.insert("alt_text_flow_id", alt_flow_id.as_str())?; + let alt_flow = ensure_flow(doc, &alt_flow_id, "alt_text")?; + replace_text_incrementally(&alt_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, alt_text)?; + + if let Some(caption) = caption { + let caption_flow_id = nested_flow_id("image_caption"); + block.insert("caption_flow_id", caption_flow_id.as_str())?; + let caption_flow = ensure_flow(doc, &caption_flow_id, "caption")?; + let caption_text = caption_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + replace_text(&caption_text, SENTINEL_NEWLINE)?; + caption_text.mark(0..1, MARK_PARAGRAPH_STYLE, 0_i64)?; + if !caption.is_empty() { + caption_text.insert(1, caption)?; + } + } + + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("alignment", alignment_name(alignment))?; + match sizing { + InputImageSizing::Intrinsic => attrs.insert("sizing", "intrinsic")?, + InputImageSizing::FitWidth => attrs.insert("sizing", "fit_width")?, + InputImageSizing::Fixed { width_px, height_px } => { + attrs.insert("sizing", "fixed")?; + attrs.insert("width_px", i64::from(width_px))?; + if let Some(height_px) = height_px { + attrs.insert("height_px", i64::from(height_px))?; + } + }, + }; + Ok(()) +} + +fn insert_image_block_with_id( + doc: &LoroDoc, + unicode_index: usize, + block_id: flowstate_document::BlockId, + image: &flowstate_document::InputImageBlock, +) -> Result<()> { + let body = body_text(doc); + body.insert(unicode_index, &OBJECT_REPLACEMENT.to_string())?; + let block_key = object_block_key("image", block_id); + let block = ensure_block_with_id(doc, &block_key, "image", ROOT_BODY_FLOW_ID, &body, unicode_index)?; + replace_image_block_from_input(doc, &block, image) +} + +fn insert_equation_block(doc: &LoroDoc, unicode_index: usize, source: &str, display: InputEquationDisplay) -> Result<()> { + let body = body_text(doc); + body.insert(unicode_index, &OBJECT_REPLACEMENT.to_string())?; + let block = ensure_block(doc, "equation", ROOT_BODY_FLOW_ID, &body, unicode_index)?; + let source_flow_id = nested_flow_id("equation_source"); + block.insert("source_flow_id", source_flow_id.as_str())?; + let source_flow = ensure_flow(doc, &source_flow_id, "equation_source")?; + replace_text(&source_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, source)?; + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("syntax", "latex")?; + attrs.insert("display", equation_display_name(display))?; + Ok(()) +} + +fn insert_equation_block_with_id( + doc: &LoroDoc, + unicode_index: usize, + block_id: flowstate_document::BlockId, + equation: &flowstate_document::InputEquationBlock, +) -> Result<()> { + let body = body_text(doc); + body.insert(unicode_index, &OBJECT_REPLACEMENT.to_string())?; + let block_key = object_block_key("equation", block_id); + let block = ensure_block_with_id(doc, &block_key, "equation", ROOT_BODY_FLOW_ID, &body, unicode_index)?; + replace_equation_block_from_input(doc, &block, equation) +} + +fn insert_table_block( + doc: &LoroDoc, + unicode_index: usize, + rows: usize, + columns: usize, + column_widths: &[InputTableColumnWidth], + header_row: bool, +) -> Result<()> { + let body = body_text(doc); + body.insert(unicode_index, &OBJECT_REPLACEMENT.to_string())?; + let block = ensure_block(doc, "table", ROOT_BODY_FLOW_ID, &body, unicode_index)?; + let table = block.ensure_mergeable_map("table")?; + table.insert("header_row", header_row)?; + let row_order = table.ensure_mergeable_movable_list("row_order")?; + let column_order = table.ensure_mergeable_movable_list("column_order")?; + let rows_by_id = table.ensure_mergeable_map("rows_by_id")?; + let columns_by_id = table.ensure_mergeable_map("columns_by_id")?; + let cells_by_id = table.ensure_mergeable_map("cells_by_id")?; + table.insert("columns_container_id", columns_by_id.id().to_string())?; + table.insert("cells_container_id", cells_by_id.id().to_string())?; + + // §P2b: a dimensions-only `InsertTable` is a genuinely-new table, so mint fresh + // durable row/column ids (mirroring the import path) and address every + // container by its `*_loro_id` string. The deterministic `cell_loro_id_for` + // keeps each coordinate's cell id a pure function of `(row, column)`. + let mut minted_columns: Vec<(ColumnId, String)> = Vec::with_capacity(columns); + for column_ix in 0..columns { + let column_id = ColumnId(Uuid::new_v4().as_u128()); + let column_id_str = column_loro_id(column_id); + column_order.push(column_id_str.as_str())?; + let column = columns_by_id.ensure_mergeable_map(&column_id_str)?; + column.insert("id", column_id_str.as_str())?; + let _attrs = column.ensure_mergeable_map("attrs")?; + write_table_column_width( + &column, + column_widths + .get(column_ix) + .unwrap_or(&InputTableColumnWidth::Auto), + )?; + minted_columns.push((column_id, column_id_str)); + } + + for _ in 0..rows { + let row_id = RowId(Uuid::new_v4().as_u128()); + let row_id_str = row_loro_id(row_id); + row_order.push(row_id_str.as_str())?; + let row = rows_by_id.ensure_mergeable_map(&row_id_str)?; + row.insert("id", row_id_str.as_str())?; + let _attrs = row.ensure_mergeable_map("attrs")?; + for (column_id, column_id_str) in &minted_columns { + let cell_id_str = cell_loro_id_for(row_id, *column_id); + let cell = cells_by_id.ensure_mergeable_map(&cell_id_str)?; + cell.insert("id", cell_id_str.as_str())?; + cell.insert("row_id", row_id_str.as_str())?; + cell.insert("column_id", column_id_str.as_str())?; + cell.insert("row_span", 1_i64)?; + cell.insert("column_span", 1_i64)?; + let _attrs = cell.ensure_mergeable_map("attrs")?; + let _nested_table_ids = cell.ensure_mergeable_movable_list("nested_table_ids")?; + let _nested_tables_by_id = cell.ensure_mergeable_map("nested_tables_by_id")?; + let flow_id = format!("{cell_id_str}.flow"); + cell.insert("flow_id", flow_id.as_str())?; + let flow = ensure_flow(doc, &flow_id, "table_cell")?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + cell.insert("text_container_id", text.id().to_string())?; + replace_text(&text, SENTINEL_NEWLINE)?; + text.mark(0..1, MARK_PARAGRAPH_STYLE, 0_i64)?; + } + } + Ok(()) +} + +fn insert_table_block_with_id( + doc: &LoroDoc, + unicode_index: usize, + block_id: flowstate_document::BlockId, + table: &InputTableBlock, +) -> Result<()> { + let body = body_text(doc); + body.insert(unicode_index, &OBJECT_REPLACEMENT.to_string())?; + let block_key = object_block_key("table", block_id); + let block = ensure_block_with_id(doc, &block_key, "table", ROOT_BODY_FLOW_ID, &body, unicode_index)?; + replace_table_block_from_input(doc, &block, table) +} + +fn ensure_flow(doc: &LoroDoc, flow_id: &str, kind: &str) -> loro::LoroResult { + let root = doc.get_map(ROOT); + let flows = root.ensure_mergeable_map(FLOWS_BY_ID)?; + let flow = flows.ensure_mergeable_map(flow_id)?; + flow.insert(FLOW_ID_KEY, flow_id)?; + flow.insert(FLOW_KIND_KEY, kind)?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + let _attrs = flow.ensure_mergeable_map(FLOW_ATTRS_KEY)?; + flow.insert("text_container_id", text.id().to_string())?; + Ok(flow) +} + +fn ensure_block(doc: &LoroDoc, kind: &str, flow_id: &str, text: &loro::LoroText, pos: usize) -> loro::LoroResult { + let id = format!("{kind}.{}", Uuid::new_v4().as_u128()); + ensure_block_with_id(doc, &id, kind, flow_id, text, pos) +} + +fn ensure_block_with_id(doc: &LoroDoc, id: &str, kind: &str, flow_id: &str, text: &loro::LoroText, pos: usize) -> loro::LoroResult { + let root = doc.get_map(ROOT); + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + let block = blocks.ensure_mergeable_map(id)?; + block.insert("id", id)?; + block.insert("kind", kind)?; + block.insert("flow_id", flow_id)?; + if let Some(cursor) = text.get_cursor(pos, Side::Left) { + block.insert("anchor_cursor", cursor.encode())?; + } + let _attrs = block.ensure_mergeable_map("attrs")?; + let _nested_refs = block.ensure_mergeable_map("nested_refs")?; + Ok(block) +} + +fn object_block_key(kind: &str, block_id: flowstate_document::BlockId) -> String { + format!("{kind}.{}", block_id.0) +} + +fn replace_text(text: &loro::LoroText, value: &str) -> loro::LoroResult<()> { + let len = text.len_unicode(); + if len > 0 { + text.delete(0, len)?; + } + if !value.is_empty() { + text.insert(0, value)?; + } + Ok(()) +} + +fn nested_flow_id(kind: &str) -> String { + format!("{kind}.{}", Uuid::new_v4().as_u128()) +} + +fn alignment_name(alignment: InputBlockAlignment) -> &'static str { + match alignment { + InputBlockAlignment::Left => "left", + InputBlockAlignment::Center => "center", + InputBlockAlignment::Right => "right", + } +} + +fn equation_display_name(display: InputEquationDisplay) -> &'static str { + match display { + InputEquationDisplay::Display => "display", + InputEquationDisplay::InlineLikeParagraph => "inline_like_paragraph", + } +} + +/// The off-gate half of a split checkpoint (field fix 2026-07-07): heavy +/// package assembly against a doc FORK plus the checked-out package. No live +/// doc access — safe on a detached worker. +pub struct CheckpointJob { + /// §A13.4.4: `Some` = a gate-held fork (legacy shape, still used by tests + /// and callers that already hold one). `None` = the job derives its doc + /// OFF-GATE from the package it owns — the package's snapshot + segments + /// ARE the live doc's state at the checkpoint frontier (the trailing + /// segment was persisted under the same gate hold that built this job), + /// so the ~110ms `doc.fork()` no longer stalls typing at save time. + pub fork: Option, + pub projection: DocumentProjection, + pub package: DocumentPackage, + pub title: String, + pub revision_id: u128, + pub revision_frontiers: Frontiers, + pub author_user_id: Option, + pub peer_id: u64, + pub write_path: Option, + /// `false` for the close-time cache flush: identical assembly, but no + /// named revision is minted (a flush is not a user save). + pub record_revision: bool, +} + +impl CheckpointJob { + /// Run the heavy assembly + disk write. Returns the package (for restore) + /// and whether it reached disk. + pub fn run(mut self) -> (DocumentPackage, io::Result) { + static CHECKPOINT_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + let probe = *CHECKPOINT_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_CHECKPOINT_PROBE").is_some()); + let result = (|| -> io::Result { + let probe_t = std::time::Instant::now(); + // §A13.4.4: reconstruct the checkpoint doc from the package when no + // fork was supplied (shallow chunk fast path, full decode fallback) — + // and verify it landed on the manifest tip before deriving anything. + let fork = match self.fork.take() { + Some(fork) => fork, + None => { + let derived = match self.package.load_loro_doc_shallow()? { + Some(doc) => doc, + None => self.package.load_loro_doc_from_validated()?, + }; + if derived.oplog_frontiers().encode() != self.package.manifest.latest_frontier { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "checkpoint doc derived from the package did not land on the manifest tip", + )); + } + derived + }, + }; + let probe_derive = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + self + .package + .replace_assets_from_document(&self.projection)?; + let probe_assets = probe_t.elapsed(); + // §act-five P9: ONE materialization feeds both the projection cache and the + // search units (was two full `document_from_loro` walks per checkpoint). + let probe_t = std::time::Instant::now(); + // §A13.4.1: the job carries the runtime's maintained projection — + // build the cache payload from IT when frontier-current (from-Loro + // walk only on mismatch). + self + .package + .rebuild_caches_from_projection(&fork, &self.projection)?; + let probe_caches = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + // §act-twelve A12.4.1 incremental persistence: routine revision + // checkpoints (explicit save + debounced autosave) skip the + // full-history snapshot export while the update-segment chain is + // short — durability is unchanged (segments are journaled per edit; + // this write persists them with fresh caches), only the seconds-scale + // consolidation moves to the close/idle cache flush or the segment + // threshold. The shallow accelerator still refreshes so the next open + // stays fast. + static ALWAYS_COMPACT: std::sync::OnceLock = std::sync::OnceLock::new(); + let always_compact = *ALWAYS_COMPACT.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_INCREMENTAL_CHECKPOINT").is_some()); + let consolidate = always_compact + || !self.record_revision + || self.package.loro_update_segments.len() >= flowstate_document::DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD; + if consolidate { + self.package.compact_to_snapshot_any(&fork)?; + } else { + self.package.refresh_shallow_snapshot(&fork); + } + let probe_compact = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + if self.record_revision { + self.package.create_named_revision_at_with_id( + &fork, + self.revision_id, + &self.revision_frontiers, + &self.title, + "Explicit save", + self.author_user_id, + Some(self.peer_id as u128), + )?; + } + let probe_revision = probe_t.elapsed(); + if let Some(path) = &self.write_path { + let probe_t = std::time::Instant::now(); + self.package.write(path)?; + if probe { + eprintln!( + "[flowstate-checkpoint-probe] derive={probe_derive:?} assets={probe_assets:?} caches={probe_caches:?} compact={probe_compact:?} revision={probe_revision:?} write={:?}", + probe_t.elapsed() + ); + } + return Ok(true); + } + if probe { + eprintln!( + "[flowstate-checkpoint-probe] assets={probe_assets:?} caches={probe_caches:?} compact={probe_compact:?} revision={probe_revision:?} write=skipped" + ); + } + Ok(false) + })(); + (self.package, result) + } +} + +/// Assemble a full package (snapshot + assets + caches) from a FORKED doc and +/// a projection clone — no live-doc access, safe off-gate and off-thread. +pub fn assemble_package_bytes(fork: &LoroDoc, projection: &DocumentProjection, title: &str) -> io::Result> { + let mut package = DocumentPackage::from_loro_snapshot_with_assets(fork, title, assets_from_document(projection))?; + package.replace_assets_from_document(projection)?; + // §act-five P9: one materialization for both caches (see `rebuild_caches_from_loro`). + package.rebuild_caches_from_loro(fork)?; + package.to_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + use flowstate_document::{DocumentPackage, ProjectionPatch, ProjectionTextDelta, loro_schema::body_text}; + + fn live_paragraph_metadata_boundaries(doc: &LoroDoc) -> Vec { + let body = body_text(doc); + let snapshot = body.to_string(); + let live_boundaries = live_boundary_positions(&snapshot); + let root = doc.get_map(ROOT); + let paragraphs = root + .ensure_mergeable_map(PARAGRAPHS_BY_ID) + .expect("paragraph registry"); + let pos_by_id = boundary_cursor_positions(doc, &body, ¶graphs, &["boundary_cursor"]); + let mut boundaries = map_keys(¶graphs) + .into_iter() + .filter_map(|key| child_map(¶graphs, &key)) + .filter_map(|paragraph| live_cursor_pos(doc, &live_boundaries, &pos_by_id, ¶graph, "boundary_cursor")) + .collect::>(); + boundaries.sort_unstable(); + boundaries + } + + fn live_paragraph_block_boundaries(doc: &LoroDoc) -> Vec { + let body = body_text(doc); + let snapshot = body.to_string(); + let live_boundaries = live_boundary_positions(&snapshot); + let root = doc.get_map(ROOT); + let blocks = root + .ensure_mergeable_map(BLOCKS_BY_ID) + .expect("block registry"); + let pos_by_id = boundary_cursor_positions(doc, &body, &blocks, &["anchor_cursor"]); + let mut boundaries = map_keys(&blocks) + .into_iter() + .filter_map(|key| child_map(&blocks, &key)) + .filter(|block| map_string_opt(block, "kind").as_deref() == Some("paragraph")) + .filter_map(|block| live_cursor_pos(doc, &live_boundaries, &pos_by_id, &block, "anchor_cursor")) + .collect::>(); + boundaries.sort_unstable(); + boundaries + } + + fn input_paragraph(text: &str) -> flowstate_document::InputParagraph { + flowstate_document::InputParagraph { + style: flowstate_document::ParagraphStyle::Normal, + runs: vec![flowstate_document::InputRun { + text: text.to_string(), + styles: flowstate_document::RunStyles::default(), + }], + } + } + + pub(crate) fn local_update_bytes(events: &[RuntimeEvent]) -> Vec { + events + .iter() + .find_map(|event| match event { + RuntimeEvent::LocalUpdate { bytes, .. } => Some(bytes.clone()), + RuntimeEvent::RemoteUpdateApplied { .. } + | RuntimeEvent::RevisionOpened { .. } + | RuntimeEvent::RevisionForked { .. } + | RuntimeEvent::ProjectionUpdated { .. } + | RuntimeEvent::ProjectionPatched { .. } + | RuntimeEvent::HistoryRebaseRequired { .. } + | RuntimeEvent::SelectionRestored { .. } => None, + }) + .expect("local update bytes") + } + + #[test] + fn local_insert_exports_update_and_invalidates_projection() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + let events = runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?; + assert!(matches!(events.first(), Some(RuntimeEvent::LocalUpdate { bytes, .. }) if !bytes.is_empty())); + assert!( + events + .iter() + .any(|event| matches!(event, RuntimeEvent::ProjectionPatched { .. })) + ); + assert_eq!(flowstate_document::paragraph_text(&runtime.projection_snapshot()?, 0), "hello"); + assert_eq!(body_text(runtime.doc()).to_string(), "\nhello"); + Ok(()) + } + + #[test] + fn semantic_insert_text_projects_inserted_run_styles() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + let styles = RunStyles { + semantic: RunSemanticStyle::Custom(2), + ..RunStyles::default() + }; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "styled".to_string(), + styles, + })?; + + let projection = runtime.projection_snapshot()?; + assert_eq!(flowstate_document::paragraph_text(&projection, 0), "styled"); + assert_eq!(projection.paragraphs[0].runs.len(), 1); + assert_eq!(projection.paragraphs[0].runs[0].styles, styles); + Ok(()) + } + + #[test] + fn comment_threads_converge_and_anchors_survive_edits() -> Result<()> { + let base = flowstate_document::new_loro_document("Comments")?; + let mut source = CrdtRuntime::from_doc(base.fork(), None, None)?; + let mut target = CrdtRuntime::from_doc(base, None, None)?; + let insert = source.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello world".to_string(), + styles: RunStyles::default(), + })?; + target.import_remote_update(&local_update_bytes(&insert))?; + source.set_author_identity(7, Some("Ada".into()))?; + let selection = EditorSelection::range(DocumentOffset { paragraph: 0, byte: 0 }, DocumentOffset { paragraph: 0, byte: 5 }); + let comment_id = source.create_comment(&selection, "Needs a source", 7, "Ada")?; + source.reply_to_comment(comment_id, "Added one", 9, "Grace")?; + source.set_comment_resolved(comment_id, true)?; + + let updates = source.take_pending_publish(); + let chunks = updates + .iter() + .filter_map(|event| match event { + RuntimeEvent::LocalUpdate { bytes, .. } => Some(bytes.as_slice()), + _ => None, + }) + .collect::>(); + target.import_remote_updates(&chunks)?; + + let local = source.comments(); + let remote = target.comments(); + assert_eq!(local, remote); + assert_eq!(local.len(), 1); + assert_eq!(local[0].quoted_text, "hello"); + assert_eq!(local[0].messages.len(), 2); + assert!(local[0].resolved); + assert!(local[0].anchor.is_some()); + + source.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "Well, ".to_string(), + styles: RunStyles::default(), + })?; + let moved = source.comments()[0] + .anchor + .expect("anchor should survive an insertion before it"); + assert_eq!(moved.0.byte, 6); + assert_eq!(moved.1.byte, 11); + Ok(()) + } + + #[test] + fn split_paragraph_creates_live_paragraph_metadata_and_block_anchor() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?; + runtime.command(SemanticCommand::SplitParagraph { + unicode_index: 3, + inherited_style: flowstate_document::ParagraphStyle::Normal, + })?; + + assert_eq!(body_text(runtime.doc()).to_string(), "\nhe\nllo"); + assert_eq!(live_paragraph_metadata_boundaries(runtime.doc()), vec![0, 3]); + assert_eq!(live_paragraph_block_boundaries(runtime.doc()), vec![0, 3]); + let projection = runtime.projection_snapshot()?; + assert_eq!(projection.paragraphs.len(), 2); + assert_eq!(flowstate_document::paragraph_text(&projection, 0), "he"); + assert_eq!(flowstate_document::paragraph_text(&projection, 1), "llo"); + Ok(()) + } + + #[test] + fn runtime_repairs_missing_paragraph_style_marks_on_takeover() -> Result<()> { + let doc = new_loro_document("Malformed")?; + let body = body_text(&doc); + body.insert(1, "bad\nnext")?; + doc.commit(); + assert_eq!(body_paragraph_boundaries_missing_style_mark(&body), vec![4]); + + let runtime = CrdtRuntime::from_doc(doc, None, None)?; + + assert!(body_paragraph_boundaries_missing_style_mark(&body_text(runtime.doc())).is_empty()); + let projection = runtime.projection_snapshot()?; + assert_eq!(projection.paragraphs.len(), 2); + assert_eq!(flowstate_document::paragraph_text(&projection, 0), "bad"); + assert_eq!(flowstate_document::paragraph_text(&projection, 1), "next"); + assert_eq!(projection.paragraphs[1].style, ParagraphStyle::Normal); + Ok(()) + } + + #[test] + fn package_open_persists_missing_paragraph_style_mark_repair() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("malformed.db8"); + let doc = new_loro_document("Malformed")?; + let body = body_text(&doc); + body.insert(1, "bad\nnext")?; + doc.commit(); + assert_eq!(body_paragraph_boundaries_missing_style_mark(&body), vec![4]); + DocumentPackage::from_loro_snapshot(&doc, "Malformed")?.write(&path)?; + + let _runtime = CrdtRuntime::open_package(&path)?; + let package = DocumentPackage::read(&path)?; + let loaded = package.load_loro_doc()?; + + assert_eq!(body_text(&loaded).to_string(), "\nbad\nnext"); + assert!(body_paragraph_boundaries_missing_style_mark(&body_text(&loaded)).is_empty()); + // §P2a: opening also runs the projection-repair pipeline, which writes the + // durable paragraph metadata this malformed doc was missing at boundary 4. + // That repair persists an additional update segment beyond the style-mark + // repair, so the count is no longer exactly one. + assert!(!package.loro_update_segments.is_empty()); + assert_eq!(live_paragraph_metadata_boundaries(&loaded), vec![0, 4]); + Ok(()) + } + + #[test] + fn remote_import_repairs_and_publishes_missing_paragraph_style_marks() -> Result<()> { + let base = new_loro_document("Malformed")?; + let source = base.fork(); + let from_vv = base.state_vv(); + body_text(&source).insert(1, "bad\nnext")?; + source.commit(); + let update = source.export(ExportMode::updates(&from_vv))?; + + let mut target = CrdtRuntime::from_doc(base, None, None)?; + let events = target.import_remote_update(&update)?; + + assert!(body_paragraph_boundaries_missing_style_mark(&body_text(target.doc())).is_empty()); + assert!( + events + .iter() + .any(|event| matches!(event, RuntimeEvent::LocalUpdate { bytes, .. } if !bytes.is_empty())) + ); + Ok(()) + } + + #[test] + fn runtime_persists_local_update_segments() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("runtime.db8"); + let doc = flowstate_document::new_loro_document("Runtime")?; + DocumentPackage::from_loro_snapshot(&doc, "Runtime")?.write(&path)?; + let mut runtime = CrdtRuntime::open_package(&path)?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "persisted".to_string(), + styles: RunStyles::default(), + })?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 10, + text: " twice".to_string(), + styles: RunStyles::default(), + })?; + let package = DocumentPackage::read(&path)?; + // §act-twelve A12.1.1: open no longer commits (replica registration is + // deferred to the first local edit), so the floor is the two edit + // segments — not the old open-repair segment plus edits. + assert!(package.loro_update_segments.len() >= 2); + assert!(package.current_search_units().is_empty()); + let loaded = package.load_loro_doc()?; + assert_eq!(body_text(&loaded).to_string(), "\npersisted twice"); + Ok(()) + } + + #[test] + fn semantic_text_commands_mutate_loro_body_flow() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello world".to_string(), + styles: RunStyles::default(), + })?; + runtime.command(SemanticCommand::DeleteRange { + unicode_index: 6, + unicode_len: 1, + })?; + runtime.command(SemanticCommand::SplitParagraph { + unicode_index: 6, + inherited_style: flowstate_document::ParagraphStyle::Custom(2), + })?; + runtime.command(SemanticCommand::SetRunStyles { + unicode_range: 1..6, + styles: flowstate_document::RunStyles { + semantic: flowstate_document::RunSemanticStyle::Custom(3), + direct_underline: true, + strikethrough: false, + highlight: Some(flowstate_document::HighlightStyle::Custom(4)), + ..Default::default() + }, + })?; + + assert_eq!(body_text(runtime.doc()).to_string(), "\nhello\nworld"); + let delta = body_text(runtime.doc()).to_delta(); + assert!(delta.iter().any(|item| matches!( + item, + loro::TextDelta::Insert { + attributes: Some(attributes), + .. + } if attributes.get(flowstate_document::MARK_RUN_SEMANTIC_STYLE).is_some() + ))); + assert!(delta.iter().any(|item| matches!( + item, + loro::TextDelta::Insert { + insert, + attributes: Some(attributes), + } if insert == "\n" && attributes.get(flowstate_document::MARK_PARAGRAPH_STYLE).is_some() + ))); + Ok(()) + } + + // FS-170: a caret can rest on either side of a block object; the two sides + // must resolve to DISTINCT document offsets (the "before" side to the previous + // paragraph's end, the "after" side to the following paragraph's start). + // Before the decode fix both collapsed onto the previous paragraph's end, + // sending remote/undo carets to the wrong side of the object. + #[test] + fn object_between_paragraphs_resolves_distinct_caret_sides() -> Result<()> { + flowstate_fidelity::set_enabled(true); + let _ = flowstate_fidelity::take_violations(); + let source = flowstate_document::document_from_input_blocks( + flowstate_document::flowstate_document_theme(), + vec![ + flowstate_document::InputBlock::Paragraph(input_paragraph("before")), + flowstate_document::InputBlock::Image(flowstate_document::InputImageBlock { + asset_id: flowstate_document::AssetId(1), + alt_text: "img".to_string(), + caption: None, + sizing: flowstate_document::InputImageSizing::Intrinsic, + alignment: flowstate_document::InputBlockAlignment::Left, + external_url: None, + }), + flowstate_document::InputBlock::Paragraph(input_paragraph("after")), + ], + ); + let doc = flowstate_document::document_to_loro(&source, "Object sides")?; + let runtime = CrdtRuntime::from_doc(doc, None, None)?; + + let index = &runtime.projection_index; + assert_eq!(index.object_placeholder_positions.len(), 1, "exactly one block-object placeholder"); + let position = index.object_placeholder_positions[0]; + let before = index + .offset_for_body_unicode(&runtime.projection, position) + .expect("object slot resolves"); + let after = index + .offset_for_body_unicode(&runtime.projection, position + 1) + .expect("post-object slot resolves"); + assert_ne!(before, after, "the two sides of a block object must resolve to distinct offsets (FS-170)"); + // "after object" belongs to the following paragraph's start; "before" to the + // preceding paragraph's end. Paragraph indices skip the object block, so the + // "after" paragraph is index 1. + assert_eq!(after.paragraph, 1, "the after-object caret is the following paragraph"); + assert_eq!(after.byte, 0, "at the following paragraph's start"); + assert_eq!(before.paragraph, 0, "the before-object caret is the preceding paragraph"); + + let violations = flowstate_fidelity::take_violations(); + flowstate_fidelity::set_enabled(false); + assert!( + !violations + .iter() + .any(|violation| violation.contains("object-side-collapse")), + "no object-side-collapse violation must fire: {violations:?}" + ); + Ok(()) + } + + #[test] + fn undo_manager_restores_selection_metadata() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + let selection = UndoSelectionSnapshot { + anchor_cursor: vec![1, 2, 3], + head_cursor: vec![4, 5, 6], + anchor_affinity: UndoSelectionAffinity::Before, + head_affinity: UndoSelectionAffinity::After, + direction: UndoSelectionDirection::Forward, + }; + + runtime.set_pending_undo_selection(Some(selection.clone()))?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "abc".to_string(), + styles: RunStyles::default(), + })?; + runtime.command(SemanticCommand::Undo)?; + + assert_eq!(runtime.take_restored_undo_selection(), Some(selection.clone())); + runtime.command(SemanticCommand::Redo)?; + assert_eq!(runtime.take_restored_undo_selection(), Some(selection)); + Ok(()) + } + + #[test] + fn semantic_object_commands_project_structured_blocks() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertImage { + unicode_index: 1, + asset_id: 7, + alt_text: "alt".to_string(), + caption: Some("caption".to_string()), + sizing: flowstate_document::InputImageSizing::Fixed { + width_px: 320, + height_px: Some(180), + }, + alignment: flowstate_document::InputBlockAlignment::Center, + })?; + runtime.command(SemanticCommand::InsertEquation { + unicode_index: 2, + source: "x^2".to_string(), + display: flowstate_document::InputEquationDisplay::InlineLikeParagraph, + })?; + runtime.command(SemanticCommand::InsertTable { + unicode_index: 3, + rows: 2, + columns: 2, + column_widths: vec![ + flowstate_document::InputTableColumnWidth::FixedPx(120), + flowstate_document::InputTableColumnWidth::Fraction(1), + ], + header_row: true, + })?; + + let projection = runtime.projection_snapshot()?; + assert!(matches!( + &projection.blocks[0], + flowstate_document::Block::Image(image) + if image.asset_id == flowstate_document::AssetId(7) + && image.alt_text.as_ref() == "alt" + && image.caption.is_some() + )); + assert!(matches!( + &projection.blocks[1], + flowstate_document::Block::Equation(equation) + if equation.source.as_ref() == "x^2" + && equation.display == flowstate_document::EquationDisplay::InlineLikeParagraph + )); + assert!(matches!( + &projection.blocks[2], + flowstate_document::Block::Table(table) + if table.rows.len() == 2 + && table.rows[0].cells.len() == 2 + && table.style.header_row + && matches!(table.columns.as_slice(), [ + flowstate_document::TableColumn { width: flowstate_document::TableColumnWidth::FixedPx(120), .. }, + flowstate_document::TableColumn { width: flowstate_document::TableColumnWidth::Fraction(1), .. } + ]) + )); + Ok(()) + } + + #[test] + fn runtime_opens_and_forks_named_revisions() -> Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("revisions.db8"); + let doc = flowstate_document::new_loro_document("Runtime")?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Runtime")?; + let blank_revision = package.create_named_revision(&doc, "Blank", "Blank document", None, None)?; + body_text(&doc).insert(1, "latest")?; + doc.commit(); + package.compact_to_named_snapshot(&doc, "Latest", "Latest document", None, None)?; + package.write(&path)?; + + let mut runtime = CrdtRuntime::open_package(&path)?; + let opened = runtime.command(SemanticCommand::OpenRevision { revision_id: blank_revision })?; + assert!(matches!( + opened.as_slice(), + [RuntimeEvent::RevisionOpened { document, .. }] if document.paragraphs.first().is_some_and(|paragraph| paragraph.runs.is_empty()) + )); + + let forked = runtime.command(SemanticCommand::ForkRevision { revision_id: blank_revision })?; + let [RuntimeEvent::RevisionForked { document, package, .. }] = forked.as_slice() else { + panic!("expected fork event"); + }; + assert_eq!( + document + .paragraphs + .first() + .map(|paragraph| paragraph.runs.is_empty()), + Some(true) + ); + assert!(!package.loro_snapshots.is_empty()); + Ok(()) + } + + #[test] + fn remote_text_insert_emits_incremental_paragraph_patch() -> Result<()> { + let base = flowstate_document::new_loro_document("Shared")?; + let mut source = CrdtRuntime::from_doc(base.fork(), None, None)?; + let mut target = CrdtRuntime::from_doc(base, None, None)?; + let setup = source.export_updates_for(&target.doc().oplog_vv())?; + target.import_remote_update(&setup)?; + let update = local_update_bytes(&source.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?); + + let events = target.import_remote_update(&update)?; + let patches = events + .iter() + .find_map(|event| match event { + RuntimeEvent::ProjectionPatched { batch, .. } => Some(&batch.patches), + RuntimeEvent::LocalUpdate { .. } + | RuntimeEvent::RemoteUpdateApplied { .. } + | RuntimeEvent::RevisionOpened { .. } + | RuntimeEvent::RevisionForked { .. } + | RuntimeEvent::ProjectionUpdated { .. } + | RuntimeEvent::HistoryRebaseRequired { .. } + | RuntimeEvent::SelectionRestored { .. } => None, + }) + .expect("remote import should emit a projection patch"); + + let [ + ProjectionPatch::ParagraphText { + row_hint, new, delta_utf8, .. + }, + ] = patches.as_ref() + else { + panic!("expected one paragraph text patch"); + }; + assert_eq!(*row_hint, 0); + assert_eq!( + new + .runs + .iter() + .map(|run| run.text.as_str()) + .collect::(), + "hello" + ); + assert_eq!(delta_utf8, &[ProjectionTextDelta::Insert("hello".len())]); + assert!(events.iter().all(|event| !matches!( + event, + RuntimeEvent::ProjectionUpdated { + invalidation: ProjectionInvalidation { + fallback_reason: Some("remote_update_projection_fallback"), + .. + }, + .. + } + ))); + Ok(()) + } + + #[test] + fn remote_text_insert_in_object_document_still_emits_incremental_patch() -> Result<()> { + let source_projection = flowstate_document::document_from_input_blocks( + flowstate_document::DocumentTheme::default(), + vec![ + InputBlock::Paragraph(input_paragraph("alpha")), + InputBlock::Image(flowstate_document::InputImageBlock { + asset_id: flowstate_document::AssetId(7), + alt_text: "figure".to_string(), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Center, + external_url: None, + }), + InputBlock::Paragraph(input_paragraph("omega")), + ], + ); + let base = flowstate_document::document_to_loro(&source_projection, "Mixed")?; + let mut source = CrdtRuntime::from_doc(base.fork(), None, None)?; + let mut target = CrdtRuntime::from_doc(base, None, None)?; + let setup = source.export_updates_for(&target.doc().oplog_vv())?; + target.import_remote_update(&setup)?; + let unicode_index = body_text(source.doc()).len_unicode(); + let update = local_update_bytes(&source.command(SemanticCommand::InsertText { + unicode_index, + text: "!".to_string(), + styles: RunStyles::default(), + })?); + + let events = target.import_remote_update(&update)?; + let patches = events + .iter() + .find_map(|event| match event { + RuntimeEvent::ProjectionPatched { batch, .. } => Some(&batch.patches), + RuntimeEvent::LocalUpdate { .. } + | RuntimeEvent::RemoteUpdateApplied { .. } + | RuntimeEvent::RevisionOpened { .. } + | RuntimeEvent::RevisionForked { .. } + | RuntimeEvent::ProjectionUpdated { .. } + | RuntimeEvent::HistoryRebaseRequired { .. } + | RuntimeEvent::SelectionRestored { .. } => None, + }) + .expect("remote import should emit a projection patch"); + + let [ + ProjectionPatch::ParagraphText { + row_hint, new, delta_utf8, .. + }, + ] = patches.as_ref() + else { + panic!("expected one paragraph text patch"); + }; + assert_eq!(*row_hint, 2); + assert_eq!( + new + .runs + .iter() + .map(|run| run.text.as_str()) + .collect::(), + "omega!" + ); + assert_eq!( + delta_utf8, + &[ProjectionTextDelta::Retain("omega".len()), ProjectionTextDelta::Insert("!".len())] + ); + assert!( + events + .iter() + .all(|event| !matches!(event, RuntimeEvent::ProjectionUpdated { .. })) + ); + Ok(()) + } + + #[test] + fn remote_import_reports_pending_dependencies() -> Result<()> { + let source = flowstate_document::new_loro_document("Source")?; + let empty_vv = VersionVector::default(); + body_text(&source).insert(1, "first")?; + source.commit(); + let mid_vv = source.state_vv(); + body_text(&source).insert(6, " second")?; + source.commit(); + let second_only = source.export(ExportMode::updates(&mid_vv))?; + + let mut target = CrdtRuntime::new_empty("Target")?; + let events = target.import_remote_update(&second_only)?; + assert!(matches!(events.first(), Some(RuntimeEvent::RemoteUpdateApplied { pending: Some(_), .. }))); + + let first_update = source.export(ExportMode::updates(&empty_vv))?; + let events = target.import_remote_update(&first_update)?; + assert!(matches!(events.first(), Some(RuntimeEvent::RemoteUpdateApplied { pending: None, .. }))); + Ok(()) + } + #[test] + fn projected_document_runtime_starts_without_eager_package_or_reprojection() -> Result<()> { + let mut source = flowstate_document::document_from_input_blocks( + flowstate_document::DocumentTheme::default(), + vec![InputBlock::Paragraph(input_paragraph("imported"))], + ); + source.theme.zoom_factor = 1.75; + let runtime = CrdtRuntime::from_document_projection(&source, "Imported")?; + + assert!(runtime.package.is_none()); + assert_eq!(runtime.projection.frontier, runtime.doc.state_frontiers().encode()); + assert_eq!(runtime.projection.theme.zoom_factor, 1.75); + assert_eq!(flowstate_document::paragraph_text(&runtime.projection, 0), "imported"); + Ok(()) + } + + #[test] + fn projection_index_materializes_paragraph_and_block_anchor_lookups() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?; + runtime.command(SemanticCommand::SplitParagraph { + unicode_index: 3, + inherited_style: ParagraphStyle::Normal, + })?; + let projection = runtime.projection_snapshot()?; + let index = ProjectionRuntimeIndex::from_projection(&projection); + + // §24 paragraph metadata index agrees with the linear `paragraph_ids` scan. + for (expected_ix, id) in projection.ids.paragraph_ids.iter().enumerate() { + assert_eq!(index.paragraph_metadata_by_id.get(id).copied(), Some(expected_ix)); + assert_eq!( + index.paragraph_index(*id), + projection + .ids + .paragraph_ids + .iter() + .position(|candidate| candidate == id), + ); + } + // §24 block anchor index agrees with the linear `block_ids` scan. + for (expected_ix, id) in projection.ids.block_ids.iter().enumerate() { + assert_eq!(index.block_anchor_by_id.get(id).copied(), Some(expected_ix)); + assert_eq!( + index.block_index(*id), + projection + .ids + .block_ids + .iter() + .position(|candidate| candidate == id), + ); + } + // Absent ids fall back to `None` through both the index and the scan. + assert_eq!(index.paragraph_index(flowstate_document::new_paragraph_id()), None); + assert_eq!(index.block_index(flowstate_document::new_block_id()), None); + + // One search-unit span per block; one style-interval list per paragraph. + assert_eq!(index.search_unit_spans.len(), projection.blocks.len()); + assert_eq!(index.style_runs_by_paragraph.len(), projection.paragraphs.len()); + Ok(()) + } + + #[test] + fn projection_index_maps_changed_ranges_to_search_units() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?; + runtime.command(SemanticCommand::SplitParagraph { + unicode_index: 3, + inherited_style: ParagraphStyle::Normal, + })?; + let projection = runtime.projection_snapshot()?; + let index = ProjectionRuntimeIndex::from_projection(&projection); + + // Body text is "\nhe\nllo": unit 0 covers [1,3), unit 1 covers [4,7). + let first = index.search_units_for_changed_ranges(&[ProjectionTextRange { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + unicode_start: 1, + unicode_len: 1, + }]); + assert_eq!(first, vec![0]); + let second = index.search_units_for_changed_ranges(&[ProjectionTextRange { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + unicode_start: 4, + unicode_len: 1, + }]); + assert_eq!(second, vec![1]); + // A non-body flow never matches a body search-unit span. + assert!( + index + .search_units_for_changed_ranges(&[ProjectionTextRange { + flow_id: "flow.other".to_string(), + unicode_start: 1, + unicode_len: 5, + }]) + .is_empty() + ); + Ok(()) + } + + #[test] + fn projection_index_style_intervals_resolve_run_styles() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + let styles = RunStyles { + semantic: RunSemanticStyle::Custom(2), + ..RunStyles::default() + }; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "styled".to_string(), + styles, + })?; + let projection = runtime.projection_snapshot()?; + let index = ProjectionRuntimeIndex::from_projection(&projection); + + // The style interval covering the paragraph's leading byte carries its run styles. + assert_eq!(index.run_styles_at(0, 0), Some(styles)); + assert_eq!(index.run_styles_at(0, 0), projection.paragraphs[0].runs.first().map(|run| run.styles),); + // Bytes beyond the paragraph text have no covering interval. + assert_eq!(index.run_styles_at(0, 1_000), None); + Ok(()) + } + + #[test] + fn projection_index_indexes_image_asset_refs_and_table_cells() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertImage { + unicode_index: 1, + asset_id: 7, + alt_text: "alt".to_string(), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Center, + })?; + runtime.command(SemanticCommand::InsertTable { + unicode_index: 2, + rows: 2, + columns: 2, + column_widths: vec![InputTableColumnWidth::FixedPx(120), InputTableColumnWidth::Fraction(1)], + header_row: true, + })?; + let projection = runtime.projection_snapshot()?; + let index = ProjectionRuntimeIndex::from_projection(&projection); + + // §24 asset reference index: the image block id is recorded under its asset. + let image_block_ix = projection + .blocks + .iter() + .position(|block| matches!(block, Block::Image(_))) + .expect("image block present"); + let image_block_id = projection.ids.block_ids[image_block_ix]; + let asset_refs = index + .asset_refs_by_id + .get(&AssetId(7)) + .expect("asset reference indexed"); + assert_eq!(asset_refs.len(), 1); + assert!(asset_refs.contains(&image_block_id)); + + // §24 table row/column/cell index: 2x2 table → 2 rows, 2 columns, 4 cells. + let table_block_ix = projection + .blocks + .iter() + .position(|block| matches!(block, Block::Table(_))) + .expect("table block present"); + let table_block_id = projection.ids.block_ids[table_block_ix]; + let entry = index + .table_cells_by_block + .get(&table_block_id) + .expect("table indexed"); + assert_eq!(entry.row_ids.len(), 2); + assert_eq!(entry.column_ids.len(), 2); + assert_eq!(entry.cells.len(), 4); + // §P2b: the index is keyed by the model's durable ids, so every projected + // cell's `(row_id, column_id)` coordinate resolves to its deterministic + // `CellId` (the fresh table minted uuid row/column ids, so probe by them). + let Block::Table(table) = &projection.blocks[table_block_ix] else { + panic!("expected table block"); + }; + for row in &table.rows { + for cell in &row.cells { + assert_eq!(entry.cells.get(&(cell.row_id, cell.column_id)), Some(&cell.id)); + } + } + + // §24 search unit index: every object block contributes a paragraph-less span. + let object_blocks = projection + .blocks + .iter() + .filter(|block| !matches!(block, Block::Paragraph(_))) + .count(); + assert_eq!( + index + .search_unit_spans + .iter() + .filter(|span| span.paragraph.is_none()) + .count(), + object_blocks, + ); + Ok(()) + } + + #[test] + fn projection_index_cursor_cache_clears_on_incremental_update() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Runtime")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?; + // Seed a stand-in cache entry, then drive a further incremental edit. Positions + // can shift on any edit, so the cache must be cleared (rebuilt empty) afterward. + runtime + .projection_index + .cursor_resolution_cache + .insert(vec![0xde, 0xad], DocumentOffset::default()); + assert!(!runtime.projection_index.cursor_resolution_cache.is_empty()); + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "x".to_string(), + styles: RunStyles::default(), + })?; + assert!(runtime.projection_index.cursor_resolution_cache.is_empty()); + Ok(()) + } + + #[test] + fn presence_after_affinity_sticks_to_following_text_across_inserts() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Presence")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "ab".to_string(), + styles: RunStyles::default(), + })?; + let offset = DocumentOffset { paragraph: 0, byte: 1 }; + let selection = EditorSelection::collapsed_with(offset, gpui_flowtext::SelectionAffinity::After, gpui_flowtext::VisualGravity::Downstream); + let presence = runtime + .presence_selection(&selection) + .expect("presence selection should encode"); + + runtime.command(SemanticCommand::InsertText { + unicode_index: 2, + text: "X".to_string(), + styles: RunStyles::default(), + })?; + let carets = runtime.resolve_presence_carets(vec![RuntimePresenceCaretRequest { + selection: presence, + color_rgb: 0xabcdef, + }]); + + assert_eq!(carets.carets.len(), 1); + assert_eq!(carets.carets[0].offset, DocumentOffset { paragraph: 0, byte: 2 }); + assert_eq!(carets.carets[0].visual_gravity, gpui_flowtext::VisualGravity::Downstream); + Ok(()) + } + + #[test] + fn presence_before_affinity_sticks_to_preceding_text_across_inserts() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Presence")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "ab".to_string(), + styles: RunStyles::default(), + })?; + let offset = DocumentOffset { paragraph: 0, byte: 1 }; + let selection = EditorSelection::collapsed_with(offset, gpui_flowtext::SelectionAffinity::Before, gpui_flowtext::VisualGravity::Upstream); + let presence = runtime + .presence_selection(&selection) + .expect("presence selection should encode"); + + runtime.command(SemanticCommand::InsertText { + unicode_index: 2, + text: "X".to_string(), + styles: RunStyles::default(), + })?; + let carets = runtime.resolve_presence_carets(vec![RuntimePresenceCaretRequest { + selection: presence, + color_rgb: 0xabcdef, + }]); + + assert_eq!(carets.carets.len(), 1); + assert_eq!(carets.carets[0].offset, offset); + assert_eq!(carets.carets[0].visual_gravity, gpui_flowtext::VisualGravity::Upstream); + Ok(()) + } + + #[test] + fn presence_non_collapsed_selection_resolves_to_a_colored_span() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Presence")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "abcd".to_string(), + styles: RunStyles::default(), + })?; + let anchor = DocumentOffset { paragraph: 0, byte: 1 }; + let head = DocumentOffset { paragraph: 0, byte: 3 }; + let selection = EditorSelection { + anchor, + head, + ..EditorSelection::default() + }; + let presence = runtime + .presence_selection(&selection) + .expect("presence selection should encode"); + + let resolved = runtime.resolve_presence_carets(vec![RuntimePresenceCaretRequest { + selection: presence, + color_rgb: 0xabcdef, + }]); + + // The head still resolves to a caret, and the non-collapsed range resolves + // to a single colored selection span carrying that peer's color. + assert_eq!(resolved.carets.len(), 1); + assert_eq!(resolved.carets[0].offset, head); + assert_eq!(resolved.selections.len(), 1); + assert_eq!(resolved.selections[0].color_rgb, 0xabcdef); + assert_eq!(resolved.selections[0].selection.anchor, anchor); + assert_eq!(resolved.selections[0].selection.head, head); + Ok(()) + } + + #[test] + fn presence_collapsed_selection_yields_no_span() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Presence")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "abcd".to_string(), + styles: RunStyles::default(), + })?; + let offset = DocumentOffset { paragraph: 0, byte: 2 }; + let selection = EditorSelection::collapsed(offset); + let presence = runtime + .presence_selection(&selection) + .expect("presence selection should encode"); + + let resolved = runtime.resolve_presence_carets(vec![RuntimePresenceCaretRequest { + selection: presence, + color_rgb: 0xabcdef, + }]); + + // A bare caret paints only a caret — never a zero-width highlight span. + assert_eq!(resolved.carets.len(), 1); + assert!(resolved.selections.is_empty()); + Ok(()) + } + + /// §P3 (act two): the close-time cache flush restores the preview fast + /// path — an edit nulls the projection cache (preview falls back to a full + /// Loro materialization), and `begin_cache_flush` + `CheckpointJob::run` + /// writes a frontier-current cache back without minting a revision. + #[test] + fn cache_flush_restores_the_preview_fast_path() -> anyhow::Result<()> { + use crate::local_write::{GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, TextAnchor}; + + let dir = tempfile::tempdir()?; + let path = dir.path().join("flush.db8"); + let document = flowstate_document::document_from_input_blocks( + flowstate_document::DocumentTheme::default(), + vec![flowstate_document::InputBlock::Paragraph(flowstate_document::InputParagraph { + style: flowstate_document::ParagraphStyle::Normal, + runs: vec![flowstate_document::InputRun { + text: "flush me".to_string(), + styles: flowstate_document::RunStyles::default(), + }], + })], + ); + let imported = flowstate_document::import_document_projection(document, "Flush")?; + let bytes = assemble_package_bytes(&imported.doc, &imported.projection, "Flush")?; + std::fs::write(&path, bytes)?; + assert!( + DocumentPackage::read_cached_projection(&path)?.is_some(), + "a freshly assembled package must carry a frontier-current projection cache" + ); + + let runtime = CrdtRuntime::open_package(&path)?; + let (handle, gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: " again".to_string(), + style_override: None, + }) + .expect("insert"); + let revisions_before = DocumentPackage::read(&path)?.revisions.len(); + let mut guard = gate.lock(GateHolder::DocumentService).expect("gate"); + assert!( + DocumentPackage::read_cached_projection(&path)?.is_none(), + "an edit must null the projection cache (preview falls back to the full read)" + ); + let job = guard + .begin_cache_flush() + .expect("edited package must need a cache flush"); + drop(guard); + let (package, wrote) = job.run(); + let wrote = wrote?; + let mut guard = gate.lock(GateHolder::DocumentService).expect("gate"); + guard.finish_checkpoint(package, wrote); + drop(guard); + assert!(wrote, "the flush must reach disk"); + let flushed = DocumentPackage::read_cached_projection(&path)?.expect("flushed package must preview via the cache fast path"); + assert_eq!(flowstate_document::paragraph_text(&flushed, 0), "flush me again"); + let revisions_after = DocumentPackage::read(&path)?.revisions.len(); + assert_eq!(revisions_before, revisions_after, "a cache flush must not mint a revision"); + let mut guard = gate.lock(GateHolder::DocumentService).expect("gate"); + assert!(guard.begin_cache_flush().is_none(), "a frontier-current package must skip the flush"); + drop(guard); + Ok(()) + } +} + +#[cfg(test)] +#[path = "crdt_runtime/projection_repair_tests.rs"] +mod projection_repair_tests; diff --git a/crates/flowstate-collab/src/crdt_runtime/import_delta.rs b/crates/flowstate-collab/src/crdt_runtime/import_delta.rs new file mode 100644 index 00000000..1a3eefab --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime/import_delta.rs @@ -0,0 +1,325 @@ +//! Exact remote-import body-delta composition (Loro-first spec §6.3-6.4). +//! +//! One remote import can apply several Loro change batches, each emitting a +//! subscription event whose text diff is positioned against the doc state at +//! that batch's application point. Treating those per-batch positions as if +//! they shared one coordinate space is how the old runtime's lossy summaries +//! missed structural deletes and grew an O(doc) body-stringify backstop. +//! +//! This module composes the per-batch deltas into ONE net delta +//! (pre-import space → post-import space), from which the import path derives +//! exact answers in O(P + ops): +//! * which pre-import positions were deleted (→ structural-delete detection by +//! binary search against the maintained boundary/object indexes); +//! * whether any inserted text carries structure (`\n`/U+FFFC); +//! * where every pre-import paragraph start lands post-import (position +//! shifting — replaces the full-body rescan); +//! * the changed post-import ranges (invalidation + touched paragraphs). +//! +//! Copied-out data only: everything here operates on plain ops harvested from +//! the subscription callback; nothing touches the doc (spec I-9b). + +/// One net-delta op over the body flow (unicode-codepoint units). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum NetOp { + Retain(usize), + Insert { len: usize, structure: bool }, + Delete(usize), +} + +/// The composed pre→post delta for one import, plus conservatism flags. +#[derive(Clone, Debug, Default)] +pub(crate) struct NetBodyDelta { + pub ops: Vec, + /// A structural insert was later (partially) deleted within the same + /// import. The net delta can no longer prove structure-neutrality, so the + /// caller must take the full-rebuild path. Rare; correctness over speed. + pub structural_churn: bool, +} + +impl NetBodyDelta { + /// No content change (pure retains) — the import was projection-neutral. + pub(crate) fn is_empty(&self) -> bool { + self.ops.iter().all(|op| matches!(op, NetOp::Retain(_))) + } + + /// Any inserted run in the net delta carries a paragraph boundary or object + /// placeholder. + pub(crate) fn inserts_structure(&self) -> bool { + self + .ops + .iter() + .any(|op| matches!(op, NetOp::Insert { structure: true, .. })) + } + + /// Deleted ranges in PRE-import coordinates, merged and ordered. + pub(crate) fn deleted_pre_ranges(&self) -> Vec<(usize, usize)> { + let mut ranges = Vec::new(); + let mut old_pos = 0usize; + for op in &self.ops { + match op { + NetOp::Retain(n) => old_pos += n, + NetOp::Insert { .. } => {}, + NetOp::Delete(n) => { + ranges.push((old_pos, *n)); + old_pos += n; + }, + } + } + ranges + } + + /// Spec §6-R: the PRE-import unicode span this net delta touches — `(lo, hi)` + /// covering every deleted range and every insertion point. `None` when the + /// delta is retain-only. + pub(crate) fn pre_change_span(&self) -> Option<(usize, usize)> { + let mut old_pos = 0_usize; + let mut lo = None; + let mut hi: Option = None; + for op in &self.ops { + match op { + NetOp::Retain(n) => old_pos += n, + NetOp::Insert { .. } => { + lo.get_or_insert(old_pos); + hi = Some(hi.map_or(old_pos, |current| current.max(old_pos))); + }, + NetOp::Delete(n) => { + lo.get_or_insert(old_pos); + let end = old_pos + n; + hi = Some(hi.map_or(end, |current| current.max(end))); + old_pos = end; + }, + } + } + Some((lo?, hi?)) + } + + /// True when any deleted pre-import range covers one of `positions` + /// (sorted ascending). O(ranges · log positions). + pub(crate) fn deletes_any_position(&self, positions: &[usize]) -> bool { + self.deleted_pre_ranges().iter().any(|(start, len)| { + if *len == 0 { + return false; + } + let from = positions.partition_point(|p| p < start); + positions.get(from).is_some_and(|p| *p < start + len) + }) + } + + /// Map sorted pre-import positions into post-import coordinates. Positions + /// inside a deleted range clamp to the deletion point (callers only shift + /// positions they have already proven undeleted). + pub(crate) fn shift_positions(&self, positions: &[usize]) -> Vec { + let mut shifted = Vec::with_capacity(positions.len()); + let mut op_ix = 0usize; + let mut old_pos = 0usize; + let mut new_pos = 0usize; + for &position in positions { + // Advance through ops until `position` falls before/inside the current op. + loop { + match self.ops.get(op_ix) { + Some(NetOp::Retain(n)) => { + if position < old_pos + n { + break; + } + old_pos += n; + new_pos += n; + op_ix += 1; + }, + Some(NetOp::Insert { len, .. }) => { + new_pos += len; + op_ix += 1; + }, + Some(NetOp::Delete(n)) => { + if position < old_pos + n { + break; + } + old_pos += n; + op_ix += 1; + }, + None => break, + } + } + match self.ops.get(op_ix) { + Some(NetOp::Retain(_)) | None => shifted.push(new_pos + (position - old_pos)), + Some(NetOp::Delete(_)) => shifted.push(new_pos), + Some(NetOp::Insert { .. }) => unreachable!("insert ops are always consumed in the advance loop"), + } + } + shifted + } +} + +/// Compose a sequence of per-batch op lists (each old→new for its own batch) +/// into one net delta. Missing tails are implicit retains. +pub(crate) fn compose_batches(batches: &[Vec]) -> NetBodyDelta { + let mut net = NetBodyDelta::default(); + for batch in batches { + let (ops, churn) = compose(&net.ops, batch); + net.ops = ops; + net.structural_churn |= churn; + } + normalize(&mut net.ops); + net +} + +/// Standard delta composition: `a` maps old→mid, `b` maps mid→new; the result +/// maps old→new. Returns `(ops, structural_churn)` where churn = `b` deleted +/// (part of) a structural insert from `a`. +fn compose(a: &[NetOp], b: &[NetOp]) -> (Vec, bool) { + let mut out: Vec = Vec::with_capacity(a.len() + b.len()); + let mut churn = false; + let mut a_iter = a.iter().copied(); + let mut b_iter = b.iter().copied(); + let mut a_head: Option = a_iter.next(); + let mut b_head: Option = b_iter.next(); + + loop { + match (a_head, b_head) { + (_, Some(NetOp::Insert { len, structure })) => { + // b inserts exist only in the new space — emit directly. + out.push(NetOp::Insert { len, structure }); + b_head = b_iter.next(); + }, + (Some(NetOp::Delete(n)), _) => { + // a's deletes consumed old text b never saw — emit directly. + out.push(NetOp::Delete(n)); + a_head = a_iter.next(); + }, + (None, None) => break, + (None, Some(op)) => { + // Implicit retain tail on a: b's op applies to untouched old text. + out.push(op); + b_head = b_iter.next(); + }, + (Some(op), None) => { + // Implicit retain tail on b: a's op passes through. + out.push(op); + a_head = a_iter.next(); + }, + (Some(a_op), Some(b_op)) => { + let a_len = match a_op { + NetOp::Retain(n) => n, + NetOp::Insert { len, .. } => len, + NetOp::Delete(_) => unreachable!("deletes handled above"), + }; + let b_len = match b_op { + NetOp::Retain(n) => n, + NetOp::Delete(n) => n, + NetOp::Insert { .. } => unreachable!("inserts handled above"), + }; + let step = a_len.min(b_len); + match (a_op, b_op) { + (NetOp::Retain(_), NetOp::Retain(_)) => out.push(NetOp::Retain(step)), + (NetOp::Retain(_), NetOp::Delete(_)) => out.push(NetOp::Delete(step)), + (NetOp::Insert { structure, .. }, NetOp::Retain(_)) => out.push(NetOp::Insert { len: step, structure }), + (NetOp::Insert { structure, .. }, NetOp::Delete(_)) => { + // b deleted text a inserted within this import: it cancels out of + // the net delta entirely. If that text carried structure we can no + // longer prove neutrality (the flag is per-op, not per-char). + churn |= structure; + }, + _ => unreachable!(), + } + a_head = shrink(a_op, step).or_else(|| a_iter.next()); + b_head = shrink(b_op, step).or_else(|| b_iter.next()); + }, + } + } + (out, churn) +} + +fn shrink(op: NetOp, by: usize) -> Option { + let remaining = match op { + NetOp::Retain(n) | NetOp::Delete(n) => n - by, + NetOp::Insert { len, .. } => len - by, + }; + if remaining == 0 { + return None; + } + Some(match op { + NetOp::Retain(_) => NetOp::Retain(remaining), + NetOp::Delete(_) => NetOp::Delete(remaining), + NetOp::Insert { structure, .. } => NetOp::Insert { len: remaining, structure }, + }) +} + +/// Merge adjacent same-kind ops and drop zero-length ones. +fn normalize(ops: &mut Vec) { + let mut merged: Vec = Vec::with_capacity(ops.len()); + for op in ops.drain(..) { + let zero = matches!(op, NetOp::Retain(0) | NetOp::Delete(0) | NetOp::Insert { len: 0, .. }); + if zero { + continue; + } + match (merged.last_mut(), op) { + (Some(NetOp::Retain(a)), NetOp::Retain(b)) => *a += b, + (Some(NetOp::Delete(a)), NetOp::Delete(b)) => *a += b, + (Some(NetOp::Insert { len: a, structure: sa }), NetOp::Insert { len: b, structure: sb }) => { + *a += b; + *sa |= sb; + }, + (_, op) => merged.push(op), + } + } + *ops = merged; +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ins(len: usize) -> NetOp { + NetOp::Insert { len, structure: false } + } + + fn ins_s(len: usize) -> NetOp { + NetOp::Insert { len, structure: true } + } + + #[test] + fn composes_insert_then_delete_of_other_text() { + // batch1: insert 3 at 5. batch2: delete 2 at 0 (old text). + let net = compose_batches(&[vec![NetOp::Retain(5), ins(3)], vec![NetOp::Delete(2)]]); + assert_eq!(net.ops, vec![NetOp::Delete(2), NetOp::Retain(3), ins(3)]); + assert_eq!(net.deleted_pre_ranges(), vec![(0, 2)]); + assert!(!net.inserts_structure()); + assert!(!net.structural_churn); + } + + #[test] + fn delete_of_own_structural_insert_flags_churn() { + // batch1 inserts a paragraph boundary; batch2 deletes it again. + let net = compose_batches(&[vec![NetOp::Retain(4), ins_s(1)], vec![NetOp::Retain(4), NetOp::Delete(1)]]); + assert!(net.structural_churn, "cancelled structural insert must force conservatism"); + assert!(net.is_empty(), "net content change cancels out"); + } + + #[test] + fn detects_deleted_boundary_positions() { + // Boundaries at 0, 6, 12. One delete of [5, 8). + let net = compose_batches(&[vec![NetOp::Retain(5), NetOp::Delete(3)]]); + assert!(net.deletes_any_position(&[0, 6, 12])); + assert!(!net.deletes_any_position(&[0, 12])); + } + + #[test] + fn shifts_positions_through_inserts_and_deletes() { + // delete [2,4), insert 5 at old-10. + let net = compose_batches(&[vec![NetOp::Retain(2), NetOp::Delete(2), NetOp::Retain(6), ins(5)]]); + // old positions: 0 1 5 9 10 20 + // post positions: 0 1 3 7 8+5=13? position 10 is AT the insert point: + // advance consumes retain(6) ending at old 10, then hits Insert → new_pos += 5, + // then implicit tail retain maps 10 → 2+6+5 = 13. old 20 → 23. + assert_eq!(net.shift_positions(&[0, 1, 5, 9, 10, 20]), vec![0, 1, 3, 7, 13, 23]); + } + + #[test] + fn multi_batch_positions_compose_exactly() { + // batch1: insert 4 at 0. batch2 positions are mid-space: delete [2,6) spans + // half the fresh insert and 2 old chars. + let net = compose_batches(&[vec![ins(4)], vec![NetOp::Retain(2), NetOp::Delete(4)]]); + assert_eq!(net.ops, vec![ins(2), NetOp::Delete(2)]); + assert_eq!(net.deleted_pre_ranges(), vec![(0, 2)]); + } +} diff --git a/crates/flowstate-collab/src/crdt_runtime/projection_patch.rs b/crates/flowstate-collab/src/crdt_runtime/projection_patch.rs new file mode 100644 index 00000000..5d505885 --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime/projection_patch.rs @@ -0,0 +1,973 @@ +use flowstate_document::{ + Block, DocumentProjection, HighlightStyle, InputBlock, InputParagraph, InputRun, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, + MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, OBJECT_REPLACEMENT, Paragraph, ParagraphStyle, ProjectionPatch, ProjectionStructuralBlock, + ProjectionTextDelta, RunSemanticStyle, RunStyles, input_block_from_block, loro_schema::body_text, new_block_id, new_paragraph_id, + paragraph_text, +}; +use loro::{LoroDoc, LoroValue}; +use rustc_hash::{FxHashMap, FxHashSet}; + +use super::{ProjectionInvalidation, paragraph_style_from_attrs}; + +pub(crate) fn projection_patches_between(before: &DocumentProjection, after: &DocumentProjection) -> Option> { + let mut patches = Vec::new(); + append_asset_patches(before, after, &mut patches); + + let before_ids = &before.ids.block_ids; + let after_ids = &after.ids.block_ids; + if before_ids == after_ids { + append_same_shape_patches(before, after, &mut patches)?; + return Some(patches); + } + + if let Some((from, to)) = single_block_move(before_ids, after_ids) { + if !retained_blocks_unchanged(before, after) { + return None; + } + patches.push(ProjectionPatch::MoveBlock { + block_id: before_ids[from], + before: after_ids.get(to + 1).copied(), + from_hint: from, + to_hint: to, + }); + return Some(patches); + } + + let prefix = common_id_prefix(before_ids, after_ids); + let suffix = common_id_suffix(before_ids, after_ids, prefix); + let before_end = before_ids.len().saturating_sub(suffix); + let after_end = after_ids.len().saturating_sub(suffix); + if before_end > prefix { + patches.push(ProjectionPatch::DeleteBlocks { + block_ids: before_ids[prefix..before_end].to_vec(), + row_hint: prefix, + }); + } + if after_end > prefix { + patches.push(ProjectionPatch::InsertBlocks { + before: after_ids.get(after_end).copied(), + row_hint: prefix, + blocks: structural_blocks(after, prefix..after_end), + }); + } + if !retained_edge_blocks_unchanged(before, after, prefix, suffix) { + return None; + } + Some(patches) +} + +fn append_same_shape_patches(before: &DocumentProjection, after: &DocumentProjection, patches: &mut Vec) -> Option<()> { + if before.blocks.len() != after.blocks.len() { + return None; + } + let mut before_paragraph_ix = 0usize; + let mut after_paragraph_ix = 0usize; + for (row, (before_block, after_block)) in before.blocks.iter().zip(after.blocks.iter()).enumerate() { + match (before_block, after_block) { + (Block::Paragraph(before_paragraph), Block::Paragraph(after_paragraph)) => { + append_paragraph_patch( + before, + after, + row, + before_paragraph_ix, + after_paragraph_ix, + before_paragraph, + after_paragraph, + patches, + ); + before_paragraph_ix += 1; + after_paragraph_ix += 1; + }, + (Block::Image(_) | Block::Equation(_) | Block::Table(_), Block::Image(_) | Block::Equation(_) | Block::Table(_)) => { + if before_block != after_block { + patches.push(ProjectionPatch::ReplaceObjectBlock { + block_id: before.ids.block_ids[row], + row_hint: row, + block: structural_block(after, row, after_paragraph_ix), + }); + } + }, + _ => return None, + } + } + Some(()) +} + +#[allow(clippy::too_many_arguments, reason = "paragraph diffs need both projection coordinates and payloads")] +fn append_paragraph_patch( + before: &DocumentProjection, + after: &DocumentProjection, + row: usize, + before_paragraph_ix: usize, + after_paragraph_ix: usize, + before_paragraph: &Paragraph, + after_paragraph: &Paragraph, + patches: &mut Vec, +) { + let before_text = paragraph_text(before, before_paragraph_ix); + let after_text = paragraph_text(after, after_paragraph_ix); + if before_text != after_text { + patches.push(ProjectionPatch::ParagraphText { + block_id: before.ids.block_ids[row], + paragraph_id: before.ids.paragraph_ids[before_paragraph_ix], + row_hint: row, + new: input_paragraph(after, after_paragraph_ix, after_paragraph), + delta_utf8: text_delta_between(&before_text, &after_text), + }); + return; + } + if before_paragraph.style != after_paragraph.style { + patches.push(ProjectionPatch::ParagraphStyle { + block_id: before.ids.block_ids[row], + paragraph_id: before.ids.paragraph_ids[before_paragraph_ix], + row_hint: row, + style: after_paragraph.style, + }); + } + if before_paragraph.runs != after_paragraph.runs { + patches.push(ProjectionPatch::ParagraphRuns { + block_id: before.ids.block_ids[row], + paragraph_id: before.ids.paragraph_ids[before_paragraph_ix], + row_hint: row, + runs: after_paragraph.runs.clone(), + }); + } +} + +fn input_paragraph(document: &DocumentProjection, paragraph_ix: usize, paragraph: &Paragraph) -> InputParagraph { + let text = paragraph_text(document, paragraph_ix); + let mut byte = 0usize; + let runs = paragraph + .runs + .iter() + .map(|run| { + let start = byte; + let end = start.saturating_add(run.len).min(text.len()); + byte = end; + InputRun { + text: text.get(start..end).unwrap_or_default().to_string(), + styles: run.styles, + } + }) + .collect(); + InputParagraph { + style: paragraph.style, + runs, + } +} + +fn structural_blocks(document: &DocumentProjection, range: std::ops::Range) -> Vec { + let mut paragraph_ix = document + .blocks + .iter() + .take(range.start) + .filter(|block| matches!(block, Block::Paragraph(_))) + .count(); + range + .map(|row| { + let structural = structural_block(document, row, paragraph_ix); + if matches!(document.blocks.get(row), Some(Block::Paragraph(_))) { + paragraph_ix += 1; + } + structural + }) + .collect() +} + +fn structural_block(document: &DocumentProjection, row: usize, paragraph_ix: usize) -> ProjectionStructuralBlock { + // A `Block::Paragraph` stores only run lengths — its text lives in the document + // Rope — so `input_block_from_block` (which has no document) would produce + // runs with EMPTY text and silently drop the paragraph's content (e.g. a + // split-then-insert batch inserting an empty paragraph instead of the typed + // text). Slice the real text via `input_paragraph`; objects carry their own + // payload and convert directly. + let block = match document.blocks.get(row) { + Some(Block::Paragraph(paragraph)) => InputBlock::Paragraph(input_paragraph(document, paragraph_ix, paragraph)), + Some(other) => input_block_from_block(other), + None => InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + }), + }; + ProjectionStructuralBlock { + block_id: document + .ids + .block_ids + .get(row) + .copied() + .unwrap_or_else(new_block_id), + paragraph_id: matches!(&block, InputBlock::Paragraph(_)).then(|| { + document + .ids + .paragraph_ids + .get(paragraph_ix) + .copied() + .unwrap_or_else(new_paragraph_id) + }), + block, + } +} + +fn append_asset_patches(before: &DocumentProjection, after: &DocumentProjection, patches: &mut Vec) { + for (id, record) in &after.assets.assets { + if before.assets.assets.get(id) != Some(record) { + patches.push(ProjectionPatch::AssetArrived { + id: *id, + record: record.clone(), + }); + } + } +} + +fn single_block_move(before: &[flowstate_document::BlockId], after: &[flowstate_document::BlockId]) -> Option<(usize, usize)> { + if before.len() != after.len() || before.len() < 2 { + return None; + } + + let mut before_positions = FxHashMap::default(); + for (index, id) in before.iter().copied().enumerate() { + if before_positions.insert(id, index).is_some() { + return None; + } + } + + let mut moved = None; + for (after_index, id) in after.iter().copied().enumerate() { + let before_index = *before_positions.get(&id)?; + if before_index != after_index { + match moved { + Some((existing_id, _, _)) if existing_id != id => return None, + Some(_) => {}, + None => moved = Some((id, before_index, after_index)), + } + } + } + + let (moved_id, from, _) = moved?; + let mut candidate = before.to_vec(); + candidate.remove(from); + let to = after.iter().position(|id| *id == moved_id)?; + candidate.insert(to, moved_id); + (candidate == after).then_some((from, to)) +} + +fn retained_blocks_unchanged(before: &DocumentProjection, after: &DocumentProjection) -> bool { + let mut before_blocks = FxHashMap::default(); + // §perf: rows are visited in ascending order, so carry a running paragraph counter + // instead of recomputing the paragraph index per row (see `input_block_at_seq`). + let mut before_paragraph_ix = 0; + for row in 0..before.ids.block_ids.len() { + let Some(id) = before.ids.block_ids.get(row).copied() else { + return false; + }; + let Some(block) = input_block_at_seq(before, row, &mut before_paragraph_ix) else { + return false; + }; + before_blocks.insert(id, block); + } + let mut after_paragraph_ix = 0; + for row in 0..after.ids.block_ids.len() { + let Some(id) = after.ids.block_ids.get(row).copied() else { + return false; + }; + let Some(before_block) = before_blocks.get(&id) else { + return false; + }; + let Some(after_block) = input_block_at_seq(after, row, &mut after_paragraph_ix) else { + return false; + }; + if before_block != &after_block { + return false; + } + } + true +} + +fn retained_edge_blocks_unchanged(before: &DocumentProjection, after: &DocumentProjection, prefix: usize, suffix: usize) -> bool { + let before_len = before.ids.block_ids.len(); + let after_len = after.ids.block_ids.len(); + // §perf: the prefix rows are contiguous from 0, so run two ascending paragraph + // counters (one per document) instead of an O(row) recount per row. + let mut before_paragraph_ix = 0; + let mut after_paragraph_ix = 0; + for row in 0..prefix { + if input_block_at_seq(before, row, &mut before_paragraph_ix) != input_block_at_seq(after, row, &mut after_paragraph_ix) { + return false; + } + } + // The suffix rows are a contiguous tail range in each document; seed each counter + // once (a single scan up to the suffix start) then advance it sequentially. + let before_suffix_start = before_len.saturating_sub(suffix); + let after_suffix_start = after_len.saturating_sub(suffix); + let mut before_paragraph_ix = count_paragraph_blocks_before(before, before_suffix_start); + let mut after_paragraph_ix = count_paragraph_blocks_before(after, after_suffix_start); + for offset in 0..suffix { + let before_row = before_suffix_start + offset; + let after_row = after_suffix_start + offset; + if input_block_at_seq(before, before_row, &mut before_paragraph_ix) != input_block_at_seq(after, after_row, &mut after_paragraph_ix) { + return false; + } + } + true +} + +// §perf: sequential variant of the former `input_block_at`. `paragraph_ix` must hold +// the number of Paragraph blocks in `document.blocks[0..row]`; callers visit rows in +// ascending order and advance it here, so building a block no longer recomputes that +// count with an O(row) scan. This turns the patch-diff verification +// (retained_blocks_unchanged / retained_edge_blocks_unchanged), which runs on every +// structural edit, from O(blocks²) into O(blocks). +fn input_block_at_seq(document: &DocumentProjection, row: usize, paragraph_ix: &mut usize) -> Option { + let block = document.blocks.get(row)?; + match block { + Block::Paragraph(paragraph) => { + let ix = *paragraph_ix; + *paragraph_ix += 1; + Some(InputBlock::Paragraph(input_paragraph(document, ix, paragraph))) + }, + _ => Some(input_block_from_block(block)), + } +} + +// Count of Paragraph blocks strictly before `row`; used once per document to seed the +// sequential counter for the suffix range. +fn count_paragraph_blocks_before(document: &DocumentProjection, row: usize) -> usize { + document + .blocks + .iter() + .take(row) + .filter(|block| matches!(block, Block::Paragraph(_))) + .count() +} + +fn common_id_prefix(left: &[flowstate_document::BlockId], right: &[flowstate_document::BlockId]) -> usize { + left + .iter() + .zip(right) + .take_while(|(left, right)| left == right) + .count() +} + +fn common_id_suffix(left: &[flowstate_document::BlockId], right: &[flowstate_document::BlockId], prefix: usize) -> usize { + left + .iter() + .skip(prefix) + .rev() + .zip(right.iter().skip(prefix).rev()) + .take_while(|(left, right)| left == right) + .count() +} + +pub(crate) fn text_delta_between(before: &str, after: &str) -> Vec { + let prefix = common_prefix_byte_len(before, after); + let suffix = common_suffix_byte_len(before, after, prefix); + text_delta( + prefix, + before.len().saturating_sub(prefix + suffix), + after.len().saturating_sub(prefix + suffix), + suffix, + ) +} + +/// Spec §6-R step 4: diff regionally rematerialized rows against the old +/// projection rows they replace (`old_lo..old_hi`, block-row range) and emit +/// the minimal patch set — content patches for retained rows, one +/// `DeleteBlocks` for dead rows, `InsertBlocks` runs anchored before their +/// following retained row. Bails (`None`) when retained rows changed relative +/// order (concurrent moves), a retained id changed kind or paragraph identity, +/// or the region produced duplicate ids — those take the rebuild. +pub(crate) fn splice_region_patches( + projection: &DocumentProjection, + old_lo: usize, + old_hi: usize, + region: &flowstate_document::RegionRows, +) -> Option> { + let old_ids = projection.ids.block_ids.get(old_lo..old_hi)?; + let new_ids = ®ion.block_ids; + if region.blocks.len() != new_ids.len() { + return None; + } + let new_set: FxHashSet = new_ids.iter().copied().collect(); + if new_set.len() != new_ids.len() { + return None; + } + let old_set: FxHashSet = old_ids.iter().copied().collect(); + let old_retained: Vec<_> = old_ids + .iter() + .copied() + .filter(|id| new_set.contains(id)) + .collect(); + let new_retained: Vec<_> = new_ids + .iter() + .copied() + .filter(|id| old_set.contains(id)) + .collect(); + if old_retained != new_retained { + return None; + } + + // Old-row lookup: block id → (block row, paragraph index if a paragraph). + let mut old_paragraph_ix = projection + .blocks + .get_range(0..old_lo)? + .filter(|block| matches!(block, Block::Paragraph(_))) + .count(); + let mut old_rows: FxHashMap)> = FxHashMap::default(); + for (offset, block) in projection.blocks.get_range(old_lo..old_hi)?.enumerate() { + let paragraph = matches!(block, Block::Paragraph(_)).then_some(old_paragraph_ix); + if paragraph.is_some() { + old_paragraph_ix += 1; + } + old_rows.insert(old_ids[offset], (old_lo + offset, paragraph)); + } + + let mut patches = Vec::new(); + let mut inserts: Vec = Vec::new(); + let mut insert_runs: Vec<(Vec, Option)> = Vec::new(); + let mut new_paragraph_pointer = 0_usize; + for (new_ix, id) in new_ids.iter().enumerate() { + let new_block = region.blocks.get(new_ix)?; + let new_paragraph_id = if matches!(new_block, InputBlock::Paragraph(_)) { + let paragraph_id = region.paragraph_ids.get(new_paragraph_pointer).copied(); + new_paragraph_pointer += 1; + paragraph_id + } else { + None + }; + if let Some((old_ix, old_paragraph)) = old_rows.get(id).copied() { + // Retained row boundary: flush any pending insert run before it. + if !inserts.is_empty() { + insert_runs.push((std::mem::take(&mut inserts), Some(*id))); + } + match (new_block, old_paragraph) { + (InputBlock::Paragraph(new_input), Some(paragraph_ix)) => { + // Identity must agree row-for-row; a swapped paragraph id under a + // retained block id is not expressible as a content patch. + if new_paragraph_id != projection.ids.paragraph_ids.get(paragraph_ix).copied() { + return None; + } + // `old_ix` is this paragraph's block row (from `old_rows`); pass it + // directly — no per-paragraph O(blocks) scan. + paragraph_patches_from_readback(projection, paragraph_ix, old_ix, new_input.clone(), &mut patches)?; + }, + (new_input, None) => { + let old_input = input_block_from_block(projection.blocks.get(old_ix)?); + if old_input != *new_input { + patches.extend(super::object_replacement_patch(projection, old_ix, new_input.clone())?); + } + }, + // Kind changed under a retained id — rebuild territory. + _ => return None, + } + } else { + inserts.push(ProjectionStructuralBlock { + block_id: *id, + paragraph_id: new_paragraph_id, + block: new_block.clone(), + }); + } + } + if !inserts.is_empty() { + // Trailing run: anchor before the first old row AFTER the region. + insert_runs.push((inserts, projection.ids.block_ids.get(old_hi).copied())); + } + + let dead: Vec<_> = old_ids + .iter() + .copied() + .filter(|id| !new_set.contains(id)) + .collect(); + if !dead.is_empty() { + patches.push(ProjectionPatch::DeleteBlocks { + block_ids: dead, + row_hint: old_lo, + }); + } + for (blocks, before) in insert_runs { + patches.push(ProjectionPatch::InsertBlocks { + before, + row_hint: old_lo, + blocks, + }); + } + Some(patches) +} + +#[hotpath::measure] +pub(crate) fn remote_nonstructural_projection_patches( + projection: &DocumentProjection, + doc: &LoroDoc, + invalidation: &ProjectionInvalidation, + touched_paragraphs: &[usize], + live_starts: &[usize], +) -> Option> { + if invalidation.rebuild_required || !invalidation.changed_sections.is_empty() { + return None; + } + // Object docs are safe on this incremental path because the RANGED readback + // resolves each touched paragraph's live range from its durable record — + // object placeholders and coalesced empties cannot mis-align the rows the + // way the legacy whole-body index walk could. A range that does not parse + // as exactly one paragraph returns None and falls back to the rebuild. + // (Convergence: proven by the N-peer structural fuzz.) + let mut patches = paragraph_projection_patches_ranged(projection, doc, live_starts, touched_paragraphs.iter().copied())?; + if !invalidation.changed_blocks.is_empty() + || !invalidation.changed_tables.is_empty() + || invalidation + .changed_flows + .iter() + .any(|flow| flow != flowstate_document::ROOT_BODY_FLOW_ID) + { + // §act-ten A10.4: try the SCOPED object readback first — a remote table-cell + // keystroke re-projects one table, not every object block in the document. + // Any unattributable change falls back to the full readback (the previous + // behavior), which itself returns None on structural surprises. + match remote_object_projection_patches_scoped(projection, doc, invalidation) { + Some(scoped) => patches.extend(scoped), + None => patches.extend(remote_object_projection_patches(projection, doc)?), + } + } + Some(patches) +} + +/// §oom-leads #4: the scoped-else-full composition as a callable — the remote +/// STRUCTURAL path was calling the full registry readback directly, so every +/// peer split (whose delta carries its own `paragraph_block.*` record write) +/// paid an O(objects) readback for zero object changes. Scoped attribution +/// already classifies paragraph-registry keys as ignorable; anything it cannot +/// attribute falls back to the full readback exactly as before. +pub(crate) fn remote_object_projection_patches_scoped_or_full( + projection: &DocumentProjection, + doc: &LoroDoc, + invalidation: &ProjectionInvalidation, +) -> Option> { + match remote_object_projection_patches_scoped(projection, doc, invalidation) { + Some(scoped) => Some(scoped), + None => { + static DERIVE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + if *DERIVE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + " scoped-object-readback BAILED to full: changed_blocks={:?} changed_tables={:?} nonbody_flows={:?}", + invalidation.changed_blocks, + invalidation.changed_tables, + invalidation + .changed_flows + .iter() + .filter(|flow| *flow != flowstate_document::ROOT_BODY_FLOW_ID) + .collect::>() + ); + } + remote_object_projection_patches(projection, doc) + }, + } +} + +/// §act-ten A10.4: the object readback scoped to the blocks that actually +/// changed. Attribution sources, in order: +/// - non-body `changed_flows` keys: the flow-id naming scheme encodes the +/// owner (`.` nested flows, `.flow` cell flows — cell ids +/// map to their table via the projection's own rows); +/// - `changed_blocks`/`changed_tables` container targets: the container's +/// path to the root names the owning registry entry. +/// +/// Returns `None` when ANY change cannot be attributed (callers run the full +/// readback), or when a scoped readback disagrees structurally with the +/// projection (new/absent block — the rebuild paths own those). +fn remote_object_projection_patches_scoped( + projection: &DocumentProjection, + doc: &LoroDoc, + invalidation: &ProjectionInvalidation, +) -> Option> { + use flowstate_document::ChangedContainerOwner; + // The projection's object rows by id — also the cell → table reverse map + // (built from already-materialized rows; no Loro reads). + let mut object_rows: FxHashMap = FxHashMap::default(); + let mut cell_to_block: FxHashMap = FxHashMap::default(); + for (row, block) in projection.blocks.iter().enumerate() { + if matches!(block, Block::Paragraph(_)) { + continue; + } + let block_id = projection.ids.block_ids.get(row)?; + object_rows.insert(block_id.0, (row, block)); + if let Block::Table(table) = block { + for table_row in &table.rows { + for cell in &table_row.cells { + cell_to_block.insert(cell.id.0, block_id.0); + } + } + } + } + let owner_block_of_key = |key: &str| -> Option { + // Flow keys encode their owner: `.`. The owner is a + // block id (caption/alt/source flows) or a cell id (cell flows). + let owner = key.rsplit_once('.').map(|(owner, _)| owner).unwrap_or(key); + let owner_id = flowstate_document::loro_schema::loro_id_trailing_u128(owner)?; + if object_rows.contains_key(&owner_id) { + return Some(owner_id); + } + cell_to_block.get(&owner_id).copied() + }; + let mut touched: Vec = Vec::new(); + for flow in &invalidation.changed_flows { + if flow == flowstate_document::ROOT_BODY_FLOW_ID { + continue; + } + touched.push(owner_block_of_key(flow)?); + } + for target in invalidation + .changed_blocks + .iter() + .chain(&invalidation.changed_tables) + { + match flowstate_document::owner_of_changed_container(doc, target)? { + ChangedContainerOwner::Block(key) => { + let id = flowstate_document::loro_schema::loro_id_trailing_u128(&key)?; + if !object_rows.contains_key(&id) { + // A paragraph record (or an id the projection does not know): the + // ranged paragraph readback owns paragraph records; unknown ids are + // structural — either way this scoped pass has nothing to patch. + continue; + } + touched.push(id); + }, + ChangedContainerOwner::Flow(key) => touched.push(owner_block_of_key(&key)?), + ChangedContainerOwner::ObjectReadbackInert => {}, + } + } + touched.sort_unstable(); + touched.dedup(); + if touched.is_empty() { + return Some(Vec::new()); + } + let projected = flowstate_document::object_input_blocks_for_ids(doc, &touched).ok()?; + if projected.len() != touched.len() { + // A touched id the registry no longer has (or a duplicate collapse): + // structural — the rebuild paths own it. + return None; + } + let mut patches = Vec::new(); + for (block_id, after) in projected { + let (row, block) = object_rows.get(&block_id.0)?; + let before = input_block_from_block(block); + if before != after { + patches.push(ProjectionPatch::ReplaceObjectBlock { + block_id: flowstate_document::BlockId(block_id.0), + row_hint: *row, + block: ProjectionStructuralBlock { + block_id: flowstate_document::BlockId(block_id.0), + paragraph_id: None, + block: after, + }, + }); + } + } + Some(patches) +} + +/// Reads each touched paragraph back through [`body_input_paragraph_at`] +/// (O(paragraph) per read instead of an O(doc) whole-body walk), with ranges +/// resolved from durable paragraph records — exact in live space even for +/// object docs, which the legacy index-walk readback could not promise. +pub(crate) fn paragraph_projection_patches_ranged( + projection: &DocumentProjection, + doc: &LoroDoc, + live_starts: &[usize], + touched_paragraphs: impl IntoIterator, +) -> Option> { + let touched: Vec = touched_paragraphs.into_iter().collect(); + if touched.is_empty() { + return Some(Vec::new()); + } + // Batch-resolve every needed paragraph start (touched ∪ their successors) in + // ONE `query_text_id_positions` pass. Per-paragraph `get_cursor_pos` is a + // linear chunk scan, so resolving each touched paragraph individually (2 + // resolutions per paragraph via the old `live_paragraph_range`) made a large + // readback — an undone replace-all storm, a mass restyle — QUADRATIC in + // document size (measured 36s for one undo on the reference doc; 69% of + // soak CPU inside `paragraph_body_start_in_loro`). + let paragraph_count = projection.ids.paragraph_ids.len(); + let mut needed: Vec = Vec::with_capacity(touched.len() * 2); + for &ix in &touched { + needed.push(ix); + if ix + 1 < paragraph_count { + needed.push(ix + 1); + } + } + needed.sort_unstable(); + needed.dedup(); + let ids: Vec<_> = needed + .iter() + .filter_map(|&ix| projection.ids.paragraph_ids.get(ix).copied()) + .collect(); + if ids.len() != needed.len() { + return None; + } + let starts = super::paragraph_body_starts_in_loro(doc, &ids); + let start_by_ix: FxHashMap = needed + .iter() + .zip(starts) + .filter_map(|(&ix, start)| { + // Durable-record resolution first (object-aware, coalesce-immune), the + // projection's shifted live start as the fallback — same law as the old + // per-paragraph `live_paragraph_range`. + start + .or_else(|| live_starts.get(ix).copied()) + .map(|start| (ix, start)) + }) + .collect(); + let body_len = body_text(doc).len_unicode(); + // One O(blocks) pass for every paragraph's row instead of an O(blocks) scan + // per touched paragraph (the §perf-heaven T2 quadratic on object docs). + let rows = flowstate_document::paragraph_block_rows(projection); + let mut patches = Vec::new(); + for paragraph_ix in touched { + let old = projection.paragraphs.get(paragraph_ix)?; + let &row = rows.get(paragraph_ix)?; + let sentinel = start_by_ix.get(¶graph_ix)?.checked_sub(1)?; + let end = if paragraph_ix + 1 < paragraph_count { + start_by_ix.get(&(paragraph_ix + 1))?.checked_sub(1)? + } else { + body_len + }; + if sentinel >= end { + return None; + } + let new_input = body_input_paragraph_at(doc, sentinel, end, input_paragraph(projection, paragraph_ix, old).style)?; + paragraph_patches_from_readback(projection, paragraph_ix, row, new_input, &mut patches)?; + } + Some(patches) +} + +fn paragraph_patches_from_readback( + projection: &DocumentProjection, + paragraph_ix: usize, + row: usize, + new_input: InputParagraph, + patches: &mut Vec, +) -> Option<()> { + let old = projection.paragraphs.get(paragraph_ix)?; + let old_input = input_paragraph(projection, paragraph_ix, old); + let old_text = old_input + .runs + .iter() + .map(|run| run.text.as_str()) + .collect::(); + let new_text = new_input + .runs + .iter() + .map(|run| run.text.as_str()) + .collect::(); + if old_text != new_text { + patches.push(ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[paragraph_ix], + row_hint: row, + delta_utf8: text_delta_between(&old_text, &new_text), + new: new_input, + }); + return Some(()); + } + if old_input.style != new_input.style { + patches.push(ProjectionPatch::ParagraphStyle { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[paragraph_ix], + row_hint: row, + style: new_input.style, + }); + } + // §act-ten A10.7: convert the runs DIRECTLY (`merge_adjacent_runs` over + // byte-length TextRuns is exactly what `document_from_input_blocks` does per + // paragraph) instead of assembling a whole throwaway DocumentProjection — + // Rope, trees and sections included — per readback paragraph (12,404 calls / + // 116 MB on a mass-op undo in the peer profiles). + let new_runs = flowstate_document::merge_adjacent_runs( + new_input + .runs + .iter() + .map(|run| flowstate_document::TextRun { + len: run.text.len(), + styles: run.styles, + }) + .collect(), + ); + if old.runs != new_runs { + patches.push(ProjectionPatch::ParagraphRuns { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[paragraph_ix], + row_hint: row, + runs: new_runs, + }); + } + Some(()) +} + +#[hotpath::measure] +pub(crate) fn remote_object_projection_patches(projection: &DocumentProjection, doc: &LoroDoc) -> Option> { + let projected = flowstate_document::object_input_blocks_from_loro(doc).ok()?; + let mut existing = projection + .blocks + .iter() + .enumerate() + .filter_map(|(row, block)| { + if matches!(block, Block::Paragraph(_)) { + return None; + } + Some((projection.ids.block_ids.get(row).copied()?, row, input_block_from_block(block))) + }) + .collect::>(); + existing.sort_by_key(|(id, _, _)| id.0); + if existing.len() != projected.len() + || existing + .iter() + .zip(&projected) + .any(|((existing_id, _, _), (projected_id, _))| existing_id != projected_id) + { + return None; + } + Some( + existing + .into_iter() + .zip(projected) + .filter_map(|((block_id, row, before), (_, after))| { + (before != after).then_some(ProjectionPatch::ReplaceObjectBlock { + block_id, + row_hint: row, + block: ProjectionStructuralBlock { + block_id, + paragraph_id: None, + block: after, + }, + }) + }) + .collect(), + ) +} + +fn text_delta(prefix_retain: usize, delete_len: usize, insert_len: usize, trailing_retain: usize) -> Vec { + let mut delta = Vec::new(); + if prefix_retain > 0 { + delta.push(ProjectionTextDelta::Retain(prefix_retain)); + } + if delete_len > 0 { + delta.push(ProjectionTextDelta::Delete(delete_len)); + } + if insert_len > 0 { + delta.push(ProjectionTextDelta::Insert(insert_len)); + } + if trailing_retain > 0 { + delta.push(ProjectionTextDelta::Retain(trailing_retain)); + } + delta +} + +fn common_prefix_byte_len(left: &str, right: &str) -> usize { + let mut len = 0; + for ((left_ix, left_ch), (_, right_ch)) in left.char_indices().zip(right.char_indices()) { + if left_ch != right_ch { + break; + } + len = left_ix + left_ch.len_utf8(); + } + len +} + +fn common_suffix_byte_len(left: &str, right: &str, prefix: usize) -> usize { + let mut len = 0; + for ((left_ix, left_ch), (right_ix, right_ch)) in left.char_indices().rev().zip(right.char_indices().rev()) { + if left_ix < prefix || right_ix < prefix || left_ch != right_ch { + break; + } + len += left_ch.len_utf8(); + } + len +} + +/// Ranged paragraph readback (§11 complexity contract): reads ONE paragraph +/// back through `slice_delta` over its live unicode range instead of +/// materializing the whole body — O(paragraph), not O(doc). +/// +/// `sentinel_unicode` addresses the paragraph's LEADING boundary `\n` (the +/// paragraph-style carrier; the seed sentinel for paragraph 0). `end_unicode` +/// is exclusive and may cover trailing object placeholders, which fold out of +/// paragraph text exactly like the legacy whole-body walk. Returns `None` +/// when the slice does not look like exactly one paragraph (missing leading +/// sentinel, or an interior boundary such as a coalesced empty) so callers +/// fall back to a full rebuild instead of patching the wrong rows. +pub(crate) fn body_input_paragraph_at( + doc: &LoroDoc, + sentinel_unicode: usize, + end_unicode: usize, + fallback_style: ParagraphStyle, +) -> Option { + let text = body_text(doc); + let end = end_unicode.min(text.len_unicode()); + if sentinel_unicode >= end { + return None; + } + let spans = text + .slice_delta(sentinel_unicode, end, loro::cursor::PosType::Unicode) + .ok()?; + let mut current = InputParagraph { + style: fallback_style, + runs: Vec::new(), + }; + let mut seen_sentinel = false; + for item in spans { + let loro::TextDelta::Insert { insert, attributes } = item else { + continue; + }; + let run_styles = run_styles_from_attrs(attributes.as_ref()); + for ch in insert.chars() { + if ch == '\n' { + if seen_sentinel { + return None; + } + seen_sentinel = true; + current.style = paragraph_style_from_attrs(attributes.as_ref()).unwrap_or(fallback_style); + } else if ch != OBJECT_REPLACEMENT { + if !seen_sentinel { + return None; + } + push_input_char(&mut current, ch, run_styles); + } + } + } + seen_sentinel.then_some(current) +} + +fn push_input_char(paragraph: &mut InputParagraph, ch: char, styles: RunStyles) { + if let Some(last) = paragraph.runs.last_mut() + && last.styles == styles + { + last.text.push(ch); + return; + } + paragraph.runs.push(InputRun { + text: ch.to_string(), + styles, + }); +} + +fn run_styles_from_attrs(attrs: Option<&FxHashMap>) -> RunStyles { + let mut styles = RunStyles::default(); + let Some(attrs) = attrs else { + return styles; + }; + if let Some(LoroValue::I64(slot)) = attrs.get(MARK_RUN_SEMANTIC_STYLE) + && let Ok(slot) = u8::try_from(*slot) + { + styles.semantic = RunSemanticStyle::Custom(slot); + } + if let Some(LoroValue::I64(slot)) = attrs.get(MARK_HIGHLIGHT_STYLE) + && let Ok(slot) = u8::try_from(*slot) + { + styles.highlight = Some(HighlightStyle::Custom(slot)); + } + if matches!(attrs.get(MARK_DIRECT_UNDERLINE), Some(LoroValue::Bool(true))) { + styles.direct_underline = true; + } + if matches!(attrs.get(MARK_STRIKETHROUGH), Some(LoroValue::Bool(true))) { + styles.strikethrough = true; + } + styles +} diff --git a/crates/flowstate-collab/src/crdt_runtime/projection_repair.rs b/crates/flowstate-collab/src/crdt_runtime/projection_repair.rs new file mode 100644 index 00000000..afa2c841 --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime/projection_repair.rs @@ -0,0 +1,485 @@ +//! §P2a projection repair. +//! +//! Projecting a Loro document reports one [`ProjectionDefect`] per malformed +//! canonical-state anomaly instead of silently laundering it (see +//! `flowstate_document::projection_defects`). This module turns each defect into +//! **one idempotent, convergent canonical Loro mutation**. Every mutation is +//! *check-before-write*, keyed on the stable container / cursor the defect names, +//! so two peers repairing the same defect concurrently converge on identical +//! state through Loro map/mark LWW semantics rather than racing to insert +//! divergent content. +//! +//! The runtime commits the batch of repairs under a dedicated `repair` origin +//! (excluded from undo), persists the resulting update segment, and re-projects. +//! A per-`stable_key` attempt cap in the runtime guarantees a defect that cannot +//! be repaired is eventually quarantined instead of looping. +//! +//! Convergence design notes (why these choices and not others): +//! * Content-*inserting* repairs (e.g. re-inserting a U+FFFC placeholder for an +//! unresolved/displaced object anchor) cannot be made convergent — two peers +//! would each insert a placeholder and duplicate the object. So anchor defects +//! resolve by *deletion* of the dangling block record, keyed on the block's +//! stable map key: concurrent deletes of the same key converge. +//! * Metadata / mark / field *writes* are keyed on a stable location (paragraph +//! boundary cursor, asset content hash), so concurrent peers compute the same +//! value and Loro LWW converges them. + +use anyhow::{Context as _, Result}; +use flowstate_document::{ + ColumnId, ProjectionDefect, RowId, TABLE_CELLS_BY_ID, TABLE_COLUMN_ORDER, TABLE_KEY, TABLE_ROW_ORDER, TableTopologyKind, cell_loro_id_for, + column_loro_id, row_loro_id, +}; +use flowstate_fidelity as fidelity; +use loro::{ContainerTrait as _, LoroDoc, LoroMap, LoroText, cursor::Cursor}; + +use super::{ + BLOCKS_BY_ID, FLOWS_BY_ID, MARK_PARAGRAPH_STYLE, OBJECT_REPLACEMENT, ParagraphStyle, ROOT, ROOT_BODY_FLOW_ID, body_text, child_map, + child_movable_list, empty_input_table_cell, ensure_paragraph_metadata_at_boundary_with_keys, flow_text, map_binary_opt, map_i64_opt, map_keys, + map_string_opt, movable_list_strings, paragraph_style_value, write_table_cell_map_from_input, +}; + +/// Apply the single canonical repair for one projection defect. +/// +/// Returns `Ok(true)` when a canonical mutation was written (the caller then +/// commits the batch under the `repair` origin), `Ok(false)` when the defect no +/// longer applies (already repaired, healed by a concurrent edit, or a peer got +/// there first — all convergent no-ops). Never commits; the caller owns the +/// commit and its origin. +#[hotpath::measure] +pub(super) fn apply_projection_repair(doc: &LoroDoc, defect: &ProjectionDefect) -> Result { + let result = match defect { + ProjectionDefect::MissingParagraphMetadata { + flow_id, boundary_unicode, .. + } + | ProjectionDefect::MissingParagraphBlock { + flow_id, boundary_unicode, .. + } => repair_missing_paragraph_metadata(doc, flow_id, *boundary_unicode), + ProjectionDefect::MissingParagraphStyleMark { flow_id, boundary_unicode } => { + repair_missing_paragraph_style_mark(doc, flow_id, *boundary_unicode) + }, + ProjectionDefect::UnresolvedObjectAnchor { block_key, flow_id, .. } => repair_unresolved_object_anchor(doc, flow_id, block_key), + ProjectionDefect::CollidingObjectAnchors { + flow_id, + kept_block_key, + displaced_block_key, + .. + } => repair_displaced_object_block(doc, flow_id, kept_block_key, displaced_block_key), + ProjectionDefect::OrphanObjectPlaceholder { flow_id, unicode_pos } => repair_orphan_object_placeholder(doc, flow_id, *unicode_pos), + ProjectionDefect::InvalidAssetId { block_key, .. } => repair_invalid_asset_id(doc, block_key), + ProjectionDefect::TableTopology { + table_block_key, + row_id, + column_id, + kind, + } => repair_table_topology(doc, table_block_key, *row_id, *column_id, *kind), + }; + // §fidelity: record the canonical mutation this defect produced (Ok(true)); a + // convergent no-op (Ok(false)) or error is left to the caller's telemetry. + if matches!(&result, Ok(true)) { + fidelity::event(super::fidelity_class_for_defect(defect), "repair-applied", || { + format!("class={} key={}", defect.class(), defect.stable_key()) + }); + } + result +} + +/// FS-004 / FS-005: write a durable paragraph metadata + paragraph-block record +/// for a boundary that had none. Only the body flow reports these (the projector +/// tracks durable ids for the body alone), so other flows are ignored. +/// +/// * `Some(boundary)` — write the durable record via the runtime's boundary-keyed +/// metadata writer, forcing *deterministic* map keys derived from the flow + +/// boundary so concurrent peers write the same keys and converge (a random uuid +/// would diverge). The writer re-checks that the boundary is a live newline. +/// * `None` — the body is empty/uninitialized: converge on the *same* canonical +/// seed that fresh-document creation uses (`seed_document_body`). +fn repair_missing_paragraph_metadata(doc: &LoroDoc, flow_id: &str, boundary_unicode: Option) -> Result { + if flow_id != ROOT_BODY_FLOW_ID { + return Ok(false); + } + match boundary_unicode { + Some(boundary) => { + let body = body_text(doc); + // Derive the SAME position-independent keys the projection fabricates for a + // missing boundary (`stable_boundary_metadata_keys`, keyed off the boundary + // newline's stable OpID). Using the identical derivation means a repaired + // record's id equals the fabricated id on every peer, so they converge via + // Loro map LWW instead of the old position-based keys (block_ix vs unicode + // offset), which diverged between the incremental projection, the full + // rebuild, and across peers. + // + // §perf note (2026-07-07, measured): carrying the WALK-TIME keys on the + // defect to skip this per-defect `get_cursor` chunk scan (~1s across a + // mass-undo pass) is NOT safe: earlier repairs in the same pass can + // mutate body text (e.g. orphan-placeholder deletion), shifting later + // boundaries, and the walk-time keys then disagree with this LIVE + // re-derivation — redo after a mass-undo produced DuplicateBlockId + // patches. Re-deriving per repair self-corrects; keep it. + static REPAIR_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + let repair_debug = *REPAIR_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()); + let Some((paragraph_key, block_key)) = flowstate_document::loro_projection::stable_boundary_metadata_keys(&body, boundary) else { + // No live anchor for this boundary yet; leave it for a later pass. + if repair_debug { + eprintln!("repair[missing-paragraph]: boundary {boundary} has no live anchor; deferred"); + } + return Ok(false); + }; + if repair_debug { + eprintln!("repair[missing-paragraph]: boundary {boundary} → keys {paragraph_key} / {block_key}"); + } + ensure_paragraph_metadata_at_boundary_with_keys(doc, &body, boundary, Some(paragraph_key), Some(block_key)) + .context("writing durable paragraph metadata for MissingParagraph* defect")?; + Ok(true) + }, + None => { + flowstate_document::loro_schema::seed_document_body(doc).context("seeding empty body for MissingParagraph* defect")?; + Ok(true) + }, + } +} + +/// adjustmentplan:224 — a paragraph boundary newline missing its paragraph-style +/// mark. Mirror the canonical mark write used by +/// `super::repair_missing_paragraph_style_marks` (the existing style-mark repair), +/// but scoped to the one boundary this defect names so it folds into the single +/// `repair`-origin commit instead of committing on its own. +/// +/// Convergent: marking the boundary `Normal` is LWW — two peers writing the same +/// value converge. Check-before-write only skips when the boundary is no longer a +/// live newline. +fn repair_missing_paragraph_style_mark(doc: &LoroDoc, flow_id: &str, boundary_unicode: usize) -> Result { + let Some(text) = flow_text_for_id(doc, flow_id) else { + return Ok(false); + }; + // O(1) boundary liveness — the whole-flow stringify this replaced ran once + // PER DEFECT, which turned the mass-undo repair pass (thousands of + // fabricated boundaries) into minutes of allocation. + if text.char_at(boundary_unicode) != Ok('\n') { + return Ok(false); + } + text + .mark( + boundary_unicode..boundary_unicode + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(ParagraphStyle::Normal), + ) + .context("repairing missing paragraph style mark")?; + Ok(true) +} + +/// FS-002: an object block whose anchor no longer resolves to a live U+FFFC +/// placeholder. Its placeholder is gone, so re-anchoring would have to *insert* +/// content (non-convergent under concurrent repair). Instead delete the dangling +/// block record, keyed on its stable map key — concurrent deletes converge. +/// +/// Check-before-write: skip if the anchor now resolves to a live placeholder +/// (a concurrent edit healed it) or the record is already gone. +fn repair_unresolved_object_anchor(doc: &LoroDoc, flow_id: &str, block_key: &str) -> Result { + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return Ok(false); + }; + let Some(block) = child_map(&blocks, block_key) else { + return Ok(false); + }; + if map_string_opt(&block, "kind").as_deref() == Some("paragraph") { + return Ok(false); + } + if let Some(text) = flow_text_for_id(doc, flow_id) + && anchor_placeholder_pos(doc, &block, &text).is_some() + { + return Ok(false); + } + blocks + .delete(block_key) + .context("deleting orphaned object block for UnresolvedObjectAnchor defect")?; + Ok(true) +} + +/// FS-003: two object blocks resolved their anchors to the *same* placeholder; +/// only one can own it. Delete the displaced record so a single owner remains — +/// convergent (deletion keyed on the displaced block's stable map key) where +/// giving it a "distinct anchor" (a fresh inserted placeholder) would not be. +/// +/// Check-before-write: only act on a still-live collision (both anchors resolve +/// to the same placeholder position); otherwise the collision already dissolved. +fn repair_displaced_object_block(doc: &LoroDoc, flow_id: &str, kept_block_key: &str, displaced_block_key: &str) -> Result { + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return Ok(false); + }; + let (Some(kept), Some(displaced)) = (child_map(&blocks, kept_block_key), child_map(&blocks, displaced_block_key)) else { + return Ok(false); + }; + let Some(text) = flow_text_for_id(doc, flow_id) else { + return Ok(false); + }; + match (anchor_placeholder_pos(doc, &kept, &text), anchor_placeholder_pos(doc, &displaced, &text)) { + (Some(kept_pos), Some(displaced_pos)) if kept_pos == displaced_pos => { + blocks + .delete(displaced_block_key) + .context("deleting displaced object block for CollidingObjectAnchors defect")?; + Ok(true) + }, + _ => Ok(false), + } +} + +/// FS-036 backstop: a U+FFFC placeholder that no object block claims. Delete the +/// stray character canonically. Only the body flow reports this. +/// +/// Check-before-write: skip unless the character is still a U+FFFC and still +/// unclaimed (a block may have been anchored to it since projection). +fn repair_orphan_object_placeholder(doc: &LoroDoc, flow_id: &str, unicode_pos: usize) -> Result { + if flow_id != ROOT_BODY_FLOW_ID { + return Ok(false); + } + let text = body_text(doc); + if text.char_at(unicode_pos) != Ok(OBJECT_REPLACEMENT) { + return Ok(false); + } + if placeholder_is_claimed(doc, &text, unicode_pos) { + return Ok(false); + } + text + .delete(unicode_pos, 1) + .context("deleting orphan object placeholder for OrphanObjectPlaceholder defect")?; + Ok(true) +} + +/// FS-011: an image block whose `asset_id` is missing/unparseable. Recover it +/// from the block's durable `content_hash` (copied onto the block at insert time) +/// by matching an asset in `assets_by_id`. Convergent: every peer computes the +/// same recovered id from the same content hash and Loro LWW converges the write. +/// +/// Check-before-write: skip when the block already has a valid id, has no content +/// hash, or no asset matches. +fn repair_invalid_asset_id(doc: &LoroDoc, block_key: &str) -> Result { + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return Ok(false); + }; + let Some(block) = find_block_by_id(&blocks, block_key) else { + return Ok(false); + }; + if map_string_opt(&block, "kind").as_deref() != Some("image") { + return Ok(false); + } + if map_string_opt(&block, "asset_id") + .and_then(|id| id.parse::().ok()) + .is_some() + { + return Ok(false); + } + let Some(content_hash) = map_string_opt(&block, "content_hash") else { + return Ok(false); + }; + let Some(assets) = child_map(&root, flowstate_document::loro_schema::ASSETS_BY_ID) else { + return Ok(false); + }; + let Some(asset_id) = asset_id_for_content_hash(&assets, &content_hash) else { + return Ok(false); + }; + block + .insert("asset_id", asset_id.as_str()) + .context("recovering asset id from content hash for InvalidAssetId defect")?; + Ok(true) +} + +/// FS-010 / §P2b: a table grid that is not a well-formed full rectangle. The +/// projector already normalized it deterministically (every peer reads the same +/// grid), so here we apply the matching **idempotent, convergent** canonical +/// mutation, keyed on the deterministic `(row, column)` coordinate cell id so two +/// peers repairing the same coordinate address the identical Loro container and +/// Loro LWW merges them: +/// * `MissingCell` — ensure the coordinate cell (with an empty cell flow) when it +/// is truly absent (check-before-write avoids clobbering a healed cell). +/// * `InvalidSpan` — clamp the cell's `row_span`/`column_span` into the grid +/// bounds (`>= 1`, `<=` the cells remaining below/right of it). +/// * `OrphanCell` — delete the coordinate cell. +/// * `DuplicateCoordinate` — cannot occur with deterministic cell ids (the same +/// key merges), so it is a no-op. +fn repair_table_topology( + doc: &LoroDoc, + table_block_key: &str, + row_id: Option, + column_id: Option, + kind: TableTopologyKind, +) -> Result { + if matches!(kind, TableTopologyKind::DuplicateCoordinate) { + return Ok(false); + } + // Every remaining kind is tied to a single coordinate; without both ids there + // is nothing addressable to repair. + let (Some(row_id), Some(column_id)) = (row_id, column_id) else { + return Ok(false); + }; + let row_id = RowId(row_id); + let column_id = ColumnId(column_id); + let Some(table) = resolve_table_map(doc, table_block_key) else { + return Ok(false); + }; + let cells_by_id = table.ensure_mergeable_map(TABLE_CELLS_BY_ID)?; + let cell_id_str = cell_loro_id_for(row_id, column_id); + match kind { + TableTopologyKind::MissingCell => { + // Check-before-write: a concurrent edit (or a peer's repair) may already + // have materialized the cell — leave its content alone. + if child_map(&cells_by_id, &cell_id_str).is_some() { + return Ok(false); + } + let cell_map = cells_by_id.ensure_mergeable_map(&cell_id_str)?; + let empty = empty_input_table_cell(row_id, column_id); + write_table_cell_map_from_input( + doc, + &cell_map, + &cell_id_str, + &row_loro_id(row_id), + &column_loro_id(column_id), + &empty, + false, + ) + .context("materializing missing table cell for TableTopology defect")?; + Ok(true) + }, + TableTopologyKind::InvalidSpan => { + let Some(cell) = child_map(&cells_by_id, &cell_id_str) else { + return Ok(false); + }; + let row_ids = child_movable_list(&table, TABLE_ROW_ORDER) + .map(|list| movable_list_strings(&list)) + .unwrap_or_default(); + let column_ids = child_movable_list(&table, TABLE_COLUMN_ORDER) + .map(|list| movable_list_strings(&list)) + .unwrap_or_default(); + let (Some(row_ix), Some(column_ix)) = ( + row_ids.iter().position(|id| id == &row_loro_id(row_id)), + column_ids + .iter() + .position(|id| id == &column_loro_id(column_id)), + ) else { + return Ok(false); + }; + let max_row_span = i64::try_from(row_ids.len() - row_ix).unwrap_or(1).max(1); + let max_column_span = i64::try_from(column_ids.len() - column_ix) + .unwrap_or(1) + .max(1); + let row_span = map_i64_opt(&cell, "row_span") + .unwrap_or(1) + .clamp(1, max_row_span); + let column_span = map_i64_opt(&cell, "column_span") + .unwrap_or(1) + .clamp(1, max_column_span); + cell.insert("row_span", row_span)?; + cell.insert("column_span", column_span)?; + Ok(true) + }, + TableTopologyKind::OrphanCell => { + if child_map(&cells_by_id, &cell_id_str).is_none() { + return Ok(false); + } + cells_by_id + .delete(&cell_id_str) + .context("deleting orphan table cell for TableTopology defect")?; + Ok(true) + }, + TableTopologyKind::DuplicateCoordinate => Ok(false), + } +} + +/// Resolve a table block's canonical `table` child map from the projector-reported +/// block id (the block's `id` field / map key), mirroring the object-anchor +/// repairs' block resolution. +fn resolve_table_map(doc: &LoroDoc, table_block_key: &str) -> Option { + let root = doc.get_map(ROOT); + let blocks = child_map(&root, BLOCKS_BY_ID)?; + let block = find_block_by_id(&blocks, table_block_key)?; + if map_string_opt(&block, "kind").as_deref() != Some("table") { + return None; + } + child_map(&block, TABLE_KEY) +} + +/// Resolve a flow's canonical text container by flow id (via `flows_by_id`). +fn flow_text_for_id(doc: &LoroDoc, flow_id: &str) -> Option { + let root = doc.get_map(ROOT); + let flows = child_map(&root, FLOWS_BY_ID)?; + let flow = child_map(&flows, flow_id)?; + flow_text(doc, &flow).ok() +} + +/// The unicode position of the live U+FFFC placeholder a block's `anchor_cursor` +/// resolves to *within `text`*, or `None` if it does not resolve to one there. +/// Mirrors the projector's object-anchor resolution (container-checked) so a +/// repair's check-before-write agrees with what projection would see. +fn anchor_placeholder_pos(doc: &LoroDoc, block: &LoroMap, text: &LoroText) -> Option { + let bytes = map_binary_opt(block, "anchor_cursor")?; + let cursor = Cursor::decode(&bytes).ok()?; + if cursor.container != text.id() { + return None; + } + // A DEAD anchor is unresolvable, not history-traced (`get_cursor_pos` walks + // update history per dead anchor — post-mass-undo every restored object + // record is dead-anchored, which multiplied into minutes). The batch query + // is a single live-state scan; a dead id is simply absent. + let id = cursor.id?; + let pos = doc + .inner() + .query_text_id_positions(&text.id(), &[id]) + .pop() + .flatten()?; + (text.char_at(pos) == Ok(OBJECT_REPLACEMENT)).then_some(pos) +} + +/// Whether any non-paragraph block's anchor resolves to `pos` in `text`. +fn placeholder_is_claimed(doc: &LoroDoc, text: &LoroText, pos: usize) -> bool { + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return false; + }; + for key in map_keys(&blocks) { + let Some(block) = child_map(&blocks, &key) else { + continue; + }; + if map_string_opt(&block, "kind").as_deref() == Some("paragraph") { + continue; + } + if anchor_placeholder_pos(doc, &block, text) == Some(pos) { + return true; + } + } + false +} + +/// Find a block record by the projector-reported block id, which is the block's +/// `id` field (equal to its map key for runtime-created blocks). Try a direct key +/// lookup first, then scan for a matching `id` field. +fn find_block_by_id(blocks: &LoroMap, block_key: &str) -> Option { + if let Some(block) = child_map(blocks, block_key) { + return Some(block); + } + for key in map_keys(blocks) { + let Some(block) = child_map(blocks, &key) else { + continue; + }; + if map_string_opt(&block, "id").as_deref() == Some(block_key) { + return Some(block); + } + } + None +} + +/// The asset id (decimal string) of the asset in `assets_by_id` whose stored +/// `content_hash` equals `content_hash`, preferring the asset's own `asset_id` +/// field and falling back to its map key. +fn asset_id_for_content_hash(assets: &LoroMap, content_hash: &str) -> Option { + for key in map_keys(assets) { + let Some(asset) = child_map(assets, &key) else { + continue; + }; + if map_string_opt(&asset, "content_hash").as_deref() == Some(content_hash) { + return Some(map_string_opt(&asset, "asset_id").unwrap_or(key)); + } + } + None +} diff --git a/crates/flowstate-collab/src/crdt_runtime/projection_repair_tests.rs b/crates/flowstate-collab/src/crdt_runtime/projection_repair_tests.rs new file mode 100644 index 00000000..0c9396ed --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime/projection_repair_tests.rs @@ -0,0 +1,163 @@ +//! §P2a projection repair-convergence tests. +//! +//! These exercise the runtime side of the repair pipeline: a reported defect is +//! repaired convergently, re-projecting the repaired document yields zero of that +//! defect, the repair is idempotent across passes, and two peers repairing the +//! same defect converge on identical canonical state. Also covers the §5 +//! sentinel-newline protection in the `DeleteRange` preflight. + +use anyhow::Result; +use flowstate_document::{ + MARK_PARAGRAPH_STYLE, ProjectionDefect, RunStyles, document_from_loro_with_defects, loro_schema::body_text, new_loro_document, +}; +use loro::ExportMode; + +use super::{CrdtRuntime, RepairEmission, SemanticCommand}; + +fn has_defect(defects: &[ProjectionDefect], predicate: impl Fn(&ProjectionDefect) -> bool) -> bool { + defects.iter().any(predicate) +} + +#[test] +fn constructor_repairs_orphan_object_placeholder() -> Result<()> { + let doc = new_loro_document("Orphan")?; + body_text(&doc).insert(1, "\u{FFFC}")?; + doc.commit(); + + let (_, before) = document_from_loro_with_defects(&doc)?; + assert!( + has_defect(&before, |d| matches!(d, ProjectionDefect::OrphanObjectPlaceholder { .. })), + "the stray placeholder must be reported before repair" + ); + + // Opening the runtime projects, finds the defect, and repairs it. + let runtime = CrdtRuntime::from_doc(doc, None, None)?; + assert!( + !body_text(runtime.doc()).to_string().contains('\u{FFFC}'), + "the orphan placeholder must be deleted canonically" + ); + let (_, after) = document_from_loro_with_defects(runtime.doc())?; + assert!( + !has_defect(&after, |d| matches!(d, ProjectionDefect::OrphanObjectPlaceholder { .. })), + "re-projecting the repaired document must yield zero orphan-placeholder defects" + ); + Ok(()) +} + +#[test] +fn repairs_missing_paragraph_metadata_boundary() -> Result<()> { + let doc = new_loro_document("Missing metadata")?; + let body = body_text(&doc); + let end = body.len_unicode(); + // A second paragraph boundary that carries a style mark but no durable record. + body.insert(end, "\nextra")?; + body.mark(end..end + 1, MARK_PARAGRAPH_STYLE, 0_i64)?; + doc.commit(); + + let (_, before) = document_from_loro_with_defects(&doc)?; + assert!( + has_defect(&before, |d| matches!(d, ProjectionDefect::MissingParagraphMetadata { .. })), + "the boundary without durable metadata must be reported" + ); + + let runtime = CrdtRuntime::from_doc(doc, None, None)?; + let (_, after) = document_from_loro_with_defects(runtime.doc())?; + assert!( + !has_defect(&after, |d| matches!( + d, + ProjectionDefect::MissingParagraphMetadata { .. } | ProjectionDefect::MissingParagraphBlock { .. } + )), + "the repair must write durable paragraph + block records so the defect clears" + ); + Ok(()) +} + +#[test] +fn repair_is_idempotent_across_passes() -> Result<()> { + let doc = new_loro_document("Idempotent")?; + body_text(&doc).insert(1, "\u{FFFC}")?; + doc.commit(); + + // First pass happens at construction. + let mut runtime = CrdtRuntime::from_doc(doc, None, None)?; + let body_after_first = body_text(runtime.doc()).to_string(); + + // A second explicit pass over freshly collected (now empty) defects is a no-op. + let (_, residual) = document_from_loro_with_defects(runtime.doc())?; + let events = runtime.schedule_projection_repairs(residual, RepairEmission::Streamed)?; + assert!(events.is_empty(), "a converged document must produce no further repair events"); + assert_eq!( + body_text(runtime.doc()).to_string(), + body_after_first, + "a second repair pass must not mutate an already-repaired document" + ); + Ok(()) +} + +#[test] +fn two_runtimes_repairing_same_orphan_converge() -> Result<()> { + let base = new_loro_document("Converge")?; + body_text(&base).insert(1, "\u{FFFC}")?; + base.commit(); + let base_vv = base.state_vv(); + + // Two independent forks (distinct peer ids) each repair the same orphan. + let mut peer_a = CrdtRuntime::from_doc(base.fork(), None, None)?; + let mut peer_b = CrdtRuntime::from_doc(base.fork(), None, None)?; + + // Exchange the concurrent repairs and let each converge them. + let update_a = peer_a.doc().export(ExportMode::updates(&base_vv))?; + let update_b = peer_b.doc().export(ExportMode::updates(&base_vv))?; + peer_b.import_remote_update(&update_a)?; + peer_a.import_remote_update(&update_b)?; + + let body_a = body_text(peer_a.doc()).to_string(); + let body_b = body_text(peer_b.doc()).to_string(); + assert_eq!(body_a, body_b, "concurrent repairs must converge to identical canonical body state"); + assert_eq!(body_a, "\n", "both peers must converge on the lone sentinel newline"); + + let (_, defects_a) = document_from_loro_with_defects(peer_a.doc())?; + let (_, defects_b) = document_from_loro_with_defects(peer_b.doc())?; + assert!(!has_defect(&defects_a, |d| matches!(d, ProjectionDefect::OrphanObjectPlaceholder { .. }))); + assert!(!has_defect(&defects_b, |d| matches!(d, ProjectionDefect::OrphanObjectPlaceholder { .. }))); + Ok(()) +} + +#[test] +fn delete_range_preserves_boundary_sentinel_newline() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Sentinel")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hello".to_string(), + styles: RunStyles::default(), + })?; + assert_eq!(body_text(runtime.doc()).to_string(), "\nhello"); + + // A whole-document delete that reaches position 0 is clamped so the sentinel + // survives; only the text after it is removed. + runtime.command(SemanticCommand::DeleteRange { + unicode_index: 0, + unicode_len: 6, + })?; + assert_eq!(body_text(runtime.doc()).to_string(), "\n"); + Ok(()) +} + +#[test] +fn delete_range_rejects_sentinel_only_delete() -> Result<()> { + let mut runtime = CrdtRuntime::new_empty("Sentinel")?; + runtime.command(SemanticCommand::InsertText { + unicode_index: 1, + text: "hi".to_string(), + styles: RunStyles::default(), + })?; + + // Deleting only position 0 (the sentinel) is rejected in preflight: no mutation. + let events = runtime.command(SemanticCommand::DeleteRange { + unicode_index: 0, + unicode_len: 1, + })?; + assert!(events.is_empty(), "a sentinel-only delete must be rejected, producing no events"); + assert_eq!(body_text(runtime.doc()).to_string(), "\nhi"); + Ok(()) +} diff --git a/crates/flowstate-collab/src/crdt_runtime/table_ops.rs b/crates/flowstate-collab/src/crdt_runtime/table_ops.rs new file mode 100644 index 00000000..d1652522 --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime/table_ops.rs @@ -0,0 +1,403 @@ +//! §P2b id-addressed canonical table operations. +//! +//! Every granular table structural command resolves its target by a **durable +//! id** (`RowId` / `ColumnId` / deterministic `CellId`) plus an anchor, never a +//! raw positional index against a possibly-merged order list. This is the +//! convergence-critical half of the table surface: two peers that concurrently +//! apply the same command (or replay a remote one) address the identical Loro +//! child containers, so Loro map / movable-list LWW merges rather than duplicates +//! or resolves a stale index. +//! +//! Position resolution is identical to the editor replay: `index_of(id)` scans +//! the order list; insert-after `Some(anchor)` places the new id immediately +//! after the anchor, `None` at the head, and an anchor that is present-but-absent +//! (concurrently deleted) falls back to the tail deterministically. Every op is +//! idempotent: containers are `ensure`d by their stable id, and an id is only +//! pushed into an order list when it is not already present. +//! +//! `loro_import.rs::import_table` (flowstate-document) is the reference for the +//! exact per-cell map fields; the whole-table writers in `crdt_runtime.rs` +//! (`write_table_map_from_input` / `write_table_cell_map_from_input`) share the +//! same id scheme so create-only merges never rekey. + +use anyhow::Result; +use flowstate_document::{ + BlockId, CellId, ColumnId, InputTableCell, InputTableColumnWidth, InputTableRow, RowId, TABLE_CELLS_BY_ID, TABLE_COLUMN_ORDER, + TABLE_COLUMNS_BY_ID, TABLE_ROW_ORDER, TABLE_ROWS_BY_ID, cell_loro_id, cell_loro_id_for, column_loro_id, parse_column_loro_id, + parse_row_loro_id, row_loro_id, +}; +use loro::{ContainerTrait as _, LoroDoc}; + +use super::{ + child_map, child_movable_list, empty_input_table_cell, map_keys, map_string_opt, movable_list_strings, projection_table_map_by_block_id, + update_table_cell_map_from_input, write_table_cell_map_from_input, write_table_column_width, +}; + +/// The order-list index at which an id anchored `after` should be inserted. +/// +/// `None` => head (0); `Some(anchor)` present => immediately after it; anchor +/// absent (concurrently deleted) => tail. Identical to the editor replay and the +/// predicted-projection path so optimistic and canonical state agree. +fn resolve_after_position(order: &[String], after: Option<&str>) -> usize { + match after { + None => 0, + Some(anchor) => order + .iter() + .position(|id| id == anchor) + .map_or(order.len(), |ix| ix + 1), + } +} + +/// §P2b `InsertTableRow`: insert `row_loro_id(new_row_id)` into `row_order` at the +/// anchor-resolved position (skipped if already present), ensure the row record, +/// and materialize one durable-coordinate cell per existing column from +/// `row.cells` (empty fallback for any short row). +pub(crate) fn insert_table_row( + doc: &LoroDoc, + table_block_id: BlockId, + new_row_id: RowId, + after_row: Option, + row: &InputTableRow, +) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + "skipping table row insert because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let row_order = table.ensure_mergeable_movable_list(TABLE_ROW_ORDER)?; + let column_order = table.ensure_mergeable_movable_list(TABLE_COLUMN_ORDER)?; + let rows_by_id = table.ensure_mergeable_map(TABLE_ROWS_BY_ID)?; + let cells_by_id = table.ensure_mergeable_map(TABLE_CELLS_BY_ID)?; + let column_ids = movable_list_strings(&column_order); + if column_ids.is_empty() { + tracing::warn!(?table_block_id, "skipping table row insert because the table has no columns"); + return Ok(false); + } + + let row_id_str = row_loro_id(new_row_id); + let row_order_ids = movable_list_strings(&row_order); + if !row_order_ids.iter().any(|id| id == &row_id_str) { + let pos = resolve_after_position(&row_order_ids, after_row.map(row_loro_id).as_deref()); + row_order.insert(pos.min(row_order.len()), row_id_str.as_str())?; + } + let row_map = rows_by_id.ensure_mergeable_map(&row_id_str)?; + row_map.insert("id", row_id_str.as_str())?; + let _attrs = row_map.ensure_mergeable_map("attrs")?; + + for (column_ix, column_id_str) in column_ids.iter().enumerate() { + let Some(column_id) = parse_column_loro_id(column_id_str) else { + continue; + }; + let cell_id_str = cell_loro_id_for(new_row_id, column_id); + let cell_map = cells_by_id.ensure_mergeable_map(&cell_id_str)?; + let fallback = empty_input_table_cell(new_row_id, column_id); + let cell = row.cells.get(column_ix).unwrap_or(&fallback); + write_table_cell_map_from_input(doc, &cell_map, &cell_id_str, &row_id_str, column_id_str, cell, true)?; + } + Ok(true) +} + +/// §P2b `DeleteTableRow`: remove the id from `row_order`, delete its row record, +/// and delete every deterministic-coordinate cell for the row (plus, defensively, +/// any cell whose stored `row_id` field still names it). Convergent no-op when the +/// row was already removed. +pub(crate) fn delete_table_row(doc: &LoroDoc, table_block_id: BlockId, row_id: RowId) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + "skipping table row delete because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let Some(row_order) = child_movable_list(&table, TABLE_ROW_ORDER) else { + tracing::warn!(?table_block_id, "skipping table row delete because the table has no row order"); + return Ok(false); + }; + let row_id_str = row_loro_id(row_id); + let row_ids = movable_list_strings(&row_order); + let Some(row_ix) = row_ids.iter().position(|id| id == &row_id_str) else { + return Ok(false); + }; + row_order.delete(row_ix, 1)?; + if let Some(rows_by_id) = child_map(&table, TABLE_ROWS_BY_ID) { + rows_by_id.delete(&row_id_str)?; + } + if let Some(cells_by_id) = child_map(&table, TABLE_CELLS_BY_ID) { + if let Some(column_order) = child_movable_list(&table, TABLE_COLUMN_ORDER) { + for column_id_str in movable_list_strings(&column_order) { + if let Some(column_id) = parse_column_loro_id(&column_id_str) { + cells_by_id.delete(&cell_loro_id_for(row_id, column_id))?; + } + } + } + for cell_id in map_keys(&cells_by_id) { + let orphaned = child_map(&cells_by_id, &cell_id) + .and_then(|cell| map_string_opt(&cell, "row_id")) + .as_deref() + == Some(row_id_str.as_str()); + if orphaned { + cells_by_id.delete(&cell_id)?; + } + } + } + Ok(true) +} + +/// §P2b `MoveTableRow`: `row_order.mov(index_of(row_id), target)` where `target` +/// is resolved from `after_row` exactly as an insert-after position. +pub(crate) fn move_table_row(doc: &LoroDoc, table_block_id: BlockId, row_id: RowId, after_row: Option) -> Result { + move_table_axis( + doc, + table_block_id, + TABLE_ROW_ORDER, + &row_loro_id(row_id), + after_row.map(row_loro_id).as_deref(), + ) +} + +/// §P2b `MoveTableColumn`: `column_order.mov(index_of(column_id), target)`. +pub(crate) fn move_table_column(doc: &LoroDoc, table_block_id: BlockId, column_id: ColumnId, after_column: Option) -> Result { + move_table_axis( + doc, + table_block_id, + TABLE_COLUMN_ORDER, + &column_loro_id(column_id), + after_column.map(column_loro_id).as_deref(), + ) +} + +/// Shared movable-list reorder for a row/column order list. Resolves the moved id +/// and the anchor by value (never a raw index) and computes the final Loro `mov` +/// target so that, after the move, the element sits immediately after the anchor +/// (head for `None`, tail when the anchor is absent). No-op when the id is absent +/// or already in place. +fn move_table_axis(doc: &LoroDoc, table_block_id: BlockId, order_key: &str, target_id: &str, after_id: Option<&str>) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + order_key, + "skipping table move because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let Some(order) = child_movable_list(&table, order_key) else { + tracing::warn!(?table_block_id, order_key, "skipping table move because its order list is missing"); + return Ok(false); + }; + let ids = movable_list_strings(&order); + let Some(from) = ids.iter().position(|id| id == target_id) else { + return Ok(false); + }; + // `mov(from, to)` removes the element then re-inserts it at `to` in the reduced + // list. Translate "immediately after the anchor" into that reduced index. + let target = match after_id { + None => 0, + Some(anchor) => match ids.iter().position(|id| id == anchor) { + Some(anchor_ix) if anchor_ix < from => anchor_ix + 1, + Some(anchor_ix) => anchor_ix, + None => ids.len().saturating_sub(1), + }, + }; + if target == from { + return Ok(false); + } + order.mov(from, target)?; + Ok(true) +} + +/// §P2b `InsertTableColumn`: insert `column_loro_id(new_column_id)` into +/// `column_order` at the anchor-resolved position, ensure the column record with +/// its width, and materialize one durable-coordinate cell per existing row from +/// `cells` (empty fallback for any short list). +pub(crate) fn insert_table_column( + doc: &LoroDoc, + table_block_id: BlockId, + new_column_id: ColumnId, + after_column: Option, + width: &InputTableColumnWidth, + cells: &[InputTableCell], +) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + "skipping table column insert because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let row_order = table.ensure_mergeable_movable_list(TABLE_ROW_ORDER)?; + let column_order = table.ensure_mergeable_movable_list(TABLE_COLUMN_ORDER)?; + let _rows_by_id = table.ensure_mergeable_map(TABLE_ROWS_BY_ID)?; + let columns_by_id = table.ensure_mergeable_map(TABLE_COLUMNS_BY_ID)?; + let cells_by_id = table.ensure_mergeable_map(TABLE_CELLS_BY_ID)?; + table.insert("columns_container_id", columns_by_id.id().to_string())?; + table.insert("cells_container_id", cells_by_id.id().to_string())?; + let row_ids = movable_list_strings(&row_order); + if row_ids.is_empty() { + tracing::warn!(?table_block_id, "skipping table column insert because the table has no rows"); + return Ok(false); + } + + let column_id_str = column_loro_id(new_column_id); + let column_order_ids = movable_list_strings(&column_order); + if !column_order_ids.iter().any(|id| id == &column_id_str) { + let pos = resolve_after_position(&column_order_ids, after_column.map(column_loro_id).as_deref()); + column_order.insert(pos.min(column_order.len()), column_id_str.as_str())?; + } + let column_map = columns_by_id.ensure_mergeable_map(&column_id_str)?; + column_map.insert("id", column_id_str.as_str())?; + let _attrs = column_map.ensure_mergeable_map("attrs")?; + write_table_column_width(&column_map, width)?; + + for (row_ix, row_id_str) in row_ids.iter().enumerate() { + let Some(row_id) = parse_row_loro_id(row_id_str) else { + continue; + }; + let cell_id_str = cell_loro_id_for(row_id, new_column_id); + let cell_map = cells_by_id.ensure_mergeable_map(&cell_id_str)?; + let fallback = empty_input_table_cell(row_id, new_column_id); + let cell = cells.get(row_ix).unwrap_or(&fallback); + write_table_cell_map_from_input(doc, &cell_map, &cell_id_str, row_id_str, &column_id_str, cell, true)?; + } + Ok(true) +} + +/// §P2b `DeleteTableColumn`: remove the id from `column_order`, delete its column +/// record, and delete every deterministic-coordinate cell for the column (plus, +/// defensively, any cell whose stored `column_id` field still names it). +pub(crate) fn delete_table_column(doc: &LoroDoc, table_block_id: BlockId, column_id: ColumnId) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + "skipping table column delete because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let Some(column_order) = child_movable_list(&table, TABLE_COLUMN_ORDER) else { + tracing::warn!(?table_block_id, "skipping table column delete because the table has no column order"); + return Ok(false); + }; + let column_id_str = column_loro_id(column_id); + let column_ids = movable_list_strings(&column_order); + let Some(column_ix) = column_ids.iter().position(|id| id == &column_id_str) else { + return Ok(false); + }; + column_order.delete(column_ix, 1)?; + if let Some(columns_by_id) = child_map(&table, TABLE_COLUMNS_BY_ID) { + columns_by_id.delete(&column_id_str)?; + } + if let Some(cells_by_id) = child_map(&table, TABLE_CELLS_BY_ID) { + if let Some(row_order) = child_movable_list(&table, TABLE_ROW_ORDER) { + for row_id_str in movable_list_strings(&row_order) { + if let Some(row_id) = parse_row_loro_id(&row_id_str) { + cells_by_id.delete(&cell_loro_id_for(row_id, column_id))?; + } + } + } + for cell_id in map_keys(&cells_by_id) { + let orphaned = child_map(&cells_by_id, &cell_id) + .and_then(|cell| map_string_opt(&cell, "column_id")) + .as_deref() + == Some(column_id_str.as_str()); + if orphaned { + cells_by_id.delete(&cell_id)?; + } + } + } + Ok(true) +} + +/// §P2b `ReplaceTableCell`: `ensure` the cell at the deterministic +/// `(row_id, column_id)` coordinate and rewrite its spans + flow text from `cell`. +pub(crate) fn replace_table_cell( + doc: &LoroDoc, + table_block_id: BlockId, + row_id: RowId, + column_id: ColumnId, + cell: &InputTableCell, +) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + "skipping table cell replace because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let cells_by_id = table.ensure_mergeable_map(TABLE_CELLS_BY_ID)?; + let cell_id_str = cell_loro_id(CellId::from_coordinate(row_id, column_id)); + let cell_map = cells_by_id.ensure_mergeable_map(&cell_id_str)?; + update_table_cell_map_from_input(doc, &cell_map, &cell_id_str, &row_loro_id(row_id), &column_loro_id(column_id), cell)?; + Ok(true) +} + +/// §P2b `SetTableCellSpan`: set the two span fields on the deterministic +/// `(row_id, column_id)` cell. No-op when the coordinate cell does not exist yet +/// (the topology repair materializes it, then a re-applied span converges). +pub(crate) fn set_table_cell_span( + doc: &LoroDoc, + table_block_id: BlockId, + row_id: RowId, + column_id: ColumnId, + row_span: u16, + column_span: u16, +) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + "skipping table span command because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let Some(cells_by_id) = child_map(&table, TABLE_CELLS_BY_ID) else { + return Ok(false); + }; + let Some(cell) = child_map(&cells_by_id, &cell_loro_id_for(row_id, column_id)) else { + return Ok(false); + }; + cell.insert("row_span", i64::from(row_span.max(1)))?; + cell.insert("column_span", i64::from(column_span.max(1)))?; + Ok(true) +} + +/// §P2b `SetTableColumnWidth`: the frozen command still addresses the column by +/// its position (`column_ix`), so resolve that index against `column_order` to the +/// durable column id, then write the width onto the id-keyed column record. +pub(crate) fn set_table_column_width(doc: &LoroDoc, table_block_id: BlockId, column_ix: usize, width: &InputTableColumnWidth) -> Result { + let Some(table) = projection_table_map_by_block_id(doc, table_block_id) else { + tracing::warn!( + ?table_block_id, + column_ix, + "skipping table column width command because no Loro table maps to the projected block id" + ); + return Ok(false); + }; + let Some(column_order) = child_movable_list(&table, TABLE_COLUMN_ORDER) else { + tracing::warn!( + ?table_block_id, + column_ix, + "skipping table column width command because the table has no column order" + ); + return Ok(false); + }; + let column_ids = movable_list_strings(&column_order); + let Some(column_id) = column_ids.get(column_ix) else { + tracing::warn!( + ?table_block_id, + column_ix, + "skipping table column width command because the column index is out of range" + ); + return Ok(false); + }; + let Some(columns_by_id) = child_map(&table, TABLE_COLUMNS_BY_ID) else { + tracing::warn!( + ?table_block_id, + column_ix, + "skipping table column width command because the table has no columns map" + ); + return Ok(false); + }; + let column = columns_by_id.ensure_mergeable_map(column_id)?; + write_table_column_width(&column, width)?; + Ok(true) +} diff --git a/crates/flowstate-collab/src/crdt_runtime/types.rs b/crates/flowstate-collab/src/crdt_runtime/types.rs new file mode 100644 index 00000000..531fbb3f --- /dev/null +++ b/crates/flowstate-collab/src/crdt_runtime/types.rs @@ -0,0 +1,312 @@ +use flowstate_document::{ + DocumentPackage, DocumentProjection, InputBlockAlignment, InputEquationDisplay, InputImageSizing, InputTableColumnWidth, ParagraphStyle, + ProjectionPatchBatch, ROOT_BODY_FLOW_ID, RunStyles, +}; +use gpui_flowtext::{EditorSelection, ExternalCaret, ExternalSelection}; +use loro::VersionRange; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug)] +pub struct RuntimeAssetMetadata { + pub asset_id: u128, + pub content_hash: [u8; 32], + pub mime_type: String, + pub original_name: Option, + pub byte_length: u64, +} + +#[derive(Clone, Debug)] +pub struct RuntimeRevisionInfo { + pub revision_id: u128, + pub title: String, + pub summary: String, + pub created_at_unix_secs: i64, + pub author_user_id: Option, + pub author_display_name: Option, + pub replica_id: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RuntimeCommentMessage { + pub message_id: u128, + pub author_user_id: u128, + pub author_display_name: String, + pub body: String, + pub created_at_unix_secs: i64, + pub updated_at_unix_secs: i64, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct RuntimeCommentThread { + pub comment_id: u128, + pub quoted_text: String, + pub resolved: bool, + pub created_at_unix_secs: i64, + pub updated_at_unix_secs: i64, + pub anchor: Option<(gpui_flowtext::DocumentOffset, gpui_flowtext::DocumentOffset)>, + pub messages: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct UndoSelectionSnapshot { + pub anchor_cursor: Vec, + pub head_cursor: Vec, + pub anchor_affinity: UndoSelectionAffinity, + pub head_affinity: UndoSelectionAffinity, + pub direction: UndoSelectionDirection, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub enum UndoSelectionAffinity { + Before, + After, + #[default] + Neutral, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub enum UndoSelectionDirection { + Forward, + Backward, + #[default] + None, +} + +#[derive(Debug, Default)] +pub(super) struct UndoSelectionState { + pub(super) pending_selection: Option>, + pub(super) restored_selection: Option, +} + +#[derive(Clone, Debug)] +pub enum SemanticCommand { + InsertText { + unicode_index: usize, + text: String, + styles: RunStyles, + }, + DeleteRange { + unicode_index: usize, + unicode_len: usize, + }, + SplitParagraph { + unicode_index: usize, + inherited_style: ParagraphStyle, + }, + SetParagraphStyle { + boundary_unicode_index: usize, + style: ParagraphStyle, + }, + SetRunStyles { + unicode_range: std::ops::Range, + styles: RunStyles, + }, + InsertImage { + unicode_index: usize, + asset_id: u128, + alt_text: String, + caption: Option, + sizing: InputImageSizing, + alignment: InputBlockAlignment, + }, + InsertEquation { + unicode_index: usize, + source: String, + display: InputEquationDisplay, + }, + InsertTable { + unicode_index: usize, + rows: usize, + columns: usize, + column_widths: Vec, + header_row: bool, + }, + OpenRevision { + revision_id: u128, + }, + ForkRevision { + revision_id: u128, + }, + Undo, + Redo, +} + +#[derive(Debug)] +pub enum RuntimeEvent { + LocalUpdate { + bytes: Vec, + frontier: Vec, + version_vector: Vec, + }, + RemoteUpdateApplied { + pending: Option, + frontier: Vec, + version_vector: Vec, + }, + RevisionOpened { + revision_id: u128, + document: Box, + }, + RevisionForked { + revision_id: u128, + document: Box, + package: Box, + }, + SelectionRestored { + selection: EditorSelection, + }, + ProjectionUpdated { + document: Box, + invalidation: ProjectionInvalidation, + frontier: Vec, + version_vector: Vec, + }, + ProjectionPatched { + batch: ProjectionPatchBatch, + invalidation: ProjectionInvalidation, + version_vector: Vec, + }, + /// §A12.1.3 slice 4: a remote payload depended on history below this + /// shallow session's root. It was durably merged into the PACKAGE (full + /// history) but the in-memory doc cannot hold it — the session must reopen + /// the document to see the merged state. Rare by construction (requires a + /// peer offline since before the shallow root reconnecting with unsynced + /// edits); until the reopen, the local view lacks those foreign edits. + HistoryRebaseRequired { + /// Merged PACKAGE tip (not the in-memory doc's frontier). + merged_frontier: Vec, + }, +} + +impl RuntimeEvent { + #[must_use] + pub fn frontier(&self) -> Option<&[u8]> { + match self { + Self::LocalUpdate { frontier, .. } | Self::RemoteUpdateApplied { frontier, .. } | Self::ProjectionUpdated { frontier, .. } => { + Some(frontier) + }, + Self::ProjectionPatched { batch, .. } => Some(&batch.new_frontier), + Self::RevisionOpened { document, .. } | Self::RevisionForked { document, .. } => Some(&document.frontier), + // The merged frontier is the PACKAGE tip, not this runtime's — never + // report it as a runtime frontier. + Self::SelectionRestored { .. } | Self::HistoryRebaseRequired { .. } => None, + } + } +} + +#[derive(Clone, Debug)] +pub struct RuntimePresenceCaretRequest { + pub selection: crate::presence::PresenceSelection, + pub color_rgb: u32, +} + +#[derive(Clone, Debug)] +pub struct RuntimePresenceCarets { + pub carets: Vec, + pub selections: Vec, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ProjectionFallbackStats { + pub total: u64, + pub by_reason: BTreeMap, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ProjectionInvalidation { + pub frontier_before: Vec, + pub frontier_after: Vec, + pub changed_flows: Vec, + pub changed_text_ranges: Vec, + pub changed_blocks: Vec, + pub changed_tables: Vec, + pub changed_assets: Vec, + pub changed_sections: Vec, + pub rebuild_required: bool, + pub fallback_reason: Option<&'static str>, + /// §23: set when at least one drained, in-epoch subscription summary carried a + /// remote origin (e.g. an imported update). Lets downstream consumers bias + /// toward conservative handling/telemetry without forcing a full rebuild for + /// ordinary remote text edits (the incremental remote fast paths still apply). + /// Defaults to `false` via `Default` for every existing constructor. + pub has_remote_origin: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProjectionTextRange { + pub flow_id: String, + pub unicode_start: usize, + pub unicode_len: usize, +} + +impl ProjectionInvalidation { + pub(crate) fn body_text(frontier_before: Vec, frontier_after: Vec, unicode_start: usize, unicode_len: usize) -> Self { + Self { + frontier_before, + frontier_after, + changed_flows: vec![ROOT_BODY_FLOW_ID.to_string()], + changed_text_ranges: vec![ProjectionTextRange { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + unicode_start, + unicode_len, + }], + ..Self::default() + } + } + + pub(crate) fn body_style(frontier_before: Vec, frontier_after: Vec, unicode_start: usize, unicode_len: usize) -> Self { + Self::body_text(frontier_before, frontier_after, unicode_start, unicode_len) + } + + /// §act-ten A10.2: multi-range body invalidation. A mass op (replace-all, + /// sparse restyle) touches MANY disjoint spans; a single covering span forces + /// the derive ladder to rematerialize every paragraph in between (O(doc) for + /// a whole-document replace-all). One range per touched span keeps the + /// readback proportional to the actual change set. + pub(crate) fn body_text_ranges(frontier_before: Vec, frontier_after: Vec, ranges: &[(usize, usize)]) -> Self { + Self { + frontier_before, + frontier_after, + changed_flows: vec![ROOT_BODY_FLOW_ID.to_string()], + changed_text_ranges: ranges + .iter() + .map(|&(unicode_start, unicode_len)| ProjectionTextRange { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + unicode_start, + unicode_len, + }) + .collect(), + ..Self::default() + } + } + + pub(crate) fn body_object(frontier_before: Vec, frontier_after: Vec, unicode_index: usize, block_kind: &'static str) -> Self { + Self { + frontier_before, + frontier_after, + changed_flows: vec![ROOT_BODY_FLOW_ID.to_string()], + changed_text_ranges: vec![ProjectionTextRange { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + unicode_start: unicode_index, + unicode_len: 1, + }], + changed_blocks: vec![block_kind.to_string()], + changed_tables: (block_kind == "table") + .then(|| block_kind.to_string()) + .into_iter() + .collect(), + ..Self::default() + } + } + + pub(crate) fn full_rebuild(frontier_before: Vec, frontier_after: Vec, reason: &'static str) -> Self { + Self { + frontier_before, + frontier_after, + rebuild_required: true, + fallback_reason: Some(reason), + ..Self::default() + } + } +} diff --git a/crates/flowstate-collab/src/discovery.rs b/crates/flowstate-collab/src/discovery.rs new file mode 100644 index 00000000..dea9efd0 --- /dev/null +++ b/crates/flowstate-collab/src/discovery.rs @@ -0,0 +1,307 @@ +//! Provider-neutral, signed collaboration rendezvous. + +use std::{future::Future, pin::Pin, sync::Arc}; + +use anyhow::{Result, ensure}; +use iroh::{EndpointAddr, PublicKey, Signature}; +use serde::{Deserialize, Serialize}; + +use crate::{ + SessionId, + identity::{PortableIdentitySecret, SignedProfile}, +}; + +const ADVERTISEMENT_CONTEXT: &[u8] = b"flowstate/discovery-advertisement/v1"; +const DOCUMENT_FINGERPRINT_CONTEXT: &[u8] = b"flowstate/document-discovery-fingerprint/v1"; +const ADMISSION_REQUEST_CONTEXT: &[u8] = b"flowstate/discovery-admission-request/v1"; + +/// Stable, title-free rendezvous key derived from the canonical document ID. +/// Forking a document changes that ID, so unrelated lineages do not discover +/// one another merely because they share a path or Dropbox filename. +#[must_use] +pub fn document_fingerprint(document_id: u128) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + hasher.update(DOCUMENT_FINGERPRINT_CONTEXT); + hasher.update(&document_id.to_le_bytes()); + *hasher.finalize().as_bytes() +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum DiscoveryTransport { + Dropbox, + Bluetooth, +} + +/// Short-lived rendezvous data. It deliberately contains no title and no +/// session admission bearer; discovery identifies a reachable trusted person, +/// after which the normal authorization pipeline exchanges an invite. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct DiscoveryAdvertisement { + pub identity: PublicKey, + pub device_id: u128, + pub document_fingerprint: [u8; 32], + pub session: SessionId, + pub endpoint: EndpointAddr, + pub expires_at_unix: u64, + pub profile: SignedProfile, + pub signature: Signature, +} + +/// Signed, short-lived proof sent over the encrypted direct channel when a +/// trusted peer asks for an invite. It is not an admission bearer and is safe +/// to reject without revealing document data. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct DiscoveryAdmissionRequest { + pub identity: PublicKey, + pub session: SessionId, + pub document_fingerprint: [u8; 32], + pub nonce: [u8; 32], + pub expires_at_unix: u64, + pub signature: Signature, +} + +impl DiscoveryAdmissionRequest { + #[must_use] + pub fn issue( + secret: &PortableIdentitySecret, + session: SessionId, + document_fingerprint: [u8; 32], + nonce: [u8; 32], + expires_at_unix: u64, + ) -> Self { + let identity = secret.public(); + let signature = secret.sign(&admission_request_payload( + &identity, + session, + &document_fingerprint, + &nonce, + expires_at_unix, + )); + Self { + identity, + session, + document_fingerprint, + nonce, + expires_at_unix, + signature, + } + } + + pub fn verify(&self, now_unix: u64) -> Result<()> { + ensure!(now_unix < self.expires_at_unix, "discovery admission request expired"); + self + .identity + .verify( + &admission_request_payload( + &self.identity, + self.session, + &self.document_fingerprint, + &self.nonce, + self.expires_at_unix, + ), + &self.signature, + ) + .map_err(|_| anyhow::anyhow!("discovery admission request signature is invalid")) + } +} + +impl DiscoveryAdvertisement { + #[must_use] + pub fn issue( + secret: &PortableIdentitySecret, + device_id: u128, + document_fingerprint: [u8; 32], + session: SessionId, + endpoint: EndpointAddr, + expires_at_unix: u64, + profile: SignedProfile, + ) -> Self { + let identity = secret.public(); + let payload = advertisement_payload(&identity, device_id, &document_fingerprint, session, &endpoint, expires_at_unix, &profile); + let signature = secret.sign(&payload); + Self { + identity, + device_id, + document_fingerprint, + session, + endpoint, + expires_at_unix, + profile, + signature, + } + } + + pub fn verify(&self, now_unix: u64) -> Result<()> { + ensure!(now_unix < self.expires_at_unix, "discovery advertisement expired"); + ensure!( + self.profile.identity == self.identity, + "discovery profile identity does not match its signer" + ); + ensure!(self.profile.verify(), "discovery profile signature is invalid"); + let payload = advertisement_payload( + &self.identity, + self.device_id, + &self.document_fingerprint, + self.session, + &self.endpoint, + self.expires_at_unix, + &self.profile, + ); + self + .identity + .verify(&payload, &self.signature) + .map_err(|_| anyhow::anyhow!("discovery advertisement signature is invalid")) + } +} + +/// Dropbox sidecars and OS Bluetooth adapters implement this same interface. +/// Backends only move signed advertisements; they never decide document access. +pub trait RendezvousBackend: Send + Sync { + fn transport(&self) -> DiscoveryTransport; + fn publish(&self, advertisement: DiscoveryAdvertisement) -> Pin> + Send + '_>>; + fn scan(&self, document_fingerprint: [u8; 32]) -> Pin>> + Send + '_>>; + fn clear(&self, identity: PublicKey, device_id: u128, document_fingerprint: [u8; 32]) + -> Pin> + Send + '_>>; +} + +/// Runs all configured transports through one lifecycle. A failed optional +/// transport is reported but does not discard peers found through another. +#[derive(Default)] +pub struct RendezvousSet { + backends: Vec>, +} + +impl RendezvousSet { + #[must_use] + pub fn new(backends: Vec>) -> Self { + Self { backends } + } + + pub async fn publish(&self, advertisement: DiscoveryAdvertisement) -> Vec<(DiscoveryTransport, anyhow::Error)> { + let mut failures = Vec::new(); + for backend in &self.backends { + if let Err(error) = backend.publish(advertisement.clone()).await { + failures.push((backend.transport(), error)); + } + } + failures + } + + pub async fn scan(&self, document_fingerprint: [u8; 32]) -> (Vec, Vec<(DiscoveryTransport, anyhow::Error)>) { + let mut advertisements = Vec::new(); + let mut failures = Vec::new(); + for backend in &self.backends { + match backend.scan(document_fingerprint).await { + Ok(mut found) => advertisements.append(&mut found), + Err(error) => failures.push((backend.transport(), error)), + } + } + (advertisements, failures) + } + + pub async fn clear(&self, identity: PublicKey, device_id: u128, document_fingerprint: [u8; 32]) -> Vec<(DiscoveryTransport, anyhow::Error)> { + let mut failures = Vec::new(); + for backend in &self.backends { + if let Err(error) = backend + .clear(identity, device_id, document_fingerprint) + .await + { + failures.push((backend.transport(), error)); + } + } + failures + } +} + +/// Verify, scope, and deduplicate untrusted backend results before UI code sees +/// them. `is_authorized` is the shared safety-code/squad/document policy gate. +pub fn eligible_advertisements( + advertisements: impl IntoIterator, + document_fingerprint: [u8; 32], + now_unix: u64, + mut is_authorized: impl FnMut(&PublicKey) -> bool, +) -> Vec { + let mut eligible: Vec = Vec::new(); + for advertisement in advertisements { + if advertisement.document_fingerprint != document_fingerprint + || advertisement.verify(now_unix).is_err() + || !is_authorized(&advertisement.identity) + { + continue; + } + if let Some(index) = eligible + .iter() + .position(|existing| existing.identity == advertisement.identity && existing.device_id == advertisement.device_id) + { + if eligible[index].expires_at_unix < advertisement.expires_at_unix { + eligible[index] = advertisement; + } + } else { + eligible.push(advertisement); + } + } + eligible +} + +fn advertisement_payload( + identity: &PublicKey, + device_id: u128, + document_fingerprint: &[u8; 32], + session: SessionId, + endpoint: &EndpointAddr, + expires_at_unix: u64, + profile: &SignedProfile, +) -> Vec { + postcard::to_stdvec(&( + ADVERTISEMENT_CONTEXT, + identity, + device_id, + document_fingerprint, + session, + endpoint, + expires_at_unix, + profile, + )) + .expect("discovery advertisement signing payload should serialize") +} + +fn admission_request_payload( + identity: &PublicKey, + session: SessionId, + document_fingerprint: &[u8; 32], + nonce: &[u8; 32], + expires_at_unix: u64, +) -> Vec { + postcard::to_stdvec(&(ADMISSION_REQUEST_CONTEXT, identity, session, document_fingerprint, nonce, expires_at_unix)) + .expect("discovery admission request signing payload should serialize") +} + +#[cfg(test)] +mod tests { + use super::*; + use iroh::SecretKey; + + fn advertisement(secret: &PortableIdentitySecret, device_id: u128, expires_at_unix: u64) -> DiscoveryAdvertisement { + let profile = secret.sign_profile(1, "Alex".into(), 0x123456, None); + DiscoveryAdvertisement::issue( + secret, + device_id, + [7; 32], + SessionId::from_bytes([8; 32]), + EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public()), + expires_at_unix, + profile, + ) + } + + #[test] + fn advertisements_are_signed_scoped_expiring_and_deduplicated() { + let secret = PortableIdentitySecret::generate(); + let stale = advertisement(&secret, 3, 20); + let fresh = advertisement(&secret, 3, 30); + let result = eligible_advertisements([stale, fresh.clone()], [7; 32], 10, |identity| *identity == secret.public()); + assert_eq!(result, vec![fresh]); + assert!(eligible_advertisements([advertisement(&secret, 4, 10)], [7; 32], 10, |_| true).is_empty()); + assert!(eligible_advertisements([advertisement(&secret, 5, 30)], [6; 32], 10, |_| true).is_empty()); + } +} diff --git a/crates/flowstate-collab/src/doc_io.rs b/crates/flowstate-collab/src/doc_io.rs new file mode 100644 index 00000000..36a3b58a --- /dev/null +++ b/crates/flowstate-collab/src/doc_io.rs @@ -0,0 +1,901 @@ +//! `DocIoService` — the background document I/O service (Loro-first spec §3/§6). +//! +//! Replaces the old CRDT actor. The critical difference: this service does NOT +//! own the document and local edits never pass through it. It shares the doc +//! core with the UI thread's `LocalDocHandle` through the write gate, and +//! acquires that gate for every doc-touching operation (I-9a — every Loro call +//! is a potential commit barrier). Its duties: +//! +//! * remote import chunks (coalesced under one gate hold, spec §6.4); +//! * draining + broadcasting the publish queue filled by local commits; +//! * update/snapshot exports (snapshots fork under the gate, export off it); +//! * presence encode/resolve (gate-held, coalescible); +//! * package/checkpoint/revision/asset services. +//! +//! Local typing can stall behind this service for AT MOST one gate slice; the +//! per-slice hold budget is enforced by the gate's measured records (§11). + +use std::io; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::{Context as _, Result}; +use async_channel::{Receiver, Sender}; + +/// Reply channel alias (clippy type-complexity). +type ReplySender = Sender>; +use flowstate_document::DocumentProjection; +use loro::{ExportMode, VersionVector}; + +use crate::crdt_runtime::{ + CrdtRuntime, RuntimeAssetMetadata, RuntimeCommentThread, RuntimeEvent, RuntimePresenceCaretRequest, RuntimePresenceCarets, + RuntimeRevisionInfo, UndoSelectionSnapshot, +}; +use crate::local_write::{GateHolder, WriteGate}; +use crate::presence::PresenceSelection; + +const IO_REQUEST_CHANNEL_CAPACITY: usize = 256; +/// Maximum queued import blobs coalesced into one gate hold (spec §6.4). The +/// gate hold-time budget is the real bound; this caps pathological queues. +const IMPORT_COALESCE_MAX: usize = 16; + +pub enum IoRequest { + ImportRemoteUpdate { + bytes: Vec, + reply: Sender>>, + }, + /// Drain the publish queue filled by gate-held local commits (`LocalUpdate` + /// bytes etc.) for network broadcast. + PumpPublish { + reply: Sender>>, + }, + ProjectionSnapshot { + reply: Sender>, + }, + OplogVersionVector { + reply: Sender>>, + }, + ExportUpdatesFor { + remote_vv: Vec, + reply: Sender>>, + }, + SnapshotBytes { + reply: Sender>>, + }, + CheckpointPackage { + title: String, + path: Option, + reply: Sender>>, + }, + PackageBytes { + title: String, + reply: Sender>>, + }, + SavePackageTo { + path: PathBuf, + reply: Sender>, + }, + /// §P3 (act two): revision-less cache flush at document close/idle so the + /// next preview/open of this package takes the projection-cache fast path. + FlushPackageCaches { + reply: Sender>, + }, + TakeRestoredUndoSelection { + reply: Sender>>, + }, + AssetMetadata { + reply: Sender>>, + }, + Revisions { + reply: Sender>>, + }, + Comments { + reply: Sender>>, + }, + CreateComment { + selection: gpui_flowtext::EditorSelection, + body: String, + author_user_id: u128, + author_display_name: String, + reply: Sender>, + }, + ReplyToComment { + comment_id: u128, + body: String, + author_user_id: u128, + author_display_name: String, + reply: Sender>, + }, + SetCommentResolved { + comment_id: u128, + resolved: bool, + reply: Sender>, + }, + EditCommentMessage { + comment_id: u128, + message_id: u128, + body: String, + actor_user_id: u128, + reply: Sender>, + }, + DeleteComment { + comment_id: u128, + actor_user_id: u128, + reply: Sender>, + }, + ProjectionFallbackStats { + reply: Sender>, + }, + PresenceSelection { + selection: gpui_flowtext::EditorSelection, + reply: Sender>>, + }, + ResolvePresenceCarets { + requests: Vec, + reply: Sender>, + }, + SetAuthorIdentity { + user_id: u128, + display_name: Option, + reply: Sender>>, + }, + /// Open a named revision read-only (projection of historical state). + OpenRevision { + revision_id: u128, + reply: Sender>>, + }, + /// Fork a named revision into a fresh document package. + ForkRevision { + revision_id: u128, + reply: Sender>>, + }, + /// Record fetched asset bytes into canonical state (joiner asset pulls). + RecordAssets { + assets: Vec, + reply: Sender>>, + }, +} + +fn io_request_kind(request: &IoRequest) -> &'static str { + match request { + IoRequest::ImportRemoteUpdate { .. } => "import-remote-update", + IoRequest::PumpPublish { .. } => "pump-publish", + IoRequest::ProjectionSnapshot { .. } => "projection-snapshot", + IoRequest::OplogVersionVector { .. } => "oplog-version-vector", + IoRequest::ExportUpdatesFor { .. } => "export-updates-for", + IoRequest::SnapshotBytes { .. } => "snapshot-bytes", + IoRequest::CheckpointPackage { .. } => "checkpoint-package", + IoRequest::PackageBytes { .. } => "package-bytes", + IoRequest::SavePackageTo { .. } => "save-package-to", + IoRequest::FlushPackageCaches { .. } => "flush-package-caches", + IoRequest::TakeRestoredUndoSelection { .. } => "take-restored-undo-selection", + IoRequest::AssetMetadata { .. } => "asset-metadata", + IoRequest::Revisions { .. } => "revisions", + IoRequest::Comments { .. } => "comments", + IoRequest::CreateComment { .. } => "create-comment", + IoRequest::ReplyToComment { .. } => "reply-to-comment", + IoRequest::SetCommentResolved { .. } => "set-comment-resolved", + IoRequest::EditCommentMessage { .. } => "edit-comment-message", + IoRequest::DeleteComment { .. } => "delete-comment", + IoRequest::ProjectionFallbackStats { .. } => "projection-fallback-stats", + IoRequest::PresenceSelection { .. } => "presence-selection", + IoRequest::ResolvePresenceCarets { .. } => "resolve-presence-carets", + IoRequest::SetAuthorIdentity { .. } => "set-author-identity", + IoRequest::OpenRevision { .. } => "open-revision", + IoRequest::ForkRevision { .. } => "fork-revision", + IoRequest::RecordAssets { .. } => "record-assets", + } +} + +/// Cloneable handle to the I/O service thread. +#[derive(Clone)] +pub struct DocIoHandle { + requests: Sender, +} + +impl DocIoHandle { + /// Spawn the I/O service over a shared, gate-protected document core. + pub fn spawn(core: Arc>) -> io::Result { + let (sender, receiver) = async_channel::bounded(IO_REQUEST_CHANNEL_CAPACITY); + std::thread::Builder::new() + .name("flowstate-doc-io".to_string()) + .spawn(move || { + // §A13.1 (Act 13): Loro's state store decodes LAZILY — a + // shallow-opened doc pays ~65ms of rich-text tree materialization on + // its FIRST body-text access, which would otherwise land on the + // user's first keystroke. Warm it here, off the interactive path, + // right as the pump starts (the A13.2 priority lane lets a racing + // local intent preempt the acquisition; the hold itself is the + // one-time materialization we are choosing to pay early). + if let Ok(guard) = core.lock(GateHolder::DocumentService) { + let body = flowstate_document::loro_schema::body_text(guard.doc()); + body.iter(|_| true); + } + io_loop(&core, &receiver); + })?; + Ok(Self { requests: sender }) + } + + async fn request(&self, build: impl FnOnce(Sender>) -> IoRequest) -> Result { + let (reply, response) = async_channel::bounded(1); + self + .requests + .send(build(reply)) + .await + .map_err(|_| anyhow::anyhow!("doc I/O service stopped"))?; + response + .recv() + .await + .map_err(|_| anyhow::anyhow!("doc I/O service dropped the reply"))? + } + + pub async fn import_remote_update(&self, bytes: Vec) -> Result> { + self + .request(|reply| IoRequest::ImportRemoteUpdate { bytes, reply }) + .await + } + + /// Drain committed-but-unpublished local events (call after local intents, + /// debounced by the session). + pub async fn pump_publish(&self) -> Result> { + self.request(|reply| IoRequest::PumpPublish { reply }).await + } + + pub async fn projection_snapshot(&self) -> Result { + self + .request(|reply| IoRequest::ProjectionSnapshot { reply }) + .await + } + + pub async fn oplog_version_vector(&self) -> Result> { + self + .request(|reply| IoRequest::OplogVersionVector { reply }) + .await + } + + pub async fn export_updates_for(&self, remote_vv: Vec) -> Result> { + self + .request(|reply| IoRequest::ExportUpdatesFor { remote_vv, reply }) + .await + } + + pub async fn snapshot_bytes(&self) -> Result> { + self + .request(|reply| IoRequest::SnapshotBytes { reply }) + .await + } + + pub async fn checkpoint_package(&self, title: String, path: Option) -> Result> { + self + .request(|reply| IoRequest::CheckpointPackage { title, path, reply }) + .await + } + + pub async fn package_bytes(&self, title: String) -> Result> { + self + .request(|reply| IoRequest::PackageBytes { title, reply }) + .await + } + + pub async fn save_package_to(&self, path: PathBuf) -> Result<()> { + self + .request(|reply| IoRequest::SavePackageTo { path, reply }) + .await + } + + pub async fn flush_package_caches(&self) -> Result<()> { + self + .request(|reply| IoRequest::FlushPackageCaches { reply }) + .await + } + + pub async fn take_restored_undo_selection(&self) -> Result> { + self + .request(|reply| IoRequest::TakeRestoredUndoSelection { reply }) + .await + } + + pub async fn asset_metadata(&self) -> Result> { + self + .request(|reply| IoRequest::AssetMetadata { reply }) + .await + } + + pub async fn revisions(&self) -> Result> { + self.request(|reply| IoRequest::Revisions { reply }).await + } + + pub async fn comments(&self) -> Result> { + self.request(|reply| IoRequest::Comments { reply }).await + } + + pub async fn create_comment( + &self, + selection: gpui_flowtext::EditorSelection, + body: String, + author_user_id: u128, + author_display_name: String, + ) -> Result { + self + .request(|reply| IoRequest::CreateComment { + selection, + body, + author_user_id, + author_display_name, + reply, + }) + .await + } + + pub async fn reply_to_comment(&self, comment_id: u128, body: String, author_user_id: u128, author_display_name: String) -> Result { + self + .request(|reply| IoRequest::ReplyToComment { + comment_id, + body, + author_user_id, + author_display_name, + reply, + }) + .await + } + + pub async fn set_comment_resolved(&self, comment_id: u128, resolved: bool) -> Result<()> { + self + .request(|reply| IoRequest::SetCommentResolved { comment_id, resolved, reply }) + .await + } + + pub async fn edit_comment_message(&self, comment_id: u128, message_id: u128, body: String, actor_user_id: u128) -> Result<()> { + self + .request(|reply| IoRequest::EditCommentMessage { + comment_id, + message_id, + body, + actor_user_id, + reply, + }) + .await + } + + pub async fn delete_comment(&self, comment_id: u128, actor_user_id: u128) -> Result<()> { + self + .request(|reply| IoRequest::DeleteComment { + comment_id, + actor_user_id, + reply, + }) + .await + } + + pub async fn projection_fallback_stats(&self) -> Result { + self + .request(|reply| IoRequest::ProjectionFallbackStats { reply }) + .await + } + + pub async fn presence_selection(&self, selection: gpui_flowtext::EditorSelection) -> Result> { + self + .request(|reply| IoRequest::PresenceSelection { selection, reply }) + .await + } + + pub async fn resolve_presence_carets(&self, requests: Vec) -> Result { + self + .request(|reply| IoRequest::ResolvePresenceCarets { requests, reply }) + .await + } + + pub async fn open_revision(&self, revision_id: u128) -> Result> { + self + .request(|reply| IoRequest::OpenRevision { revision_id, reply }) + .await + } + + pub async fn fork_revision(&self, revision_id: u128) -> Result> { + self + .request(|reply| IoRequest::ForkRevision { revision_id, reply }) + .await + } + + /// Record fetched asset bytes into canonical state so saves/packages made by + /// this replica include assets it pulled from peers. + pub async fn record_assets(&self, assets: Vec) -> Result> { + self + .request(|reply| IoRequest::RecordAssets { assets, reply }) + .await + } + + pub async fn set_author_identity(&self, user_id: u128, display_name: Option) -> Result> { + self + .request(|reply| IoRequest::SetAuthorIdentity { + user_id, + display_name, + reply, + }) + .await + } +} + +fn send_reply(reply: &Sender>, value: Result) { + if let Err(error) = reply.try_send(value) { + tracing::warn!(%error, "doc I/O reply channel dropped before the reply was sent"); + } +} + +fn io_loop(core: &Arc>, receiver: &Receiver) { + // Buffered non-import requests popped while coalescing an import chunk. + let mut deferred: Vec = Vec::new(); + // Bytes imported since the diff-calculator/history caches were last freed. + // Bursts free at chunk end; a steady one-blob drip previously NEVER freed + // (the old `coalesced > 1` gate), so the caches grew without bound across a + // session — part of the receiving peer's 13.1 GB/run import churn. + let mut unfreed_import_bytes: usize = 0; + const IMPORT_CACHE_FREE_BYTES: usize = 4 * 1024 * 1024; + // §perf-heaven T8.18 (first cold import): warm the retained import calculator's + // trackers ONCE when this doc first shows collaboration activity, so the first + // remote import reuses the built index rather than paying the cold tracker + // build synchronously on the receive path. Gated so a single-user doc never + // pays for it. + let mut import_calc_warmed = false; + loop { + let request = if let Some(request) = deferred.pop() { + request + } else { + match receiver.recv_blocking() { + Ok(request) => request, + Err(_) => break, + } + }; + let kind = io_request_kind(&request); + let started = std::time::Instant::now(); + match request { + IoRequest::ImportRemoteUpdate { bytes, reply } => { + // Spec §6.4: coalesce immediately-available import blobs into ONE gate + // acquisition. Non-import requests popped while draining are deferred + // (processed right after, order preserved among themselves). + // §A14.1.1: the coalescer respects the sender's mass-op slicing — a + // BYTE budget stops it from gluing chunked slices back into one + // mass hold (which would resurrect the ~240ms typing stall the + // chunking exists to kill). Small drips still coalesce as before. + const IMPORT_COALESCE_BYTE_BUDGET: usize = 64 * 1024; + let mut coalesced_bytes = bytes.len(); + let mut chunk: Vec<(Vec, ReplySender>)> = vec![(bytes, reply)]; + while chunk.len() < IMPORT_COALESCE_MAX && coalesced_bytes < IMPORT_COALESCE_BYTE_BUDGET { + match receiver.try_recv() { + Ok(IoRequest::ImportRemoteUpdate { bytes, reply }) => { + coalesced_bytes += bytes.len(); + chunk.push((bytes, reply)); + }, + Ok(other) => { + deferred.push(other); + break; + }, + Err(_) => break, + } + } + match core.lock(GateHolder::ImportChunk) { + Ok(mut guard) => { + // §act-five P1.A: a SECOND drain after the gate is held. Acquiring the + // gate can BLOCK behind a checkpoint/save; every remote blob that + // arrived during that wait is now queued. Folding them into this same + // chunk makes a steady remote drip coalesce into ONE import + ONE + // derive (each import pays a fixed per-call rich-text tracker rebuild, + // so fewer imports is a direct win) — with NO added latency, since we + // were already blocked on the gate. + while chunk.len() < IMPORT_COALESCE_MAX && coalesced_bytes < IMPORT_COALESCE_BYTE_BUDGET { + match receiver.try_recv() { + Ok(IoRequest::ImportRemoteUpdate { bytes, reply }) => { + coalesced_bytes += bytes.len(); + chunk.push((bytes, reply)); + }, + Ok(other) => { + deferred.push(other); + break; + }, + Err(_) => break, + } + } + let coalesced = chunk.len(); + // §6.4 + field fix 2026-07-07: ONE batched import for the whole + // chunk — N cheap Loro imports, then ONE projection derive — where + // the old shape ran a full derive per blob (41.6% of a receiving + // peer's runtime). + let blobs: Vec<&[u8]> = chunk.iter().map(|(bytes, _)| bytes.as_slice()).collect(); + unfreed_import_bytes += blobs.iter().map(|bytes| bytes.len()).sum::(); + match guard.import_remote_updates(&blobs) { + Ok(event_batches) => { + for ((_, reply), events) in chunk.iter().zip(event_batches) { + send_reply(reply, Ok(events)); + } + }, + Err(error) => { + let error = format!("{error:#}"); + for (_, reply) in &chunk { + send_reply(reply, Err(anyhow::anyhow!("batched remote import failed: {error}"))); + } + }, + } + // Spec §6.4 memory hygiene: imports build diff-calculator and + // history caches; free them at burst end OR once a steady drip + // has accumulated a real footprint (OOM history). Cheap no-ops + // when the caches are cold. + if coalesced > 1 || unfreed_import_bytes >= IMPORT_CACHE_FREE_BYTES { + guard.doc().free_diff_calculator(); + guard.doc().free_history_cache(); + unfreed_import_bytes = 0; + } + if coalesced > 1 { + tracing::debug!(coalesced, "coalesced import chunk under one gate hold"); + } + }, + Err(poisoned) => { + for (_, reply) in chunk { + send_reply(&reply, Err(anyhow::anyhow!(poisoned))); + } + }, + } + }, + IoRequest::PumpPublish { reply } => { + let result = core + .lock(GateHolder::ExportUpdates) + .map(|mut guard| { + // §perf-heaven T8.18: the first publish marks this doc as an ACTIVE + // collaboration participant. Warm the retained import calculator now + // (off the receive path) so the FIRST remote import reuses the + // id_to_cursor index instead of building it cold. Safe: a stale warmed + // tracker is rebuilt by `start_tracking`'s validity guard, so warming + // can only save work, never change a computed diff. + if !import_calc_warmed { + guard.doc().inner().warm_import_diff_calculator(); + import_calc_warmed = true; + } + guard.take_pending_publish() + }) + .map_err(|poisoned| anyhow::anyhow!(poisoned)); + send_reply(&reply, result); + }, + IoRequest::ProjectionSnapshot { reply } => send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| runtime.projection_snapshot()), + ), + IoRequest::OplogVersionVector { reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::ExportUpdates, |runtime| Ok(runtime.doc().oplog_vv().encode())), + ); + }, + IoRequest::ExportUpdatesFor { remote_vv, reply } => { + let result = VersionVector::decode(&remote_vv) + .context("decoding remote Loro version vector") + .and_then(|vv| gate_call(core, GateHolder::ExportUpdates, |runtime| runtime.export_updates_for(&vv))); + send_reply(&reply, result); + }, + IoRequest::SnapshotBytes { reply } => { + // I-9a long-export rule: fork under the gate (brief), export the fork + // off-gate so a large snapshot never stalls typing. + // + // §A12.1.3: a SHALLOW session's fork exports a silently shallow + // snapshot — a joiner bootstrapped from it would be shallow AND + // package-less, unable to absorb below-root history later. Clone the + // package under the gate too (memcpy-scale, rare — joins only) and + // reconstruct the FULL doc off-gate so joiners keep full history. + let forked = gate_call(core, GateHolder::ExportFork, |runtime| { + let package = runtime + .doc() + .is_shallow() + .then(|| runtime.package().cloned()) + .flatten(); + Ok((runtime.doc().fork(), package)) + }); + let result = forked.and_then(|(fork, package)| { + let source = match package { + Some(package) => package + .reconstruct_full_doc(&fork) + .context("reconstructing full history for a join snapshot")?, + None => fork, + }; + source + .export(ExportMode::Snapshot) + .context("exporting Loro snapshot from fork") + }); + send_reply(&reply, result); + }, + IoRequest::CheckpointPackage { title, path, reply } => { + // Field fix 2026-07-07: split checkpoint — live-doc phase under a + // short gate hold, heavy assembly + disk write on a detached worker, + // package restored under the gate afterwards. First-save (no package + // yet) falls back to the in-place path. + let begun = gate_call(core, GateHolder::DocumentService, |runtime| { + runtime + .begin_checkpoint(&title, path.clone()) + .map_err(anyhow::Error::from) + }); + match begun { + Ok(Some((job, events))) => { + let core = Arc::clone(core); + if let Err(error) = std::thread::Builder::new() + .name("flowstate-checkpoint".to_string()) + .spawn(move || { + let (package, wrote) = job.run(); + let restore = core + .lock(GateHolder::DocumentService) + .map(|mut guard| match wrote { + Ok(wrote) => { + guard.finish_checkpoint(package, wrote); + Ok(()) + }, + Err(error) => { + guard.finish_checkpoint(package, false); + Err(anyhow::Error::from(error)) + }, + }); + match restore { + Ok(Ok(())) => send_reply(&reply, Ok(events)), + Ok(Err(error)) => send_reply(&reply, Err(error)), + Err(poisoned) => send_reply(&reply, Err(anyhow::anyhow!(poisoned))), + } + }) + { + tracing::error!(%error, "spawning checkpoint worker failed"); + } + }, + Ok(None) => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime + .checkpoint_package(&title, path) + .map_err(anyhow::Error::from) + }), + ); + }, + Err(error) => send_reply(&reply, Err(error)), + } + }, + IoRequest::PackageBytes { title, reply } => { + // Field fix 2026-07-07: fork under a SHORT gate hold, assemble the + // package on a detached worker. The 18s whole-assembly-under-gate + // holds in the field logs froze typing and starved imports. + match gate_call(core, GateHolder::ExportFork, |runtime| Ok(runtime.package_export_context())) { + Ok((fork, projection)) => { + let title = title.clone(); + if let Err(error) = std::thread::Builder::new() + .name("flowstate-package-export".to_string()) + .spawn(move || { + let result = crate::crdt_runtime::assemble_package_bytes(&fork, &projection, &title).map_err(anyhow::Error::from); + send_reply(&reply, result); + }) + { + tracing::error!(%error, "spawning package export worker failed"); + } + }, + Err(error) => send_reply(&reply, Err(error)), + } + }, + IoRequest::SavePackageTo { path, reply } => { + // Save-as = full checkpoint targeting the new path (split path); the + // in-place fallback covers package-less first saves. + let begun = gate_call(core, GateHolder::DocumentService, |runtime| { + runtime + .begin_checkpoint("Save", Some(path.clone())) + .map_err(anyhow::Error::from) + }); + match begun { + Ok(Some((job, _events))) => { + let core = Arc::clone(core); + if let Err(error) = std::thread::Builder::new() + .name("flowstate-save".to_string()) + .spawn(move || { + let (package, wrote) = job.run(); + let restore = core + .lock(GateHolder::DocumentService) + .map(|mut guard| match wrote { + Ok(wrote) => { + guard.finish_checkpoint(package, wrote); + Ok(()) + }, + Err(error) => { + guard.finish_checkpoint(package, false); + Err(anyhow::Error::from(error)) + }, + }); + match restore { + Ok(result) => send_reply(&reply, result), + Err(poisoned) => send_reply(&reply, Err(anyhow::anyhow!(poisoned))), + } + }) + { + tracing::error!(%error, "spawning save worker failed"); + } + }, + Ok(None) => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.save_package_to(path).map_err(anyhow::Error::from) + }), + ); + }, + Err(error) => send_reply(&reply, Err(error)), + } + }, + IoRequest::FlushPackageCaches { reply } => { + // Same split shape as SavePackageTo: brief gate hold to fork + + // check out the package, heavy assembly + write off-thread, restore + // under the gate. `None` = no package or cache already current. + let begun = gate_call(core, GateHolder::DocumentService, |runtime| Ok(runtime.begin_cache_flush())); + match begun { + Ok(Some(job)) => { + let core = Arc::clone(core); + if let Err(error) = std::thread::Builder::new() + .name("flowstate-cache-flush".to_string()) + .spawn(move || { + let (package, wrote) = job.run(); + let restore = core + .lock(GateHolder::DocumentService) + .map(|mut guard| match wrote { + Ok(wrote) => { + guard.finish_checkpoint(package, wrote); + Ok(()) + }, + Err(error) => { + guard.finish_checkpoint(package, false); + Err(anyhow::Error::from(error)) + }, + }); + match restore { + Ok(result) => send_reply(&reply, result), + Err(poisoned) => send_reply(&reply, Err(anyhow::anyhow!(poisoned))), + } + }) + { + tracing::error!(%error, "spawning cache-flush worker failed"); + } + }, + Ok(None) => send_reply(&reply, Ok(())), + Err(error) => send_reply(&reply, Err(error)), + } + }, + IoRequest::TakeRestoredUndoSelection { reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| Ok(runtime.take_restored_undo_selection())), + ); + }, + IoRequest::AssetMetadata { reply } => { + send_reply(&reply, gate_call(core, GateHolder::DocumentService, |runtime| runtime.asset_metadata())); + }, + IoRequest::Revisions { reply } => { + send_reply(&reply, gate_call(core, GateHolder::DocumentService, |runtime| Ok(runtime.revisions()))); + }, + IoRequest::Comments { reply } => { + send_reply(&reply, gate_call(core, GateHolder::DocumentService, |runtime| Ok(runtime.comments()))); + }, + IoRequest::CreateComment { + selection, + body, + author_user_id, + author_display_name, + reply, + } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.create_comment(&selection, &body, author_user_id, &author_display_name) + }), + ); + }, + IoRequest::ReplyToComment { + comment_id, + body, + author_user_id, + author_display_name, + reply, + } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.reply_to_comment(comment_id, &body, author_user_id, &author_display_name) + }), + ); + }, + IoRequest::SetCommentResolved { comment_id, resolved, reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.set_comment_resolved(comment_id, resolved) + }), + ); + }, + IoRequest::EditCommentMessage { + comment_id, + message_id, + body, + actor_user_id, + reply, + } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.edit_comment_message(comment_id, message_id, &body, actor_user_id) + }), + ); + }, + IoRequest::DeleteComment { + comment_id, + actor_user_id, + reply, + } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.delete_comment(comment_id, actor_user_id) + }), + ); + }, + IoRequest::ProjectionFallbackStats { reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| Ok(runtime.projection_fallback_stats())), + ); + }, + IoRequest::PresenceSelection { selection, reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::Presence, |runtime| Ok(runtime.presence_selection(&selection))), + ); + }, + IoRequest::ResolvePresenceCarets { requests, reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::Presence, |runtime| Ok(runtime.resolve_presence_carets(requests))), + ); + }, + IoRequest::SetAuthorIdentity { + user_id, + display_name, + reply, + } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.set_author_identity(user_id, display_name) + }), + ); + }, + IoRequest::OpenRevision { revision_id, reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.command(crate::crdt_runtime::SemanticCommand::OpenRevision { revision_id }) + }), + ); + }, + IoRequest::ForkRevision { revision_id, reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| { + runtime.command(crate::crdt_runtime::SemanticCommand::ForkRevision { revision_id }) + }), + ); + }, + IoRequest::RecordAssets { assets, reply } => { + send_reply( + &reply, + gate_call(core, GateHolder::DocumentService, |runtime| runtime.merge_asset_records(assets)), + ); + }, + } + let elapsed_ms = started.elapsed().as_millis(); + if elapsed_ms > 250 { + tracing::error!(kind, elapsed_ms, "slow doc I/O request"); + } + } +} + +fn gate_call(core: &Arc>, holder: GateHolder, call: impl FnOnce(&mut CrdtRuntime) -> Result) -> Result { + let mut guard = core + .lock(holder) + .map_err(|poisoned| anyhow::anyhow!(poisoned))?; + call(&mut guard) +} diff --git a/crates/flowstate-collab/src/dropbox.rs b/crates/flowstate-collab/src/dropbox.rs new file mode 100644 index 00000000..f062b852 --- /dev/null +++ b/crates/flowstate-collab/src/dropbox.rs @@ -0,0 +1,616 @@ +//! Dropbox-backed rendezvous sidecars and revision-conditional checkpoints. +//! +//! The API client intentionally has no opinion about trust or admission. It +//! stores signed discovery records and opaque document bytes; callers still +//! run [`crate::discovery::eligible_advertisements`] before showing peers. + +use std::{ + future::Future, + pin::Pin, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{Context as _, Result, anyhow, bail, ensure}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; +use iroh::PublicKey; +use rand::RngCore as _; +use reqwest::{Client, Response, StatusCode, header}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest as _, Sha256}; +use tokio::sync::Mutex; +use url::Url; + +use crate::discovery::{DiscoveryAdvertisement, DiscoveryTransport, RendezvousBackend}; + +const API_BASE: &str = "https://api.dropboxapi.com/2"; +const CONTENT_BASE: &str = "https://content.dropboxapi.com/2"; +const TOKEN_URL: &str = "https://api.dropboxapi.com/oauth2/token"; +const API_ARG: &str = "Dropbox-API-Arg"; +const API_RESULT: &str = "Dropbox-API-Result"; +/// Large checkpoints upload/download in one request, so there is no overall +/// request deadline — only bounded connect and read-idle times, so a dead +/// connection can never hang a discovery refresh or checkpoint save forever. +const HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(15); +const HTTP_READ_TIMEOUT: Duration = Duration::from_secs(60); +/// Upper bound on a downloaded checkpoint/advertisement body. Snapshots are +/// zstd-compressed and orders of magnitude smaller than this in practice. +const MAX_DOWNLOAD_LEN: u64 = 256 * 1024 * 1024; + +fn http_client() -> Client { + Client::builder() + .connect_timeout(HTTP_CONNECT_TIMEOUT) + .read_timeout(HTTP_READ_TIMEOUT) + .build() + .unwrap_or_default() +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct DropboxCredentials { + pub app_key: String, + pub access_token: String, + pub refresh_token: Option, + /// Unix seconds. `None` also supports manually generated development tokens. + pub access_token_expires_at: Option, +} + +/// Transient state for Dropbox's desktop OAuth code flow. The verifier and +/// state must be held only until the custom-URL callback arrives. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DropboxPkceFlow { + pub authorization_url: Url, + pub code_verifier: String, + pub state: String, + pub redirect_uri: String, + pub app_key: String, +} + +impl DropboxPkceFlow { + pub fn begin(app_key: impl Into, redirect_uri: impl Into) -> Result { + let app_key = app_key.into(); + let redirect_uri = redirect_uri.into(); + ensure!(!app_key.trim().is_empty(), "Dropbox app key is missing"); + let mut verifier_bytes = [0_u8; 64]; + rand::rng().fill_bytes(&mut verifier_bytes); + let code_verifier = URL_SAFE_NO_PAD.encode(verifier_bytes); + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(code_verifier.as_bytes())); + let mut state_bytes = [0_u8; 24]; + rand::rng().fill_bytes(&mut state_bytes); + let state = URL_SAFE_NO_PAD.encode(state_bytes); + let mut authorization_url = Url::parse("https://www.dropbox.com/oauth2/authorize")?; + authorization_url + .query_pairs_mut() + .append_pair("client_id", &app_key) + .append_pair("response_type", "code") + .append_pair("redirect_uri", &redirect_uri) + .append_pair("code_challenge_method", "S256") + .append_pair("code_challenge", &challenge) + .append_pair("token_access_type", "offline") + .append_pair("state", &state); + Ok(Self { + authorization_url, + code_verifier, + state, + redirect_uri, + app_key, + }) + } + + pub fn verify_callback(&self, callback: &Url) -> Result { + let query: std::collections::HashMap<_, _> = callback.query_pairs().collect(); + ensure!( + query.get("state").is_some_and(|state| state == &self.state), + "Dropbox authorization state did not match" + ); + if let Some(error) = query + .get("error_description") + .or_else(|| query.get("error")) + { + bail!("Dropbox authorization was declined: {error}"); + } + query + .get("code") + .map(|code| code.to_string()) + .context("Dropbox authorization callback omitted its code") + } + + pub async fn exchange(&self, callback: &Url) -> Result { + let code = self.verify_callback(callback)?; + let response = http_client() + .post(TOKEN_URL) + .form(&[ + ("grant_type", "authorization_code"), + ("code", code.as_str()), + ("client_id", self.app_key.as_str()), + ("code_verifier", self.code_verifier.as_str()), + ("redirect_uri", self.redirect_uri.as_str()), + ]) + .send() + .await + .context("exchanging Dropbox authorization code")?; + let response = checked(response, "OAuth code exchange").await?; + let token: TokenResponse = response + .json() + .await + .context("decoding Dropbox authorization response")?; + Ok(DropboxCredentials { + app_key: self.app_key.clone(), + access_token: token.access_token, + refresh_token: token.refresh_token, + access_token_expires_at: token + .expires_in + .map(|seconds| unix_now().saturating_add(seconds)), + }) + } +} + +#[derive(Clone)] +pub struct DropboxClient { + http: Client, + credentials: Arc>, + api_base: Arc, + content_base: Arc, + token_url: Arc, +} + +impl DropboxClient { + #[must_use] + pub fn new(credentials: DropboxCredentials) -> Self { + Self::with_endpoints(credentials, API_BASE, CONTENT_BASE, TOKEN_URL) + } + + /// Endpoint injection is useful for deterministic integration tests and for + /// Dropbox-compatible enterprise gateways. + #[must_use] + pub fn with_endpoints(credentials: DropboxCredentials, api_base: &str, content_base: &str, token_url: &str) -> Self { + Self { + http: http_client(), + credentials: Arc::new(Mutex::new(credentials)), + api_base: api_base.trim_end_matches('/').into(), + content_base: content_base.trim_end_matches('/').into(), + token_url: token_url.into(), + } + } + + pub async fn credentials(&self) -> DropboxCredentials { + self.credentials.lock().await.clone() + } + + async fn bearer_token(&self) -> Result { + let mut credentials = self.credentials.lock().await; + ensure!(!credentials.access_token.is_empty(), "Dropbox is not connected"); + let now = unix_now(); + let should_refresh = credentials + .access_token_expires_at + .is_some_and(|expires| expires <= now.saturating_add(60)); + if !should_refresh { + return Ok(credentials.access_token.clone()); + } + let refresh_token = credentials + .refresh_token + .as_deref() + .context("Dropbox access expired and no refresh token is available")?; + ensure!(!credentials.app_key.is_empty(), "Dropbox app key is missing"); + let response = self + .http + .post(self.token_url.as_ref()) + .form(&[ + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("client_id", credentials.app_key.as_str()), + ]) + .send() + .await + .context("refreshing Dropbox access token")?; + let response = checked(response, "refreshing Dropbox access token").await?; + let token: TokenResponse = response + .json() + .await + .context("decoding Dropbox token response")?; + credentials.access_token = token.access_token.clone(); + credentials.access_token_expires_at = token.expires_in.map(|seconds| now.saturating_add(seconds)); + drop(credentials); + Ok(token.access_token) + } + + async fn rpc(&self, route: &str, body: Value) -> Result { + let token = self.bearer_token().await?; + let response = self + .http + .post(format!("{}{route}", self.api_base)) + .bearer_auth(token) + .json(&body) + .send() + .await + .with_context(|| format!("calling Dropbox {route}"))?; + checked(response, route).await + } + + async fn upload(&self, path: &str, bytes: Vec, mode: Value) -> Result { + let token = self + .bearer_token() + .await + .map_err(DropboxWriteError::Other)?; + let argument = json!({ + "path": path, + "mode": mode, + "autorename": false, + "mute": true, + "strict_conflict": true + }); + let response = self + .http + .post(format!("{}/files/upload", self.content_base)) + .bearer_auth(token) + .header(API_ARG, argument.to_string()) + .header(header::CONTENT_TYPE, "application/octet-stream") + .body(bytes) + .send() + .await + .map_err(|error| DropboxWriteError::Other(error.into()))?; + if response.status() == StatusCode::CONFLICT { + return Err(DropboxWriteError::Conflict { current_revision: None }); + } + let response = checked(response, "files/upload") + .await + .map_err(DropboxWriteError::Other)?; + response + .json() + .await + .context("decoding Dropbox upload metadata") + .map_err(DropboxWriteError::Other) + } + + pub async fn download(&self, path: &str) -> Result { + let token = self.bearer_token().await?; + let response = self + .http + .post(format!("{}/files/download", self.content_base)) + .bearer_auth(token) + .header(API_ARG, json!({ "path": path }).to_string()) + .send() + .await + .context("downloading Dropbox file")?; + let response = checked(response, "files/download").await?; + let metadata = response + .headers() + .get(API_RESULT) + .context("Dropbox download omitted file metadata")? + .to_str() + .context("Dropbox download metadata is not UTF-8")?; + let metadata: DropboxFileMetadata = serde_json::from_str(metadata).context("decoding Dropbox download metadata")?; + ensure!( + response.content_length().unwrap_or(0) <= MAX_DOWNLOAD_LEN, + "Dropbox download exceeds the {MAX_DOWNLOAD_LEN}-byte limit" + ); + let mut response = response; + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .context("reading Dropbox download body")? + { + ensure!( + (bytes.len() as u64).saturating_add(chunk.len() as u64) <= MAX_DOWNLOAD_LEN, + "Dropbox download exceeds the {MAX_DOWNLOAD_LEN}-byte limit" + ); + bytes.extend_from_slice(&chunk); + } + Ok(DropboxDownloadedFile { metadata, bytes }) + } + + async fn ensure_folder(&self, path: &str) -> Result<()> { + let response = self + .rpc("/files/create_folder_v2", json!({ "path": path, "autorename": false })) + .await; + match response { + Ok(_) => Ok(()), + Err(error) if dropbox_conflict(&error) => Ok(()), + Err(error) => Err(error), + } + } + + async fn ensure_folder_tree(&self, path: &str) -> Result<()> { + let mut current = String::new(); + for component in path.split('/').filter(|component| !component.is_empty()) { + current.push('/'); + current.push_str(component); + self.ensure_folder(¤t).await?; + } + Ok(()) + } + + async fn list_folder(&self, path: &str) -> Result> { + let mut page: ListFolderResponse = self + .rpc( + "/files/list_folder", + json!({ "path": path, "recursive": false, "include_deleted": false }), + ) + .await? + .json() + .await + .context("decoding Dropbox folder listing")?; + let mut entries = page.entries; + while page.has_more { + page = self + .rpc("/files/list_folder/continue", json!({ "cursor": page.cursor })) + .await? + .json() + .await + .context("decoding continued Dropbox folder listing")?; + entries.append(&mut page.entries); + } + Ok(entries) + } + + async fn delete(&self, path: &str) -> Result<()> { + let response = self.rpc("/files/delete_v2", json!({ "path": path })).await; + match response { + Ok(_) => Ok(()), + Err(error) if dropbox_not_found(&error) => Ok(()), + Err(error) => Err(error), + } + } + + pub async fn metadata(&self, path: &str) -> Result { + self + .rpc("/files/get_metadata", json!({ "path": path, "include_deleted": false })) + .await? + .json() + .await + .context("decoding Dropbox file metadata") + } + + /// Upload a package only if Dropbox still has the revision the caller read. + /// `None` is a create-only write and therefore cannot overwrite an unknown + /// remote document. + pub async fn put_checkpoint( + &self, + path: &str, + package: Vec, + expected_revision: Option<&str>, + ) -> Result { + let mode = expected_revision.map_or_else(|| json!({ ".tag": "add" }), |rev| json!({ ".tag": "update", "update": rev })); + match self.upload(path, package, mode).await { + Err(DropboxWriteError::Conflict { .. }) => { + let current_revision = self.metadata(path).await.ok().map(|metadata| metadata.rev); + Err(DropboxWriteError::Conflict { current_revision }) + }, + result => result, + } + } +} + +#[derive(Clone)] +pub struct DropboxRendezvousBackend { + client: DropboxClient, + root: Arc, +} + +impl DropboxRendezvousBackend { + #[must_use] + pub fn new(client: DropboxClient, root: impl Into) -> Self { + let root = normalized_root(&root.into()); + Self { client, root: root.into() } + } + + fn document_folder(&self, fingerprint: &[u8; 32]) -> String { + format!("{}/{}/{}", self.root, "presence", hex(fingerprint)) + } + + fn advertisement_path(&self, identity: &PublicKey, device_id: u128, fingerprint: &[u8; 32]) -> String { + let identity_hash = blake3::hash(&postcard::to_stdvec(identity).expect("public key should serialize")); + format!( + "{}/{}-{device_id:032x}.ad", + self.document_folder(fingerprint), + hex(identity_hash.as_bytes()) + ) + } +} + +impl RendezvousBackend for DropboxRendezvousBackend { + fn transport(&self) -> DiscoveryTransport { + DiscoveryTransport::Dropbox + } + + fn publish(&self, advertisement: DiscoveryAdvertisement) -> Pin> + Send + '_>> { + Box::pin(async move { + let folder = self.document_folder(&advertisement.document_fingerprint); + self.client.ensure_folder_tree(&folder).await?; + let path = self.advertisement_path(&advertisement.identity, advertisement.device_id, &advertisement.document_fingerprint); + let bytes = postcard::to_stdvec(&advertisement).context("encoding Dropbox discovery advertisement")?; + self + .client + .upload(&path, bytes, json!({ ".tag": "overwrite" })) + .await + .map_err(DropboxWriteError::into_anyhow)?; + Ok(()) + }) + } + + fn scan(&self, document_fingerprint: [u8; 32]) -> Pin>> + Send + '_>> { + Box::pin(async move { + let folder = self.document_folder(&document_fingerprint); + let entries = match self.client.list_folder(&folder).await { + Ok(entries) => entries, + Err(error) if dropbox_not_found(&error) => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut advertisements = Vec::new(); + for entry in entries { + let is_sidecar = std::path::Path::new(&entry.name) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("ad")); + if entry.tag != "file" || !is_sidecar { + continue; + } + let Some(path) = entry.path_lower.as_deref() else { continue }; + match self.client.download(path).await { + Ok(file) => match postcard::from_bytes::(&file.bytes) { + Ok(advertisement) => advertisements.push(advertisement), + Err(error) => tracing::warn!(%error, %path, "ignoring malformed Dropbox discovery sidecar"), + }, + Err(error) => tracing::warn!(%error, %path, "failed to fetch Dropbox discovery sidecar"), + } + } + Ok(advertisements) + }) + } + + fn clear( + &self, + identity: PublicKey, + device_id: u128, + document_fingerprint: [u8; 32], + ) -> Pin> + Send + '_>> { + Box::pin(async move { + self + .client + .delete(&self.advertisement_path(&identity, device_id, &document_fingerprint)) + .await + }) + } +} + +#[derive(Clone, Debug, Deserialize)] +struct TokenResponse { + access_token: String, + expires_in: Option, + refresh_token: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct DropboxFileMetadata { + pub id: String, + pub name: String, + pub rev: String, + pub path_lower: Option, + pub size: Option, +} + +#[derive(Clone, Debug)] +pub struct DropboxDownloadedFile { + pub metadata: DropboxFileMetadata, + pub bytes: Vec, +} + +#[derive(Debug)] +pub enum DropboxWriteError { + Conflict { current_revision: Option }, + Other(anyhow::Error), +} + +impl DropboxWriteError { + #[must_use] + pub fn into_anyhow(self) -> anyhow::Error { + match self { + Self::Conflict { current_revision } => current_revision.map_or_else( + || anyhow!("Dropbox file changed since it was read"), + |revision| anyhow!("Dropbox file changed since it was read (current revision: {revision})"), + ), + Self::Other(error) => error, + } + } +} + +#[derive(Debug, Deserialize)] +struct ListFolderResponse { + entries: Vec, + cursor: String, + has_more: bool, +} + +#[derive(Debug, Deserialize)] +struct DropboxEntry { + #[serde(rename = ".tag")] + tag: String, + name: String, + path_lower: Option, +} + +async fn checked(response: Response, operation: &str) -> Result { + if response.status().is_success() { + return Ok(response); + } + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + bail!("Dropbox {operation} failed ({status}): {body}") +} + +fn dropbox_conflict(error: &anyhow::Error) -> bool { + error.to_string().contains("(409 Conflict)") +} + +fn dropbox_not_found(error: &anyhow::Error) -> bool { + let message = error.to_string(); + message.contains("not_found") || message.contains("path/not_found") +} + +fn normalized_root(root: &str) -> String { + let root = root.trim().trim_matches('/'); + if root.is_empty() { + "/.flowstate".into() + } else { + format!("/{root}/.flowstate") + } +} + +fn hex(bytes: &[u8]) -> String { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut value = String::with_capacity(bytes.len() * 2); + for byte in bytes { + value.push(char::from(DIGITS[usize::from(byte >> 4)])); + value.push(char::from(DIGITS[usize::from(byte & 0x0f)])); + } + value +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_secs() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roots_and_sidecar_names_are_stable_and_dropbox_safe() { + let client = DropboxClient::new(DropboxCredentials::default()); + let backend = DropboxRendezvousBackend::new(client, "/Debates/"); + let identity = crate::identity::PortableIdentitySecret::generate().public(); + let path = backend.advertisement_path(&identity, 7, &[0xab; 32]); + assert!(path.starts_with("/Debates/.flowstate/presence/abab")); + assert!(path.ends_with("-00000000000000000000000000000007.ad")); + assert!(!path.contains(' ')); + } + + #[test] + fn empty_root_uses_app_folder() { + assert_eq!(normalized_root(""), "/.flowstate"); + assert_eq!(normalized_root("/Team"), "/Team/.flowstate"); + } + + #[test] + fn pkce_uses_s256_offline_tokens_and_validates_state() { + let flow = DropboxPkceFlow::begin("app-key", "flowstate://oauth/dropbox").unwrap(); + let query: std::collections::HashMap<_, _> = flow.authorization_url.query_pairs().collect(); + assert_eq!( + query + .get("code_challenge_method") + .map(std::borrow::Cow::as_ref), + Some("S256") + ); + assert_eq!(query.get("token_access_type").map(std::borrow::Cow::as_ref), Some("offline")); + let callback = Url::parse(&format!("flowstate://oauth/dropbox?code=abc&state={}", flow.state)).unwrap(); + assert_eq!(flow.verify_callback(&callback).unwrap(), "abc"); + assert!( + flow + .verify_callback(&Url::parse("flowstate://oauth/dropbox?code=abc&state=wrong").unwrap()) + .is_err() + ); + } +} diff --git a/crates/flowstate-collab/src/identity.rs b/crates/flowstate-collab/src/identity.rs new file mode 100644 index 00000000..8487c8df --- /dev/null +++ b/crates/flowstate-collab/src/identity.rs @@ -0,0 +1,211 @@ +//! Durable person identity and mutual trust primitives. + +use anyhow::{Result, anyhow, ensure}; +use iroh::{PublicKey, SecretKey, Signature}; +use serde::{Deserialize, Serialize}; + +const PROFILE_CONTEXT: &[u8] = b"flowstate/person-profile/v1"; +const TRUST_CONTEXT: &[u8] = b"flowstate/trust-attestation/v1"; + +/// Plaintext-at-rest signing seed. Callers decide how it is protected and +/// persisted; its `Debug` implementation never exposes key material. +#[derive(Clone)] +pub struct PortableIdentitySecret(SecretKey); + +impl std::fmt::Debug for PortableIdentitySecret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("PortableIdentitySecret([REDACTED])") + } +} + +impl PortableIdentitySecret { + #[must_use] + pub fn generate() -> Self { + Self(SecretKey::generate()) + } + + pub fn from_hex(text: &str) -> Result { + ensure!( + text.len() == 64 && text.is_ascii(), + "identity signing seed must contain 64 hexadecimal characters" + ); + let mut bytes = [0u8; 32]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&text[index * 2..index * 2 + 2], 16).map_err(|_| anyhow!("identity signing seed is not hexadecimal"))?; + } + Ok(Self(SecretKey::from_bytes(&bytes))) + } + + #[must_use] + pub fn to_hex(&self) -> String { + use std::fmt::Write as _; + self + .0 + .to_bytes() + .iter() + .fold(String::with_capacity(64), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) + } + + #[must_use] + pub fn public(&self) -> PublicKey { + self.0.public() + } + + #[must_use] + pub(crate) fn sign(&self, payload: &[u8]) -> Signature { + self.0.sign(payload) + } + + #[must_use] + pub fn sign_profile(&self, sequence: u64, mut display_name: String, color_rgb: u32, avatar_digest: Option<[u8; 32]>) -> SignedProfile { + // `SignedProfile::verify` rejects names over 64 bytes; clamp here so a + // locally-entered long name can never mint a profile every peer discards. + if display_name.len() > 64 { + let mut cut = 64; + while !display_name.is_char_boundary(cut) { + cut -= 1; + } + display_name.truncate(cut); + } + let identity = self.public(); + let payload = profile_payload(&identity, sequence, &display_name, color_rgb, avatar_digest.as_ref()); + SignedProfile { + identity, + sequence, + display_name, + color_rgb: color_rgb & 0x00ff_ffff, + avatar_digest, + signature: self.0.sign(&payload), + } + } + + #[must_use] + pub fn attest_trust(&self, subject: PublicKey, issued_at_unix: u64) -> TrustAttestation { + let signer = self.public(); + let signature = self + .0 + .sign(&trust_payload(&signer, &subject, issued_at_unix)); + TrustAttestation { + signer, + subject, + issued_at_unix, + signature, + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct SignedProfile { + pub identity: PublicKey, + pub sequence: u64, + pub display_name: String, + pub color_rgb: u32, + pub avatar_digest: Option<[u8; 32]>, + pub signature: Signature, +} + +impl SignedProfile { + #[must_use] + pub fn verify(&self) -> bool { + self.color_rgb <= 0x00ff_ffff + && self.display_name.len() <= 64 + && self + .identity + .verify( + &profile_payload( + &self.identity, + self.sequence, + &self.display_name, + self.color_rgb, + self.avatar_digest.as_ref(), + ), + &self.signature, + ) + .is_ok() + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct TrustAttestation { + pub signer: PublicKey, + pub subject: PublicKey, + pub issued_at_unix: u64, + pub signature: Signature, +} + +impl TrustAttestation { + #[must_use] + pub fn verify(&self) -> bool { + self + .signer + .verify(&trust_payload(&self.signer, &self.subject, self.issued_at_unix), &self.signature) + .is_ok() + } +} + +/// Short code people compare out-of-band before exchanging mutual standing +/// access attestations. Sorting makes both screens show the same code. +#[must_use] +pub fn safety_code(left: &PublicKey, right: &PublicKey) -> String { + let (first, second) = if left.as_bytes() <= right.as_bytes() { + (left, right) + } else { + (right, left) + }; + let mut input = Vec::with_capacity(64); + input.extend_from_slice(first.as_bytes()); + input.extend_from_slice(second.as_bytes()); + let digest = blake3::hash(&input); + let number = u64::from_le_bytes( + digest.as_bytes()[..8] + .try_into() + .expect("eight digest bytes"), + ) % 1_000_000_000_000; + format!("{:04} {:04} {:04}", number / 100_000_000, (number / 10_000) % 10_000, number % 10_000) +} + +fn profile_payload(identity: &PublicKey, sequence: u64, display_name: &str, color_rgb: u32, avatar_digest: Option<&[u8; 32]>) -> Vec { + postcard::to_stdvec(&(PROFILE_CONTEXT, identity, sequence, display_name, color_rgb & 0x00ff_ffff, avatar_digest)) + .expect("portable profile signing payload should serialize") +} + +fn trust_payload(signer: &PublicKey, subject: &PublicKey, issued_at_unix: u64) -> Vec { + postcard::to_stdvec(&(TRUST_CONTEXT, signer, subject, issued_at_unix)).expect("trust signing payload should serialize") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn profile_updates_and_attestations_are_signed() { + let alex = PortableIdentitySecret::generate(); + let blair = PortableIdentitySecret::generate(); + let profile = alex.sign_profile(7, "Alex".into(), 0x123456, None); + assert!(profile.verify()); + let attestation = alex.attest_trust(blair.public(), 42); + assert!(attestation.verify()); + assert_eq!(safety_code(&alex.public(), &blair.public()), safety_code(&blair.public(), &alex.public())); + } + + #[test] + fn signing_seed_round_trips_without_debug_disclosure() { + let secret = PortableIdentitySecret::generate(); + let restored = PortableIdentitySecret::from_hex(&secret.to_hex()).unwrap(); + assert_eq!(restored.public(), secret.public()); + assert!(!format!("{secret:?}").contains(&secret.to_hex())); + // 64 BYTES of non-ASCII must error, not panic on a char boundary. + assert!(PortableIdentitySecret::from_hex(&"é".repeat(32)).is_err()); + } + + #[test] + fn oversized_display_names_are_clamped_to_verifiable_profiles() { + let secret = PortableIdentitySecret::generate(); + let profile = secret.sign_profile(1, "é".repeat(64), 0x123456, None); + assert!(profile.display_name.len() <= 64); + assert!(profile.verify()); + } +} diff --git a/crates/flowstate-collab/src/ids.rs b/crates/flowstate-collab/src/ids.rs new file mode 100644 index 00000000..72ed12aa --- /dev/null +++ b/crates/flowstate-collab/src/ids.rs @@ -0,0 +1,124 @@ +use std::{fmt, str::FromStr}; + +use rand::{TryRngCore as _, rngs::OsRng}; +use serde::{Deserialize, Serialize}; + +pub type PeerId = iroh::EndpointId; + +pub const SESSION_ID_LEN: usize = 32; + +pub const PALETTE: [u32; 8] = [0x3b82f6, 0xef4444, 0x22c55e, 0xf59e0b, 0x8b5cf6, 0x06b6d4, 0xec4899, 0x84cc16]; + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub struct SessionId([u8; SESSION_ID_LEN]); + +impl SessionId { + #[must_use] + pub fn new() -> Self { + let mut bytes = [0; SESSION_ID_LEN]; + OsRng + .try_fill_bytes(&mut bytes) + .expect("OS randomness must be available to create collaboration session IDs"); + Self(bytes) + } + + #[must_use] + pub const fn from_bytes(bytes: [u8; SESSION_ID_LEN]) -> Self { + Self(bytes) + } + + #[must_use] + pub const fn as_bytes(&self) -> &[u8; SESSION_ID_LEN] { + &self.0 + } + + #[must_use] + pub fn color_index_for_peer(peer: &PeerId) -> usize { + Self::color_index_for_peer_bytes(peer.as_bytes()) + } + + #[must_use] + pub fn color_index_for_peer_bytes(bytes: &[u8]) -> usize { + let mut hash = 0usize; + for byte in bytes.iter().take(8) { + hash = hash.wrapping_mul(257).wrapping_add(usize::from(*byte)); + } + hash % PALETTE.len() + } +} + +impl Default for SessionId { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Display for SessionId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // §perf: manual nibble→hex lookup avoids invoking the fmt machinery per byte. + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut buf = String::with_capacity(self.0.len() * 2); + for &byte in &self.0 { + buf.push(HEX[(byte >> 4) as usize] as char); + buf.push(HEX[(byte & 0x0f) as usize] as char); + } + f.write_str(&buf) + } +} + +impl FromStr for SessionId { + type Err = SessionIdParseError; + + fn from_str(value: &str) -> Result { + if value.len() != SESSION_ID_LEN * 2 { + return Err(SessionIdParseError); + } + + let mut bytes = [0; SESSION_ID_LEN]; + for (ix, chunk) in value.as_bytes().chunks_exact(2).enumerate() { + let text = std::str::from_utf8(chunk).map_err(|_| SessionIdParseError)?; + bytes[ix] = u8::from_str_radix(text, 16).map_err(|_| SessionIdParseError)?; + } + Ok(Self(bytes)) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SessionIdParseError; + +impl fmt::Display for SessionIdParseError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("expected a 64-character hex session id") + } +} + +#[cfg(test)] +mod tests { + use super::SessionId; + + #[test] + fn parses_uppercase_session_id_hex() { + let parsed = "0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210" + .parse::() + .expect("uppercase hex should parse"); + assert_eq!(parsed.to_string(), "0123456789abcdeffedcba98765432100123456789abcdeffedcba9876543210"); + } +} + +impl std::error::Error for SessionIdParseError {} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] +pub struct BlobId(pub u128); + +impl BlobId { + #[must_use] + pub fn new() -> Self { + Self(uuid::Uuid::new_v4().as_u128()) + } +} + +impl Default for BlobId { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/flowstate-collab/src/lib.rs b/crates/flowstate-collab/src/lib.rs new file mode 100644 index 00000000..fefaca9b --- /dev/null +++ b/crates/flowstate-collab/src/lib.rs @@ -0,0 +1,26 @@ +//! Peer-to-peer collaboration support for Flowstate rich-text documents. +//! +//! This crate contains the GPUI-free collaboration core: the Loro-native CRDT +//! runtime, transport protocol types, presence, and networking state. +//! Application/UI integration lives in `crates/flowstate/src/collab`. + +pub mod admission; +pub mod bluetooth; +pub mod crdt_runtime; +pub mod discovery; +pub mod doc_io; +pub mod dropbox; +pub mod identity; +pub mod ids; +pub mod local_write; +pub mod net; +pub mod presence; +pub mod proto_direct; +pub mod proto_gossip; +pub mod self_check; +pub mod ticket; + +pub use admission::SessionAdmission; +pub use ids::{BlobId, SessionId}; +pub use proto_direct::DIRECT_ALPN; +pub use proto_gossip::PROTOCOL_VERSION; diff --git a/crates/flowstate-collab/src/local_write/commit.rs b/crates/flowstate-collab/src/local_write/commit.rs new file mode 100644 index 00000000..863ec457 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/commit.rs @@ -0,0 +1,1350 @@ +//! Intent execution: resolve → mutate → commit → patch (spec §4, I-10). +//! +//! Runs with the write gate already held by the caller ([`super::handle`]). +//! Execution phases: +//! +//! 1. **Resolve** every anchor/identity against live Loro state. Any failure +//! rejects the intent before the doc is touched (I-15). +//! 2. **Mutate** the doc through the surviving Loro-authority helpers. A +//! mid-apply error triggers the I-10 compensation path (`revert_to`), never +//! a partial commit escaping. +//! 3. **Commit** exactly once, origin `local`, commit message = intent class +//! (field forensics, spec §16). +//! 4. **Synthesize** exact projection patches from the resolved intent +//! (O(edit)); fall back to a full rebuild only for op classes that +//! legitimately require it — loudly, counted. +//! +//! One intent = one Loro commit = one undo-group member. + +use anyhow::{Context as _, Result}; +use flowstate_document::{ + BlockId, DocumentProjection, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, + OBJECT_REPLACEMENT, ParagraphId, ProjectionPatchBatch, loro_schema::body_text, +}; +use gpui_flowtext::{DocumentOffset, SelectionAffinity, VisualGravity}; +use loro::LoroDoc; +use uuid::Uuid; + +use super::intents::{ + CursorEndpoint, FragmentBlock, IntentCounters, LocalCommit, LocalIntent, LocalWriteOutcome, ProjectionReplace, SelectionSnapshot, TableIntent, + WriteRejected, +}; +use super::patch_synthesis::{PatchPlan, synthesize_patches}; +use super::resolve::{ResolvedTextPosition, resolve_text_anchor, resolve_text_range}; +use crate::crdt_runtime::{ + CrdtRuntime, cursor_for_boundary, delete_projection_object_block, delete_projection_paragraph_metadata, insert_projection_object_block, + join_projection_paragraphs, mark_run_styles, move_projection_object_block, paragraph_boundary_loro_unicode_index, paragraph_style_value, + prune_orphaned_body_object_blocks, repair_paragraph_metadata_after_stable_split, replace_projection_equation_source_range, + replace_projection_image_alt_text, replace_projection_image_caption, replace_projection_object_block, sentinel_protected_delete_range, + set_projection_image_layout, table_ops, +}; + +/// The fully-resolved execution plan for one intent, produced before any +/// mutation. Everything positional in here is live-Loro-space, resolved inside +/// the gate. +#[derive(Debug)] +pub(crate) enum ResolvedPlan { + InsertText { + at: ResolvedTextPosition, + text: String, + style_override: Option, + }, + DeleteRange { + start: ResolvedTextPosition, + end: ResolvedTextPosition, + }, + SplitParagraph { + at: ResolvedTextPosition, + inherited_style: flowstate_document::ParagraphStyle, + new_paragraph: ParagraphId, + new_block: BlockId, + }, + JoinParagraphs { + first: ParagraphId, + second: ParagraphId, + first_ix: usize, + }, + SetMarks { + start: ResolvedTextPosition, + end: ResolvedTextPosition, + styles: flowstate_document::RunStyles, + }, + SetParagraphStyle { + paragraph: ParagraphId, + paragraph_ix: usize, + style: flowstate_document::ParagraphStyle, + }, + SetParagraphStyles { + /// `(paragraph id, projection index, boundary loro-unicode position)`, + /// input order; stale and non-representable targets already skipped. + /// Marks never shift text, so pre-resolved boundaries stay valid across + /// the whole batch. + targets: Vec<(ParagraphId, usize, usize)>, + style: flowstate_document::ParagraphStyle, + }, + InsertObject { + at: ResolvedTextPosition, + block_ix: usize, + new_block: BlockId, + block: flowstate_document::InputBlock, + }, + ReplaceObject { + block: BlockId, + block_ix: usize, + after: flowstate_document::InputBlock, + }, + DeleteBlocks { + blocks: Vec<(BlockId, usize)>, + }, + MoveBlock { + block: BlockId, + to_ix: usize, + }, + InsertRichFragment { + at: ResolvedTextPosition, + blocks: Vec, + }, + ReplaceMatches { + /// Resolved (start, end, replacement styles) triples, sorted DESCENDING + /// by start and pruned of collapsed/cross-paragraph/overlapping ranges, + /// so back-to-front application never shifts a later range. + matches: Vec<(ResolvedTextPosition, ResolvedTextPosition, Option)>, + replacement: String, + }, + ReplaceEquationSourceRange { + equation: BlockId, + range: std::ops::Range, + text: String, + }, + ReplaceImageAltText { + image: BlockId, + text: String, + }, + ReplaceImageCaption { + image: BlockId, + caption: Option, + }, + SetImageLayout { + image: BlockId, + sizing: flowstate_document::InputImageSizing, + alignment: flowstate_document::InputBlockAlignment, + }, + Table { + table: BlockId, + table_ix: usize, + op: ResolvedTableOp, + }, +} + +#[derive(Debug)] +pub(crate) enum ResolvedTableOp { + InsertRow { + new_row: flowstate_document::InputTableRow, + after_row: Option, + }, + DeleteRow { + row: gpui_flowtext::RowId, + }, + MoveRow { + row: gpui_flowtext::RowId, + after_row: Option, + }, + InsertColumn { + new_column: gpui_flowtext::ColumnId, + after_column: Option, + width: flowstate_document::InputTableColumnWidth, + cells: Vec, + }, + DeleteColumn { + column: gpui_flowtext::ColumnId, + }, + MoveColumn { + column: gpui_flowtext::ColumnId, + after_column: Option, + }, + ReplaceCell { + row: gpui_flowtext::RowId, + column: gpui_flowtext::ColumnId, + cell: flowstate_document::InputTableCell, + }, + SetCellSpan { + row: gpui_flowtext::RowId, + column: gpui_flowtext::ColumnId, + row_span: u16, + column_span: u16, + }, + SetColumnWidth { + column_ix: usize, + width: flowstate_document::InputTableColumnWidth, + }, +} + +/// Mutation-phase output: what the plan did, for patch synthesis and caret +/// placement. +#[derive(Debug, Default)] +pub(crate) struct MutationSummary { + pub containers_touched: u32, + pub marks_emitted: u32, + /// Live body-unicode caret position after the mutation, when the intent + /// defines one. + pub caret_body_unicode: Option, +} + +/// Execute one local intent against the gate-held core. +#[hotpath::measure] +pub(crate) fn apply_local_intent(core: &mut CrdtRuntime, intent: &LocalIntent) -> Result { + let mut counters = IntentCounters::default(); + + // ---- Phase 1: resolve (no mutation on any failure path) ----------------- + let plan = resolve_intent(core, intent)?; + + // Deferred identity/replica registration (spec §16, §act-twelve A12.1.1): + // AFTER resolution (a rejected intent must leave the doc untouched — the + // I-15 zero-mutation law) but BEFORE the edit's frontier/counter captures. + // The meta commit persists its own update segment, so it must wholly + // precede the edit's span: running it mid-commit appended segments out of + // causal order (the edit's segment was dropped), and running it between + // grouped edits split their counter contiguity (group undo stopped + // merging). + core.register_pending_author_identity(); + + let frontier_before = core.doc().state_frontiers(); + let vv_before = core.doc().state_vv(); + let base_frontier = frontier_before.encode(); + + // §act-four M1: a qualifying whole-doc op (mass DeleteRange, select-all + // restyle, …) records its inverse BEFORE the mutation changes the content + // (doc + projection are still pre-op here). `None` for anything that doesn't + // qualify — the op proceeds identically either way. + let pending_inverse = super::recorded_inverse::capture_before_execute(core, &plan); + let peer_counter_before = local_peer_counter_end(core); + + // ---- Phase 2: mutate (Err → I-10 compensation) --------------------------- + let summary = match execute_plan(core, &plan) { + Ok(summary) => summary, + Err(error) => return Err(compensate_failed_intent(core, &frontier_before, &vv_before, intent.class(), &error)), + }; + counters.containers_touched = summary.containers_touched; + counters.marks_emitted = summary.marks_emitted; + + // ---- Phase 3: commit ------------------------------------------------------ + // Single-threaded inside the gate, so the doc-global next-commit options are + // race-free here (semantics-audit C4). Origin `local` keeps the commit inside + // the undo stack; the class message is §16 field-forensics attribution. + counters.loro_ops = pending_op_count(core.doc()); + hotpath::measure_block!("loro_commit", { + core.doc().set_next_commit_origin("local"); + core.doc().set_next_commit_message(intent.class()); + core.doc().commit(); + }); + if let Err(error) = core.record_undo_checkpoint() { + // The commit itself succeeded; a checkpoint failure only degrades undo + // granularity. Loud, not fatal. + tracing::error!(%error, class = intent.class(), "recording undo checkpoint after local intent failed"); + } + + let new_frontier = core.doc().state_frontiers().encode(); + + // ---- Phase 4: patches + publish ------------------------------------------- + let invalidation_patches = synthesize_patches(core, intent, &plan); + let transaction_id = Uuid::new_v4().as_u128(); + + let outcome = match invalidation_patches { + PatchPlan::Patches { patches, invalidation } => { + counters.patch_count = u32::try_from(patches.len()).unwrap_or(u32::MAX); + // §act-three B.1: arm (or clear) the recorded-inverse slot. The delete's + // own synthesized patches are the verbatim redo replay material; the + // committed counter span is the undo item this record must match. + match pending_inverse { + Some(pending) => { + let delete_span = loro::CounterSpan::new(peer_counter_before, local_peer_counter_end(core)); + super::recorded_inverse::finalize_capture(core, pending, delete_span, &patches); + }, + None => core.clear_recorded_inverse(), + } + let mut invalidation = invalidation; + core.merge_subscription_invalidation(&mut invalidation); + core.apply_projection_patch_set(&patches); + core.set_projection_frontier(new_frontier.clone()); + queue_publish_events(core, frontier_before, vv_before, invalidation); + audit_patched_projection(core, intent.class()); + let batch = ProjectionPatchBatch { + transaction_id, + base_frontier, + new_frontier: new_frontier.clone(), + patches: patches.into(), + }; + // Ordered stream (field fix): the editor drains this batch in commit + // order together with any remote batches that preceded it. + core.push_editor_stream(gpui_flowtext::ProjectionStreamItem::Patches(batch.clone())); + LocalWriteOutcome::Committed(LocalCommit { + patches: batch, + frontier: new_frontier, + version_vector: core.doc().state_vv().encode(), + selection_after: selection_after(core, &summary), + counters, + }) + }, + PatchPlan::FullRebuild { invalidation, reason } => { + counters.full_rebuild = true; + // §act-three B.1: no patch material to replay on redo — drop any capture. + core.clear_recorded_inverse(); + tracing::warn!(class = intent.class(), reason, "full-rebuild-after-local-write"); + let mut invalidation = invalidation; + core.merge_subscription_invalidation(&mut invalidation); + if let Err(error) = core.refresh_projection() { + return Err(compensate_failed_intent(core, &frontier_before, &vv_before, intent.class(), &error)); + } + queue_publish_events(core, frontier_before, vv_before, invalidation); + let replace = ProjectionReplace { + document: core.projection_ref().clone(), + frontier: new_frontier.clone(), + version_vector: core.doc().state_vv().encode(), + }; + core.push_editor_stream(gpui_flowtext::ProjectionStreamItem::Replace(Box::new(replace.document.clone()))); + LocalWriteOutcome::CommittedWithRebuild { + commit: LocalCommit { + patches: ProjectionPatchBatch { + transaction_id, + base_frontier, + new_frontier: new_frontier.clone(), + patches: std::sync::Arc::from(Vec::new()), + }, + frontier: new_frontier, + version_vector: core.doc().state_vv().encode(), + selection_after: selection_after(core, &summary), + counters, + }, + replace: Box::new(replace), + } + }, + }; + Ok(outcome) +} + +/// Ops recorded in the pending (uncommitted) transaction — the source-of-truth +/// per-intent op count for the §11 complexity contract. +fn pending_op_count(doc: &LoroDoc) -> u64 { + u64::try_from(doc.get_pending_txn_len()).unwrap_or(u64::MAX) +} + +/// §act-three B.1: local-peer oplog counter end — the coordinate the +/// `UndoManager` records item spans in. +fn local_peer_counter_end(core: &CrdtRuntime) -> loro::Counter { + let peer = core.doc().peer_id(); + core.doc().oplog_vv().get(&peer).copied().unwrap_or(0) +} + +/// I-10 recovery. Order is load-bearing (semantics-audit F4): +/// 1. repair origin BEFORE `revert_to` — its internal `diff()` implicitly +/// commits the pending partial mutation, and that commit must be +/// undo-excluded; +/// 2. `revert_to` leaves the inverse ops uncommitted — finish with an explicit +/// repair-origin commit; +/// 3. both commits leave through ONE publish payload (`local_update_bytes` +/// exports the covering range in one blob), so no peer can import the +/// partial without its inverse. +fn compensate_failed_intent( + core: &mut CrdtRuntime, + frontier_before: &loro::Frontiers, + vv_before: &loro::VersionVector, + class: &'static str, + error: &anyhow::Error, +) -> WriteRejected { + tracing::error!(class, %error, "local intent failed mid-apply; compensating via revert_to (I-10)"); + // §act-three B.1: the compensation commits change the frontier anyway, but + // drop the (possibly large) capture eagerly. + core.clear_recorded_inverse(); + core.doc().set_next_commit_origin("repair"); + core.doc().set_next_commit_message("intent-compensation"); + if let Err(revert_error) = core.doc().revert_to(frontier_before) { + tracing::error!(class, %revert_error, "revert_to during intent compensation FAILED; core must be reloaded"); + return WriteRejected::CompensationFailed { + class, + diagnostic: format!("mid-apply error: {error:#}; revert_to error: {revert_error}"), + }; + } + // The inverse ops sit in the renewed pending txn — commit them under the + // repair origin. + core.doc().set_next_commit_origin("repair"); + core + .doc() + .set_next_commit_message("intent-compensation-inverse"); + core.doc().commit(); + + // One atomic publish payload covering partial + inverse: the export starts + // from the PRE-mutation version vector, so the pair travels together and no + // peer can import one half (spec I-10c). + queue_publish_events( + core, + frontier_before.clone(), + vv_before.clone(), + flowstate_document_invalidation_full(core, "intent-compensation"), + ); + + // The projection never saw the partial mutation (no patches escaped), and the + // doc is back at pre-intent state — but refresh defensively so any drifted + // bookkeeping self-heals, and so the failure is observable in metrics. + if let Err(refresh_error) = core.refresh_projection() { + tracing::error!(class, %refresh_error, "projection refresh after compensation failed; core must be reloaded"); + return WriteRejected::CompensationFailed { + class, + diagnostic: format!("mid-apply error: {error:#}; post-compensation refresh error: {refresh_error:#}"), + }; + } + WriteRejected::CompensatedFailure { + class, + diagnostic: format!("{error:#}"), + } +} + +fn flowstate_document_invalidation_full(core: &CrdtRuntime, reason: &'static str) -> crate::crdt_runtime::ProjectionInvalidation { + crate::crdt_runtime::ProjectionInvalidation::full_rebuild( + core.projection_ref().frontier.clone(), + core.doc().state_frontiers().encode(), + reason, + ) +} + +/// Produce the publish-side events (`LocalUpdate` bytes + persistence) for the +/// committed change and queue them for the I/O service to drain (spec §6). +#[hotpath::measure] +fn queue_publish_events( + core: &mut CrdtRuntime, + frontier_before: loro::Frontiers, + vv_before: loro::VersionVector, + invalidation: crate::crdt_runtime::ProjectionInvalidation, +) { + match core.events_after_local_change(frontier_before, vv_before, invalidation, false) { + Ok(events) => core.queue_publish(events), + Err(error) => { + // Export/persist failures must not fail the already-committed intent; + // anti-entropy recovers the update. Loud. + tracing::error!(%error, "exporting/persisting committed local intent update failed; anti-entropy must recover it"); + }, + } +} + +/// Debug/CI audit (spec §7): the patch-applied projection must equal an +/// independent full materialization. Release sampling is wired by the handle's +/// config; this function is the debug-build always-on arm. +#[hotpath::measure] +pub(crate) fn audit_patched_projection(core: &mut CrdtRuntime, class: &'static str) { + #[cfg(debug_assertions)] + { + if let Err(error) = core.audit_projection_against_full_rebuild(class) { + // The audit itself repairs via snapshot fallback; surfacing here is for + // test visibility. + tracing::error!(class, %error, "audit-mismatch: patch-applied projection diverged from full rebuild"); + } + } + #[cfg(not(debug_assertions))] + { + let _ = (core, class); + } +} + +// --------------------------------------------------------------------------- +// Resolution +// --------------------------------------------------------------------------- + +#[hotpath::measure] +fn resolve_intent(core: &CrdtRuntime, intent: &LocalIntent) -> Result { + let doc = core.doc(); + let projection = core.projection_ref(); + let index = core.projection_index_ref(); + match intent { + LocalIntent::InsertText(insert) => { + if insert.text.is_empty() { + return Err(WriteRejected::EmptyIntent); + } + if insert.text.contains('\n') || insert.text.contains(OBJECT_REPLACEMENT) { + // Structure enters through split/object/fragment intents only; a text + // insert is always intra-paragraph (keeps patch synthesis exact and + // repair-free — spec I-8). + return Err(WriteRejected::StructureViolation( + "InsertText must not contain paragraph breaks or object placeholders; use SplitParagraph / InsertObject / InsertRichFragment", + )); + } + Ok(ResolvedPlan::InsertText { + at: resolve_text_anchor(doc, projection, index, &insert.at)?, + text: insert.text.clone(), + style_override: insert.style_override, + }) + }, + LocalIntent::DeleteRange(delete) => { + let (start, end) = resolve_text_range(doc, projection, index, &delete.start, &delete.end)?; + if start.body_unicode == end.body_unicode { + return Err(WriteRejected::EmptyIntent); + } + Ok(ResolvedPlan::DeleteRange { start, end }) + }, + LocalIntent::SplitParagraph(split) => Ok(ResolvedPlan::SplitParagraph { + at: resolve_text_anchor(doc, projection, index, &split.at)?, + inherited_style: split.inherited_style, + new_paragraph: ParagraphId(Uuid::new_v4().as_u128()), + new_block: BlockId(Uuid::new_v4().as_u128()), + }), + LocalIntent::JoinParagraphs(join) => { + let first_ix = index + .paragraph_index(join.first) + .ok_or(WriteRejected::UnresolvedParagraph(join.first))?; + let second_ix = index + .paragraph_index(join.second) + .ok_or(WriteRejected::UnresolvedParagraph(join.second))?; + if second_ix != first_ix + 1 { + return Err(WriteRejected::StructureViolation("JoinParagraphs requires adjacent paragraphs")); + } + Ok(ResolvedPlan::JoinParagraphs { + first: join.first, + second: join.second, + first_ix, + }) + }, + LocalIntent::SetMarks(marks) => { + let (start, end) = resolve_text_range(doc, projection, index, &marks.start, &marks.end)?; + if start.body_unicode == end.body_unicode { + return Err(WriteRejected::EmptyIntent); + } + Ok(ResolvedPlan::SetMarks { + start, + end, + styles: marks.styles, + }) + }, + LocalIntent::SetParagraphStyle(style) => { + let paragraph_ix = index + .paragraph_index(style.paragraph) + .ok_or(WriteRejected::UnresolvedParagraph(style.paragraph))?; + // An interstitial paragraph (one that directly follows an object row) + // has no boundary `\n` of its own — paragraph style is CARRIED by the + // boundary mark, so its style is canonically inherited and cannot be + // set: marking the nearest newline would restyle a DIFFERENT paragraph + // (the object-fuzz caught exactly that as a maintained-vs-canonical + // style divergence). Reject before mutation (I-15) until the schema + // grows a per-record style field for boundary-less rows. + let interstitial = flowstate_document::block_ix_for_paragraph(projection, paragraph_ix) + .is_some_and(|row| row > 0 && !matches!(projection.blocks.get(row - 1), Some(flowstate_document::Block::Paragraph(_)))); + if interstitial { + return Err(WriteRejected::StructureViolation( + "paragraph style on an object-following paragraph is not representable (boundary-less row)", + )); + } + Ok(ResolvedPlan::SetParagraphStyle { + paragraph: style.paragraph, + paragraph_ix, + style: style.style, + }) + }, + LocalIntent::SetParagraphStyles(styles) => { + if styles.paragraphs.is_empty() { + return Err(WriteRejected::EmptyIntent); + } + // Batched resolution (§11): stale identities and non-representable + // (object-following, boundary-less) rows are SKIPPED, not rejected — + // the surviving targets carry the user's intent, mirroring + // ReplaceMatches. Rows resolve in one pass; boundaries resolve through + // ONE batched cursor query (per-target `get_cursor_pos` is a linear + // chunk scan — quadratic over a select-all restyle without this). + let rows = flowstate_document::paragraph_block_rows(projection); + let mut targets: Vec<(ParagraphId, usize)> = Vec::with_capacity(styles.paragraphs.len()); + for paragraph in &styles.paragraphs { + let Some(paragraph_ix) = index.paragraph_index(*paragraph) else { + continue; + }; + if projection.ids.paragraph_ids.get(paragraph_ix).copied() != Some(*paragraph) { + continue; + } + let interstitial = rows + .get(paragraph_ix) + .is_some_and(|&row| row > 0 && !matches!(projection.blocks.get(row - 1), Some(flowstate_document::Block::Paragraph(_)))); + if interstitial { + continue; + } + targets.push((*paragraph, paragraph_ix)); + } + if targets.is_empty() { + return Err(WriteRejected::EmptyIntent); + } + let indices: Vec = targets + .iter() + .map(|(_, paragraph_ix)| *paragraph_ix) + .collect(); + let boundaries = crate::crdt_runtime::paragraph_boundaries_loro_unicode_indices(doc, projection, &indices); + Ok(ResolvedPlan::SetParagraphStyles { + targets: targets + .into_iter() + .zip(boundaries) + .map(|((paragraph, paragraph_ix), boundary)| (paragraph, paragraph_ix, boundary)) + .collect(), + style: styles.style, + }) + }, + LocalIntent::InsertObject(insert) => { + let at = resolve_text_anchor(doc, projection, index, &insert.at)?; + let block_ix = flowstate_document::block_ix_for_paragraph(projection, at.paragraph_ix) + .map(|ix| if at.byte > 0 { ix + 1 } else { ix }) + .ok_or(WriteRejected::UnresolvedParagraph(insert.at.paragraph))?; + Ok(ResolvedPlan::InsertObject { + at, + block_ix, + new_block: BlockId(Uuid::new_v4().as_u128()), + block: insert.block.clone(), + }) + }, + LocalIntent::ReplaceObject(replace) => Ok(ResolvedPlan::ReplaceObject { + block: replace.block, + block_ix: index + .block_index(replace.block) + .ok_or(WriteRejected::UnresolvedBlock(replace.block))?, + after: replace.after.clone(), + }), + LocalIntent::DeleteBlocks(delete) => { + if delete.blocks.is_empty() { + return Err(WriteRejected::EmptyIntent); + } + let blocks = delete + .blocks + .iter() + .map(|block| { + index + .block_index(*block) + .map(|ix| (*block, ix)) + .ok_or(WriteRejected::UnresolvedBlock(*block)) + }) + .collect::, _>>()?; + Ok(ResolvedPlan::DeleteBlocks { blocks }) + }, + LocalIntent::MoveBlock(move_block) => { + // Resolve-before-mutate (I-15): the moved block must exist even though + // only the destination feeds the plan. + index + .block_index(move_block.block) + .ok_or(WriteRejected::UnresolvedBlock(move_block.block))?; + let to_ix = match move_block.before { + Some(before) => index + .block_index(before) + .ok_or(WriteRejected::UnresolvedBlock(before))?, + None => projection.blocks.len(), + }; + Ok(ResolvedPlan::MoveBlock { + block: move_block.block, + to_ix, + }) + }, + LocalIntent::InsertRichFragment(fragment) => { + if fragment.blocks.is_empty() { + return Err(WriteRejected::EmptyIntent); + } + Ok(ResolvedPlan::InsertRichFragment { + at: resolve_text_anchor(doc, projection, index, &fragment.at)?, + blocks: fragment.blocks.clone(), + }) + }, + LocalIntent::ReplaceMatches(replace) => { + if replace.replacement.contains('\n') || replace.replacement.contains(OBJECT_REPLACEMENT) { + return Err(WriteRejected::StructureViolation( + "ReplaceMatches replacement must not contain paragraph breaks or object placeholders", + )); + } + // Batched resolution (§11): a storm carries thousands of matches over + // thousands of paragraphs, and per-anchor cursor resolution is a linear + // chunk scan per call (the batch-resolver lesson: 27k matches on the + // reference doc = 125s). Resolve each DISTINCT paragraph's body start + // once through ONE batched query and derive every match position from + // projection bytes; only anchors that carry explicit cursors (none from + // the editor's replace-all) take the per-anchor path. + let mut distinct: Vec = Vec::new(); + let mut distinct_slots: std::collections::HashMap = std::collections::HashMap::new(); + for entry in &replace.matches { + if entry.start.cursor.is_none() && entry.end.cursor.is_none() && entry.start.paragraph == entry.end.paragraph { + distinct_slots + .entry(entry.start.paragraph) + .or_insert_with(|| { + distinct.push(entry.start.paragraph); + distinct.len() - 1 + }); + } + } + let starts = crate::crdt_runtime::paragraph_body_starts_in_loro(doc, &distinct); + let mut matches = Vec::with_capacity(replace.matches.len()); + for entry in &replace.matches { + let batched = entry.start.cursor.is_none() && entry.end.cursor.is_none() && entry.start.paragraph == entry.end.paragraph; + let (start, end) = if batched { + // Stale identity ⇒ skip this match (not reject): the surviving + // matches still carry the user's intent. + let Some(paragraph_ix) = index.paragraph_index(entry.start.paragraph) else { + continue; + }; + if projection.ids.paragraph_ids.get(paragraph_ix).copied() != Some(entry.start.paragraph) { + continue; + } + // Records without live cursors (freshly seeded paragraphs awaiting + // repair) fall back to the index's maintained starts — in-sync with + // the live doc inside the gate, same basis the per-anchor path uses. + let paragraph_start = match starts[distinct_slots[&entry.start.paragraph]] { + Some(start) => start, + None => { + let Some(start) = index.body_unicode_for_offset_in_loro( + doc, + projection, + DocumentOffset { + paragraph: paragraph_ix, + byte: 0, + }, + ) else { + continue; + }; + start + }, + }; + let text = flowstate_document::paragraph_text(projection, paragraph_ix); + let start_byte = super::resolve::clamp_byte_to_char_boundary(projection, paragraph_ix, entry.start.byte_hint); + let end_byte = super::resolve::clamp_byte_to_char_boundary(projection, paragraph_ix, entry.end.byte_hint); + let (start_byte, end_byte) = if start_byte <= end_byte { + (start_byte, end_byte) + } else { + (end_byte, start_byte) + }; + let start_unicode = paragraph_start + text[..start_byte].chars().count(); + let end_unicode = start_unicode + text[start_byte..end_byte].chars().count(); + ( + ResolvedTextPosition { + paragraph_ix, + byte: start_byte, + body_unicode: start_unicode, + }, + ResolvedTextPosition { + paragraph_ix, + byte: end_byte, + body_unicode: end_unicode, + }, + ) + } else { + let Ok(range) = resolve_text_range(doc, projection, index, &entry.start, &entry.end) else { + continue; + }; + range + }; + // Skip (not reject) matches that concurrent edits collapsed or moved + // across a paragraph boundary — the surviving matches still carry the + // user's intent (intent contract). + if start.body_unicode == end.body_unicode || start.paragraph_ix != end.paragraph_ix { + continue; + } + matches.push((start, end, entry.styles)); + } + // Prune overlaps preferring the EARLIER match (search matches are + // disjoint; an overlap means concurrency moved them, and the first + // match is the user-visible winner), then flip to descending for + // back-to-front application. + matches.sort_by_key(|entry| entry.0.body_unicode); + let mut kept: Vec<(ResolvedTextPosition, ResolvedTextPosition, Option)> = Vec::with_capacity(matches.len()); + for entry in matches { + if kept + .last() + .is_none_or(|prev| entry.0.body_unicode >= prev.1.body_unicode) + { + kept.push(entry); + } + } + kept.reverse(); + if kept.is_empty() { + return Err(WriteRejected::EmptyIntent); + } + Ok(ResolvedPlan::ReplaceMatches { + matches: kept, + replacement: replace.replacement.clone(), + }) + }, + LocalIntent::ReplaceEquationSourceRange(eq) => { + index + .block_index(eq.equation) + .ok_or(WriteRejected::UnresolvedBlock(eq.equation))?; + Ok(ResolvedPlan::ReplaceEquationSourceRange { + equation: eq.equation, + range: eq.range.clone(), + text: eq.text.clone(), + }) + }, + LocalIntent::ReplaceImageAltText(img) => { + index + .block_index(img.image) + .ok_or(WriteRejected::UnresolvedBlock(img.image))?; + Ok(ResolvedPlan::ReplaceImageAltText { + image: img.image, + text: img.text.clone(), + }) + }, + LocalIntent::ReplaceImageCaption(img) => { + index + .block_index(img.image) + .ok_or(WriteRejected::UnresolvedBlock(img.image))?; + Ok(ResolvedPlan::ReplaceImageCaption { + image: img.image, + caption: img.caption.clone(), + }) + }, + LocalIntent::SetImageLayout(img) => { + index + .block_index(img.image) + .ok_or(WriteRejected::UnresolvedBlock(img.image))?; + Ok(ResolvedPlan::SetImageLayout { + image: img.image, + sizing: img.sizing.clone(), + alignment: img.alignment, + }) + }, + LocalIntent::Table(table_intent) => resolve_table_intent(core, table_intent), + } +} + +fn resolve_table_intent(core: &CrdtRuntime, intent: &TableIntent) -> Result { + let index = core.projection_index_ref(); + let projection = core.projection_ref(); + let table = *match intent { + TableIntent::InsertRow { table, .. } + | TableIntent::DeleteRow { table, .. } + | TableIntent::MoveRow { table, .. } + | TableIntent::InsertColumn { table, .. } + | TableIntent::DeleteColumn { table, .. } + | TableIntent::MoveColumn { table, .. } + | TableIntent::ReplaceCell { table, .. } + | TableIntent::SetCellSpan { table, .. } + | TableIntent::SetColumnWidth { table, .. } => table, + }; + let table_ix = index + .block_index(table) + .ok_or(WriteRejected::UnresolvedBlock(table))?; + let table_block = match projection.blocks.get(table_ix) { + Some(flowstate_document::Block::Table(block)) => block, + _ => { + return Err(WriteRejected::UnresolvedTableEntity { + table, + detail: "projection block is not a table".to_string(), + }); + }, + }; + let op = match intent { + TableIntent::InsertRow { after_row, row, .. } => ResolvedTableOp::InsertRow { + new_row: row.clone(), + after_row: *after_row, + }, + TableIntent::DeleteRow { row, .. } => { + require_row(table_block, table, *row)?; + ResolvedTableOp::DeleteRow { row: *row } + }, + TableIntent::MoveRow { row, after_row, .. } => { + require_row(table_block, table, *row)?; + ResolvedTableOp::MoveRow { + row: *row, + after_row: *after_row, + } + }, + TableIntent::InsertColumn { after_column, width, .. } => { + let new_column = gpui_flowtext::ColumnId(Uuid::new_v4().as_u128()); + let cells = table_block + .rows + .iter() + .map(|row| crate::crdt_runtime::empty_input_table_cell(row.id, new_column)) + .collect(); + ResolvedTableOp::InsertColumn { + new_column, + after_column: *after_column, + width: width.clone(), + cells, + } + }, + TableIntent::DeleteColumn { column, .. } => { + require_column(table_block, table, *column)?; + ResolvedTableOp::DeleteColumn { column: *column } + }, + TableIntent::MoveColumn { column, after_column, .. } => { + require_column(table_block, table, *column)?; + ResolvedTableOp::MoveColumn { + column: *column, + after_column: *after_column, + } + }, + TableIntent::ReplaceCell { row, column, cell, .. } => { + require_row(table_block, table, *row)?; + require_column(table_block, table, *column)?; + ResolvedTableOp::ReplaceCell { + row: *row, + column: *column, + cell: cell.clone(), + } + }, + TableIntent::SetCellSpan { + row, + column, + row_span, + column_span, + .. + } => { + require_row(table_block, table, *row)?; + require_column(table_block, table, *column)?; + ResolvedTableOp::SetCellSpan { + row: *row, + column: *column, + row_span: *row_span, + column_span: *column_span, + } + }, + TableIntent::SetColumnWidth { column, width, .. } => { + let column_ix = require_column(table_block, table, *column)?; + ResolvedTableOp::SetColumnWidth { + column_ix, + width: width.clone(), + } + }, + }; + Ok(ResolvedPlan::Table { table, table_ix, op }) +} + +fn require_row(table: &flowstate_document::TableBlock, table_id: BlockId, row: gpui_flowtext::RowId) -> Result { + table + .rows + .iter() + .position(|candidate| candidate.id == row) + .ok_or(WriteRejected::UnresolvedTableEntity { + table: table_id, + detail: format!("row {} not present", row.0), + }) +} + +fn require_column(table: &flowstate_document::TableBlock, table_id: BlockId, column: gpui_flowtext::ColumnId) -> Result { + table + .columns + .iter() + .position(|candidate| candidate.id == column) + .ok_or(WriteRejected::UnresolvedTableEntity { + table: table_id, + detail: format!("column {} not present", column.0), + }) +} + +// --------------------------------------------------------------------------- +// Mutation +// --------------------------------------------------------------------------- + +#[hotpath::measure] +/// §act-four M6: if `targets` is a GAP-FREE paragraph run (e.g. select-all), a +/// uniform restyle can be one range mark over `[first_boundary, last_boundary+1)` +/// instead of N per-boundary marks — the marked span covers exactly those +/// paragraphs' boundary `\n`s and nothing beyond the last target's boundary. +/// Returns `None` (keep the per-boundary path) for fewer than two targets or any +/// gap, so a non-contiguous multi-select never restyles a paragraph between the +/// gaps. `boundary` is `targets`' third field; `paragraph_ix` the second. +fn contiguous_boundary_span(targets: &[(ParagraphId, usize, usize)]) -> Option<(usize, usize)> { + if targets.len() < 2 { + return None; + } + let mut indices: Vec = targets + .iter() + .map(|(_, paragraph_ix, _)| *paragraph_ix) + .collect(); + indices.sort_unstable(); + if indices.windows(2).any(|pair| pair[1] != pair[0] + 1) { + return None; // a gap → a range mark would restyle a non-target paragraph. + } + let lo = targets.iter().map(|(_, _, boundary)| *boundary).min()?; + let hi = targets.iter().map(|(_, _, boundary)| *boundary).max()?; + Some((lo, hi + 1)) +} + +fn execute_plan(core: &mut CrdtRuntime, plan: &ResolvedPlan) -> Result { + let doc = core.doc().clone(); + let projection = hotpath::measure_block!("execute_plan_projection_clone", core.projection_ref().clone()); + let mut summary = MutationSummary::default(); + match plan { + ResolvedPlan::InsertText { at, text, style_override } => { + let body = body_text(&doc); + body + .insert(at.body_unicode, text) + .context("inserting intent text into Loro body flow")?; + summary.containers_touched = 1; + let inserted = text.chars().count(); + // Style inheritance is expand-`After`'s job (spec §9); an override marks + // exactly the inserted range. + if let Some(styles) = style_override + && inserted > 0 + { + mark_run_styles(&body, at.body_unicode..at.body_unicode + inserted, *styles).context("marking caret style override")?; + summary.marks_emitted = style_mark_count(*styles); + } + summary.caret_body_unicode = Some(at.body_unicode + inserted); + }, + ResolvedPlan::DeleteRange { start, end } => { + let body = body_text(&doc); + let len = end.body_unicode - start.body_unicode; + let Some((clamped_start, clamped_len)) = sentinel_protected_delete_range(start.body_unicode, len) else { + anyhow::bail!("delete range collapsed by sentinel protection"); + }; + hotpath::measure_block!("delete_range_body_delete", { + body + .delete(clamped_start, clamped_len) + .context("deleting intent range from Loro body flow")?; + }); + summary.containers_touched = 1; + // Cross-paragraph deletes remove newline boundaries: orphaned object + // blocks and dead paragraph records must be retired with the text. The + // dead set is known exactly from the resolved plan — every paragraph + // after the first absorbs into it — so retirement is O(edit), not the + // O(records) prune sweep (§11). + if start.paragraph_ix != end.paragraph_ix { + // The prune sweep stringifies the whole body (O(doc) + a multi-MB + // allocation); only a delete whose PRE-STATE range covered an object + // placeholder can orphan an object block, so consult the maintained + // placeholder index first — the common cross-paragraph TEXT delete + // skips the sweep entirely (measured 66ms per delete on the + // reference doc). + let object_in_range = { + let positions = core.projection_index_ref().object_positions(); + let from = positions.partition_point(|&pos| pos < clamped_start); + positions + .get(from) + .is_some_and(|&pos| pos < clamped_start + clamped_len) + }; + if object_in_range { + hotpath::measure_block!("delete_range_prune_objects", { + prune_orphaned_body_object_blocks(&doc, &body).context("pruning orphaned object blocks after cross-paragraph delete")?; + }); + } + hotpath::measure_block!("delete_range_retire_records", { + // Rows in one O(blocks) pass instead of an O(blocks) scan per retired + // paragraph (§perf-heaven T2 quadratic on object docs). + let rows = flowstate_document::paragraph_block_rows(&projection); + for paragraph_ix in (start.paragraph_ix + 1)..=end.paragraph_ix { + let (Some(paragraph_id), Some(block_id)) = ( + projection.ids.paragraph_ids.get(paragraph_ix).copied(), + rows + .get(paragraph_ix) + .and_then(|&ix| projection.ids.block_ids.get(ix).copied()), + ) else { + continue; + }; + delete_projection_paragraph_metadata(&doc, paragraph_id, block_id) + .context("retiring merged-away paragraph records after cross-paragraph delete")?; + } + }); + } + summary.caret_body_unicode = Some(clamped_start); + }, + ResolvedPlan::SplitParagraph { + at, + inherited_style, + new_paragraph, + new_block, + } => { + let body = body_text(&doc); + body + .insert(at.body_unicode, "\n") + .context("inserting split boundary")?; + body + .mark( + at.body_unicode..at.body_unicode + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(*inherited_style), + ) + .context("marking split paragraph style")?; + // Sentinel hygiene (spec §9): expand-`After` run marks ending exactly at + // the split point absorb the inserted newline; strip run-style keys from + // the sentinel so styling never bleeds across the boundary. + unmark_run_style_keys(&body, at.body_unicode..at.body_unicode + 1)?; + summary.marks_emitted = 1; + repair_paragraph_metadata_after_stable_split(&doc, &body, at.body_unicode, *new_paragraph, *new_block, "local_split_paragraph") + .context("writing split paragraph records")?; + summary.containers_touched = 3; // body text + paragraph record + block record + summary.caret_body_unicode = Some(at.body_unicode + 1); + }, + ResolvedPlan::JoinParagraphs { first, second, .. } => { + if !join_projection_paragraphs(&doc, &projection, *first, *second).context("joining paragraphs")? { + anyhow::bail!("join mutation reported no-op for resolved adjacent paragraphs"); + } + summary.containers_touched = 2; + }, + ResolvedPlan::SetMarks { start, end, styles } => { + let body = body_text(&doc); + mark_run_styles(&body, start.body_unicode..end.body_unicode, *styles).context("marking style range")?; + summary.containers_touched = 1; + summary.marks_emitted = style_mark_count(*styles); + }, + ResolvedPlan::SetParagraphStyle { paragraph_ix, style, .. } => { + let boundary = paragraph_boundary_loro_unicode_index(&doc, &projection, *paragraph_ix); + body_text(&doc) + .mark(boundary..boundary + 1, MARK_PARAGRAPH_STYLE, paragraph_style_value(*style)) + .context("marking paragraph style")?; + summary.containers_touched = 1; + summary.marks_emitted = 1; + }, + ResolvedPlan::SetParagraphStyles { targets, style } => { + let body = body_text(&doc); + let value = paragraph_style_value(*style); + // §act-four M6 (range-marked styles): a UNIFORM restyle of a CONTIGUOUS + // paragraph run collapses from `O(paragraphs)` per-boundary marks to ONE + // range mark over the boundary span — the sub-`O(change)` write lever for + // select-all-restyle. A Loro range mark sets `paragraph_style` on every + // covered boundary `\n`, which is exactly what the materializer reads + // per boundary; the same key on the spanned TEXT chars is inert (the + // materializer reads `paragraph_style` only at boundaries, and it is a + // distinct mark key from run styles). Gated on `targets` being a + // gap-free paragraph run so no non-target paragraph is restyled. + if let Some((lo, hi)) = contiguous_boundary_span(targets) { + body + .mark(lo..hi, MARK_PARAGRAPH_STYLE, value) + .context("marking paragraph style (uniform range)")?; + summary.marks_emitted = 1; + } else { + for (_, _, boundary) in targets { + body + .mark(*boundary..*boundary + 1, MARK_PARAGRAPH_STYLE, value) + .context("marking paragraph style (batched)")?; + } + summary.marks_emitted = u32::try_from(targets.len()).unwrap_or(u32::MAX); + } + summary.containers_touched = 1; + }, + ResolvedPlan::InsertObject { + block_ix, new_block, block, .. + } => { + if !insert_projection_object_block(&doc, &projection, *new_block, *block_ix, block).context("inserting object block")? { + anyhow::bail!("object insert reported no-op for freshly minted block id"); + } + summary.containers_touched = 2; + }, + ResolvedPlan::ReplaceObject { block, block_ix, after } => { + if !replace_projection_object_block(&doc, &projection, Some(*block), *block_ix, after).context("replacing object block")? { + anyhow::bail!("object replace reported no-op for resolved block"); + } + summary.containers_touched = 2; + }, + ResolvedPlan::DeleteBlocks { blocks } => { + for (block, _) in blocks { + if !delete_projection_object_block(&doc, *block).context("deleting object block")? { + anyhow::bail!("object delete reported no-op for resolved block {}", block.0); + } + } + summary.containers_touched = u32::try_from(blocks.len()).unwrap_or(u32::MAX); + }, + ResolvedPlan::MoveBlock { block, to_ix, .. } => { + if !move_projection_object_block(&doc, &projection, *block, *to_ix).context("moving object block")? { + anyhow::bail!("object move reported no-op for resolved block"); + } + summary.containers_touched = 1; + }, + ResolvedPlan::InsertRichFragment { at, blocks } => { + summary.containers_touched = execute_rich_fragment(&doc, &projection, at, blocks)?; + summary.caret_body_unicode = None; // computed from patch plan by caller policy + }, + ResolvedPlan::ReplaceEquationSourceRange { equation, range, text } => { + if !replace_projection_equation_source_range(&doc, *equation, range, text).context("replacing equation source range")? { + anyhow::bail!("equation source replace reported no-op for resolved block"); + } + summary.containers_touched = 1; + }, + ResolvedPlan::ReplaceImageAltText { image, text } => { + if !replace_projection_image_alt_text(&doc, *image, text).context("replacing image alt text")? { + anyhow::bail!("image alt-text replace reported no-op for resolved block"); + } + summary.containers_touched = 1; + }, + ResolvedPlan::ReplaceImageCaption { image, caption } => { + if !replace_projection_image_caption(&doc, *image, caption.as_ref()).context("replacing image caption")? { + anyhow::bail!("image caption replace reported no-op for resolved block"); + } + summary.containers_touched = 1; + }, + ResolvedPlan::SetImageLayout { image, sizing, alignment } => { + if !set_projection_image_layout(&doc, *image, sizing, *alignment).context("setting image layout")? { + anyhow::bail!("image layout set reported no-op for resolved block"); + } + summary.containers_touched = 1; + }, + ResolvedPlan::Table { table, op, .. } => { + summary.containers_touched = execute_table_op(&doc, *table, op)?; + }, + ResolvedPlan::ReplaceMatches { matches, replacement } => { + let body = body_text(&doc); + let replacement_chars = replacement.chars().count(); + // Descending order (resolution contract): applying back-to-front means + // no earlier range's position shifts. + for (start, end, styles) in matches { + let len = end.body_unicode - start.body_unicode; + hotpath::measure_block!("replace_matches_body_edit", { + body + .delete(start.body_unicode, len) + .context("deleting matched range from Loro body flow")?; + if replacement_chars > 0 { + body + .insert(start.body_unicode, replacement) + .context("inserting replacement into Loro body flow")?; + } + }); + if replacement_chars > 0 + && let Some(styles) = styles + { + mark_run_styles(&body, start.body_unicode..start.body_unicode + replacement_chars, *styles) + .context("marking replacement run styles")?; + summary.marks_emitted += style_mark_count(*styles); + } + } + summary.containers_touched = 1; + // First entry = LAST match in document order; the caret lands after its + // replacement (find/replace UX contract). + summary.caret_body_unicode = matches + .first() + .map(|(start, _, _)| start.body_unicode + replacement_chars); + }, + } + Ok(summary) +} + +// Test-only deterministic fault: when set, `execute_rich_fragment` fails +// after its first block — the §13.6 compound-intent atomicity proof. +// THREAD-LOCAL: the local write path runs synchronously on the calling +// thread, and a process-global flag leaked the injected fault into every +// concurrently-running test that seeds a multi-block fragment (a ~1-in-5 +// suite flake once more fragment-seeding tests existed). +#[cfg(test)] +thread_local! { + pub(crate) static INJECT_FRAGMENT_FAULT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Compound fragment insertion: paragraphs splice into body text (with +/// boundaries + styles + marks), objects insert at their positions. One gate +/// hold, one commit; a failure anywhere trips I-10 compensation in the caller. +fn execute_rich_fragment(doc: &LoroDoc, projection: &DocumentProjection, at: &ResolvedTextPosition, blocks: &[FragmentBlock]) -> Result { + let body = body_text(doc); + let mut containers: u32 = 1; + let mut cursor_unicode = at.body_unicode; + let mut first = true; + for (fragment_block_ix, block) in blocks.iter().enumerate() { + #[cfg(test)] + if fragment_block_ix > 0 && INJECT_FRAGMENT_FAULT.with(std::cell::Cell::get) { + anyhow::bail!("injected fragment fault (test)"); + } + #[cfg(not(test))] + let _ = fragment_block_ix; + match block { + FragmentBlock::Paragraph(paragraph) => { + if !first { + // New paragraph boundary before this fragment paragraph. + body + .insert(cursor_unicode, "\n") + .context("inserting fragment paragraph boundary")?; + body + .mark( + cursor_unicode..cursor_unicode + 1, + MARK_PARAGRAPH_STYLE, + paragraph_style_value(paragraph.style), + ) + .context("marking fragment paragraph style")?; + unmark_run_style_keys(&body, cursor_unicode..cursor_unicode + 1)?; + let new_paragraph = ParagraphId(Uuid::new_v4().as_u128()); + let new_block = BlockId(Uuid::new_v4().as_u128()); + repair_paragraph_metadata_after_stable_split(doc, &body, cursor_unicode, new_paragraph, new_block, "local_fragment_paragraph") + .context("writing fragment paragraph records")?; + containers = containers.saturating_add(2); + cursor_unicode += 1; + } + for run in ¶graph.runs { + if run.text.is_empty() { + continue; + } + let len = run.text.chars().count(); + body + .insert(cursor_unicode, &run.text) + .context("inserting fragment run text")?; + mark_run_styles(&body, cursor_unicode..cursor_unicode + len, run.styles).context("marking fragment run styles")?; + cursor_unicode += len; + } + first = false; + }, + FragmentBlock::Object(input) => { + let new_block = BlockId(Uuid::new_v4().as_u128()); + // Objects sit between paragraphs; land it at the current position. + crate::crdt_runtime::insert_input_object_block(doc, cursor_unicode, new_block, input).context("inserting fragment object block")?; + containers = containers.saturating_add(2); + cursor_unicode += 1; + first = false; + }, + } + } + let _ = projection; + Ok(containers) +} + +fn execute_table_op(doc: &LoroDoc, table: BlockId, op: &ResolvedTableOp) -> Result { + let applied = match op { + ResolvedTableOp::InsertRow { new_row, after_row } => table_ops::insert_table_row(doc, table, new_row.id, *after_row, new_row)?, + ResolvedTableOp::DeleteRow { row } => table_ops::delete_table_row(doc, table, *row)?, + ResolvedTableOp::MoveRow { row, after_row } => table_ops::move_table_row(doc, table, *row, *after_row)?, + ResolvedTableOp::InsertColumn { + new_column, + after_column, + width, + cells, + } => table_ops::insert_table_column(doc, table, *new_column, *after_column, width, cells)?, + ResolvedTableOp::DeleteColumn { column } => table_ops::delete_table_column(doc, table, *column)?, + ResolvedTableOp::MoveColumn { column, after_column } => table_ops::move_table_column(doc, table, *column, *after_column)?, + ResolvedTableOp::ReplaceCell { row, column, cell } => table_ops::replace_table_cell(doc, table, *row, *column, cell)?, + ResolvedTableOp::SetCellSpan { + row, + column, + row_span, + column_span, + } => table_ops::set_table_cell_span(doc, table, *row, *column, *row_span, *column_span)?, + ResolvedTableOp::SetColumnWidth { column_ix, width, .. } => table_ops::set_table_column_width(doc, table, *column_ix, width)?, + }; + if !applied { + anyhow::bail!("table op reported no-op for resolved entities"); + } + Ok(2) +} + +/// Strip run-style mark keys from a range (split-sentinel hygiene, spec §9). +fn unmark_run_style_keys(body: &loro::LoroText, range: std::ops::Range) -> Result<()> { + for key in [MARK_RUN_SEMANTIC_STYLE, MARK_HIGHLIGHT_STYLE, MARK_DIRECT_UNDERLINE, MARK_STRIKETHROUGH] { + body + .unmark(range.clone(), key) + .with_context(|| format!("unmarking run-style key {key} on split sentinel"))?; + } + Ok(()) +} + +fn style_mark_count(styles: flowstate_document::RunStyles) -> u32 { + // One mark op per style key the mark writer touches; RunStyles is a small + // fixed set, so this is a constant bound rather than an exact per-key count. + let _ = styles; + 4 +} + +// --------------------------------------------------------------------------- +// Selection +// --------------------------------------------------------------------------- + +/// Cursor-backed selection after the intent (spec §8): anchor the character at +/// the caret's live body position and let the app-side delta express +/// stickiness. +#[hotpath::measure] +fn selection_after(core: &CrdtRuntime, summary: &MutationSummary) -> Option { + let caret_unicode = summary.caret_body_unicode?; + let body = body_text(core.doc()); + let cursor = cursor_for_boundary(&body, caret_unicode, crate::presence::SelectionAffinity::Neutral)?; + let offset = core + .projection_index_ref() + .offset_for_body_unicode(core.projection_ref(), caret_unicode) + .unwrap_or(DocumentOffset { paragraph: 0, byte: 0 }); + let endpoint = CursorEndpoint { + cursor: cursor.encode(), + delta: 0, + affinity: SelectionAffinity::Neutral, + gravity: VisualGravity::Neutral, + offset, + }; + Some(SelectionSnapshot { + anchor: endpoint.clone(), + head: endpoint, + }) +} diff --git a/crates/flowstate-collab/src/local_write/gate.rs b/crates/flowstate-collab/src/local_write/gate.rs new file mode 100644 index 00000000..5b918767 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/gate.rs @@ -0,0 +1,375 @@ +//! The exclusive write gate (spec I-9/I-9a/I-9b). +//! +//! Every access to the shared `LoroDoc` — mutation, import, export, fork, +//! checkout, cursor resolution — happens inside this gate. Loro 1.13.1 makes +//! virtually every API call a potential commit barrier (export/fork/diff/ +//! checkout/`get_cursor_pos`-on-deleted-anchor all `with_barrier`, which +//! commits any pending transaction), so an ungated call from another thread +//! mid-intent would seal and publish a half-applied intent. The gate widens +//! Loro's per-call serialization to whole-intent serialization. +//! +//! The gate is a *measured critical section*: every acquisition records the +//! holder, wait time, hold time, and whether another thread was blocked on it +//! (`contended`). The import-chunk hold-time budget in the perf suite is +//! enforced against these records — one import chunk's gate hold is the +//! maximum possible typing stall. +//! +//! Panic policy (spec I-10d): a panic while holding the gate poisons it, and a +//! poisoned gate means the underlying Loro doc may hold a half-applied intent +//! whose Loro mutexes are themselves poisoned. There is no in-place recovery; +//! the owner must reload from persisted state. `WriteGate::lock` therefore +//! propagates poisoning as a hard error instead of hiding it. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Instant; + +/// Who is holding (or asking for) the gate. Used for structured hold records +/// and the per-holder hold-time budgets. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GateHolder { + /// A local editing intent on the UI thread (resolve → mutate → commit → patch). + LocalIntent, + /// Undo/redo executed through the Loro `UndoManager`. + UndoRedo, + /// One remote import chunk (a single update blob or one coalesced + /// `import_batch` slice) plus its post-import patch derivation. + ImportChunk, + /// An incremental update export (`export(updates(vv))`) for publish or + /// anti-entropy. + ExportUpdates, + /// The brief hold needed to `fork()` the doc so a snapshot/package export can + /// run off-gate against the fork. + ExportFork, + /// Presence caret/selection resolution (cursor decode + position lookup). + /// Gate-held because `get_cursor_pos` on a deleted anchor is a commit + /// barrier (spec I-9a). + Presence, + /// Revision open/fork, package checkpoint, and other document services. + DocumentService, + /// Test-only holder. + #[cfg(test)] + Test, +} + +impl GateHolder { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::LocalIntent => "local-intent", + Self::UndoRedo => "undo-redo", + Self::ImportChunk => "import-chunk", + Self::ExportUpdates => "export-updates", + Self::ExportFork => "export-fork", + Self::Presence => "presence", + Self::DocumentService => "document-service", + #[cfg(test)] + Self::Test => "test", + } + } +} + +/// One completed gate hold, emitted to tracing and folded into [`GateMetrics`]. +#[derive(Clone, Copy, Debug)] +pub struct GateHoldRecord { + pub holder: GateHolder, + pub wait_micros: u64, + pub hold_micros: u64, + /// True when at least one other thread started waiting on the gate while this + /// hold was in progress (i.e. this hold *caused* blocking), or when this + /// acquisition itself had to wait. Either direction is a contention signal. + pub contended: bool, +} + +/// Aggregate gate counters (always-on; cheap atomics). The perf suite asserts +/// budget ceilings against the per-holder maxima. +#[derive(Debug, Default)] +pub struct GateMetrics { + pub acquisitions: AtomicU64, + pub contended_acquisitions: AtomicU64, + pub total_wait_micros: AtomicU64, + pub total_hold_micros: AtomicU64, + pub max_hold_micros_local_intent: AtomicU64, + pub max_hold_micros_import_chunk: AtomicU64, + pub max_hold_micros_other: AtomicU64, + pub max_wait_micros_local_intent: AtomicU64, +} + +impl GateMetrics { + fn record(&self, record: GateHoldRecord) { + self.acquisitions.fetch_add(1, Ordering::Relaxed); + if record.contended { + self.contended_acquisitions.fetch_add(1, Ordering::Relaxed); + } + self + .total_wait_micros + .fetch_add(record.wait_micros, Ordering::Relaxed); + self + .total_hold_micros + .fetch_add(record.hold_micros, Ordering::Relaxed); + let max_hold = match record.holder { + GateHolder::LocalIntent | GateHolder::UndoRedo => &self.max_hold_micros_local_intent, + GateHolder::ImportChunk => &self.max_hold_micros_import_chunk, + _ => &self.max_hold_micros_other, + }; + max_hold.fetch_max(record.hold_micros, Ordering::Relaxed); + if matches!(record.holder, GateHolder::LocalIntent | GateHolder::UndoRedo) { + self + .max_wait_micros_local_intent + .fetch_max(record.wait_micros, Ordering::Relaxed); + } + } +} + +/// The write gate: a poisoning mutex around the shared document core, with +/// measured acquisitions. +pub struct WriteGate { + inner: Mutex, + metrics: Arc, + /// Number of threads currently blocked in `lock()`. Lets a holder learn, at + /// release time, that somebody was waiting on it (the `contended` flag on its + /// hold record). + waiters: AtomicU64, + /// §A13.2: number of PRIORITY holders (local intents / undo) currently + /// blocked. Background holders defer briefly while this is non-zero — + /// `std::sync::Mutex` is unfair, so a hot background loop (import pump + /// under remote traffic) otherwise re-wins the gate release race for tens + /// of consecutive holds while a keystroke starves (measured: 2ms max hold + /// convoying into 37ms local waits). + priority_waiters: AtomicU64, + /// Set while any hold is active with waiters observed; cleared on release. + contention_seen: AtomicBool, +} + +/// A held gate. Releasing (dropping) emits the hold record. +pub struct GateGuard<'a, T> { + guard: Option>, + gate: &'a WriteGate, + holder: GateHolder, + wait_micros: u64, + acquired_at: Instant, + waited: bool, +} + +/// The gate was poisoned by a panic while held: the doc may contain a +/// half-applied intent and Loro's own mutexes are unusable. Spec I-10d: reload +/// from persisted state; do not attempt in-place recovery. +#[derive(Clone, Copy, Debug)] +pub struct GatePoisonedError; + +impl std::fmt::Display for GatePoisonedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("write gate poisoned by a panic while held; document core must be reloaded from persisted state (spec I-10d)") + } +} + +impl std::error::Error for GatePoisonedError {} + +impl WriteGate { + #[must_use] + pub fn new(inner: T) -> Self { + Self { + inner: Mutex::new(inner), + metrics: Arc::new(GateMetrics::default()), + waiters: AtomicU64::new(0), + priority_waiters: AtomicU64::new(0), + contention_seen: AtomicBool::new(false), + } + } + + #[must_use] + pub fn metrics(&self) -> Arc { + Arc::clone(&self.metrics) + } + + /// Acquire the gate for `holder`. Blocks until exclusive; the returned guard + /// measures the hold and emits a structured record on release. + pub fn lock(&self, holder: GateHolder) -> Result, GatePoisonedError> { + let requested_at = Instant::now(); + let priority = matches!(holder, GateHolder::LocalIntent | GateHolder::UndoRedo); + if !priority { + // §A13.2 priority lane: while a local edit is waiting, background + // holders step aside instead of racing it for the just-released gate. + // Bounded defer so a sustained typing burst can only delay (never + // starve) background progress; each deferral is at most one local + // hold's worth of time anyway (~0.1-0.4ms). + const MAX_PRIORITY_DEFER: std::time::Duration = std::time::Duration::from_millis(2); + if self.priority_waiters.load(Ordering::SeqCst) > 0 { + let defer_start = Instant::now(); + while self.priority_waiters.load(Ordering::SeqCst) > 0 && defer_start.elapsed() < MAX_PRIORITY_DEFER { + std::thread::yield_now(); + } + } + } + let (guard, waited) = match self.inner.try_lock() { + Ok(guard) => (guard, false), + Err(std::sync::TryLockError::Poisoned(_)) => return Err(GatePoisonedError), + Err(std::sync::TryLockError::WouldBlock) => { + self.waiters.fetch_add(1, Ordering::SeqCst); + if priority { + self.priority_waiters.fetch_add(1, Ordering::SeqCst); + } + self.contention_seen.store(true, Ordering::SeqCst); + let result = self.inner.lock(); + self.waiters.fetch_sub(1, Ordering::SeqCst); + if priority { + self.priority_waiters.fetch_sub(1, Ordering::SeqCst); + } + match result { + Ok(guard) => (guard, true), + Err(_) => return Err(GatePoisonedError), + } + }, + }; + let wait_micros = u64::try_from(requested_at.elapsed().as_micros()).unwrap_or(u64::MAX); + // A fresh hold starts with no observed waiters; anyone who blocks from here + // on flips `contention_seen`, which this hold reads at release. + self + .contention_seen + .store(self.waiters.load(Ordering::SeqCst) > 0, Ordering::SeqCst); + Ok(GateGuard { + guard: Some(guard), + gate: self, + holder, + wait_micros, + acquired_at: Instant::now(), + waited, + }) + } +} + +impl std::ops::Deref for GateGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + self + .guard + .as_ref() + .expect("gate guard accessed after release") + } +} + +impl std::ops::DerefMut for GateGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + self + .guard + .as_mut() + .expect("gate guard accessed after release") + } +} + +impl Drop for GateGuard<'_, T> { + fn drop(&mut self) { + let hold_micros = u64::try_from(self.acquired_at.elapsed().as_micros()).unwrap_or(u64::MAX); + let others_waited = self.gate.contention_seen.swap(false, Ordering::SeqCst) || self.gate.waiters.load(Ordering::SeqCst) > 0; + let record = GateHoldRecord { + holder: self.holder, + wait_micros: self.wait_micros, + hold_micros, + contended: self.waited || others_waited, + }; + // Release the doc before doing any bookkeeping I/O. + drop(self.guard.take()); + self.gate.metrics.record(record); + tracing::trace!( + holder = record.holder.as_str(), + wait_us = record.wait_micros, + hold_us = record.hold_micros, + contended = record.contended, + "write-gate hold", + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::Ordering; + + #[test] + fn gate_measures_holds_and_contention() { + let gate = Arc::new(WriteGate::new(0_u64)); + let guard_scope = gate.lock(GateHolder::Test).expect("gate healthy"); + drop(guard_scope); + let metrics = gate.metrics(); + assert_eq!(metrics.acquisitions.load(Ordering::Relaxed), 1); + assert_eq!(metrics.contended_acquisitions.load(Ordering::Relaxed), 0); + + // Contention: hold the gate on one thread while another waits. + let gate_b = Arc::clone(&gate); + let held = gate.lock(GateHolder::Test).expect("gate healthy"); + let waiter = std::thread::spawn(move || { + let guard = gate_b.lock(GateHolder::Test).expect("gate healthy"); + *guard + }); + // Give the waiter time to block, then release. + std::thread::sleep(std::time::Duration::from_millis(20)); + drop(held); + waiter.join().expect("waiter completes"); + assert!( + gate + .metrics() + .contended_acquisitions + .load(Ordering::Relaxed) + >= 1, + "waiting acquisition must be recorded as contended" + ); + } + + #[test] + fn priority_waiter_beats_a_hot_background_loop() { + // §A13.2: a background thread re-acquiring in a hot loop must not convoy + // out a waiting local intent (std Mutex is unfair without the lane). + let gate = Arc::new(WriteGate::new(0_u64)); + let stop = Arc::new(AtomicBool::new(false)); + let background = { + let gate = Arc::clone(&gate); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + let mut holds = 0u64; + while !stop.load(Ordering::Relaxed) { + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + *guard += 1; + std::thread::sleep(std::time::Duration::from_micros(200)); + drop(guard); + holds += 1; + } + holds + }) + }; + // Let the loop get hot, then time priority acquisitions from this thread. + std::thread::sleep(std::time::Duration::from_millis(10)); + let mut worst = std::time::Duration::ZERO; + for _ in 0..50 { + let t = Instant::now(); + let guard = gate.lock(GateHolder::LocalIntent).expect("gate healthy"); + worst = worst.max(t.elapsed()); + drop(guard); + std::thread::sleep(std::time::Duration::from_micros(300)); + } + stop.store(true, Ordering::Relaxed); + let holds = background.join().expect("background thread"); + assert!(holds > 0, "background loop must make progress (no starvation)"); + // Without the lane this convoys to many consecutive 200us holds; with it + // the worst wait is bounded by ~one hold plus scheduling noise. + assert!( + worst < std::time::Duration::from_millis(5), + "priority acquisition waited {worst:?} behind a hot background loop" + ); + } + + #[test] + fn gate_reports_poisoning_as_error() { + let gate = Arc::new(WriteGate::new(0_u64)); + let gate_b = Arc::clone(&gate); + let _ = std::thread::spawn(move || { + let _guard = gate_b.lock(GateHolder::Test).expect("gate healthy"); + panic!("poison the gate"); + }) + .join(); + assert!( + gate.lock(GateHolder::Test).is_err(), + "poisoned gate must surface as an error, not silence" + ); + } +} diff --git a/crates/flowstate-collab/src/local_write/handle.rs b/crates/flowstate-collab/src/local_write/handle.rs new file mode 100644 index 00000000..22a4ead8 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/handle.rs @@ -0,0 +1,359 @@ +//! `LocalDocHandle` — the ONE local write API (spec §4, D1/D2, invariant 5). +//! +//! Owned by the UI thread. Every method acquires the write gate, executes the +//! whole intent (resolve → mutate → commit → patch) inside it, and returns the +//! committed outcome synchronously — the "optimistic" apply IS the committed +//! apply. Solo documents and collaborative documents use this identical +//! handle; collaboration only adds the I/O service draining the publish queue +//! on the other side of the same gate. + +use std::num::NonZeroU32; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Instant; + +use flowstate_document::DocumentProjection; + +use super::commit::apply_local_intent; +use super::gate::{GateHolder, GateMetrics, WriteGate}; +use super::intents::{ + DeleteBlocksIntent, DeleteRangeIntent, InsertObjectIntent, InsertRichFragmentIntent, InsertTextIntent, JoinParagraphsIntent, LocalIntent, + LocalWriteAuthority, LocalWriteOutcome, MoveBlockIntent, ReplaceEquationSourceRangeIntent, ReplaceImageAltTextIntent, + ReplaceImageCaptionIntent, ReplaceMatchesIntent, ReplaceObjectIntent, SetImageLayoutIntent, SetMarksIntent, SetParagraphStyleIntent, + SetParagraphStylesIntent, SplitParagraphIntent, TableIntent, UndoOutcome, WriteRejected, +}; +use crate::crdt_runtime::{CrdtRuntime, RuntimeEvent, SemanticCommand}; + +/// Write-path configuration (spec §7 audit knob). +#[derive(Clone, Copy, Debug, Default)] +pub struct LocalWriteConfig { + /// Release-build projection-audit sampling: `Some(1)` audits every commit, + /// `Some(n)` one-in-n, `None` disables release auditing entirely (the + /// ratified one-line off switch). Debug/CI builds audit unconditionally + /// regardless of this knob. + /// + /// Default is OFF (2026-07-07): fidelity bugs have dried up in the field + /// while perf is under active measurement, and the audit's ~350ms sampled + /// hiccup on large docs pollutes that read. Re-enable for fidelity-hunting + /// builds. + pub release_audit_sample: Option, +} + +/// The single local write authority over one shared document core. +pub struct LocalDocHandle { + core: Arc>, + config: LocalWriteConfig, + intents_committed: AtomicU64, +} + +impl LocalDocHandle { + /// Wrap a document core. The returned gate handle is shared with the I/O + /// service (`DocIoService`) so imports/exports serialize with local writes. + #[must_use] + pub fn new(core: CrdtRuntime, config: LocalWriteConfig) -> (Self, Arc>) { + let gate = Arc::new(WriteGate::new(core)); + ( + Self { + core: Arc::clone(&gate), + config, + intents_committed: AtomicU64::new(0), + }, + gate, + ) + } + + /// Attach to an existing shared core (e.g. handed over from document open). + #[must_use] + pub fn attach(core: Arc>, config: LocalWriteConfig) -> Self { + Self { + core, + config, + intents_committed: AtomicU64::new(0), + } + } + + #[must_use] + pub fn gate_metrics(&self) -> Arc { + self.core.metrics() + } + + /// Clone the canonical projection (editor attach; afterwards the editor + /// advances its copy exclusively by patch batches). + pub fn projection(&self) -> Result { + let guard = self + .core + .lock(GateHolder::LocalIntent) + .map_err(|_| WriteRejected::GatePoisoned)?; + let projection = guard.projection_ref().clone(); + drop(guard); + Ok(projection) + } + + // ---- The intent API (spec §4) -------------------------------------------- + + pub fn insert_text(&self, intent: InsertTextIntent) -> Result { + self.apply_intent(LocalIntent::InsertText(intent)) + } + + pub fn delete_range(&self, intent: DeleteRangeIntent) -> Result { + self.apply_intent(LocalIntent::DeleteRange(intent)) + } + + pub fn split_paragraph(&self, intent: SplitParagraphIntent) -> Result { + self.apply_intent(LocalIntent::SplitParagraph(intent)) + } + + pub fn join_paragraphs(&self, intent: JoinParagraphsIntent) -> Result { + self.apply_intent(LocalIntent::JoinParagraphs(intent)) + } + + pub fn set_marks(&self, intent: SetMarksIntent) -> Result { + self.apply_intent(LocalIntent::SetMarks(intent)) + } + + pub fn set_paragraph_style(&self, intent: SetParagraphStyleIntent) -> Result { + self.apply_intent(LocalIntent::SetParagraphStyle(intent)) + } + + pub fn set_paragraph_styles(&self, intent: SetParagraphStylesIntent) -> Result { + self.apply_intent(LocalIntent::SetParagraphStyles(intent)) + } + + pub fn insert_object(&self, intent: InsertObjectIntent) -> Result { + self.apply_intent(LocalIntent::InsertObject(intent)) + } + + pub fn replace_object(&self, intent: ReplaceObjectIntent) -> Result { + self.apply_intent(LocalIntent::ReplaceObject(intent)) + } + + pub fn delete_blocks(&self, intent: DeleteBlocksIntent) -> Result { + self.apply_intent(LocalIntent::DeleteBlocks(intent)) + } + + pub fn move_block(&self, intent: MoveBlockIntent) -> Result { + self.apply_intent(LocalIntent::MoveBlock(intent)) + } + + pub fn insert_rich_fragment(&self, intent: InsertRichFragmentIntent) -> Result { + self.apply_intent(LocalIntent::InsertRichFragment(intent)) + } + + pub fn replace_matches(&self, intent: ReplaceMatchesIntent) -> Result { + self.apply_intent(LocalIntent::ReplaceMatches(intent)) + } + + pub fn replace_equation_source_range(&self, intent: ReplaceEquationSourceRangeIntent) -> Result { + self.apply_intent(LocalIntent::ReplaceEquationSourceRange(intent)) + } + + pub fn replace_image_alt_text(&self, intent: ReplaceImageAltTextIntent) -> Result { + self.apply_intent(LocalIntent::ReplaceImageAltText(intent)) + } + + pub fn replace_image_caption(&self, intent: ReplaceImageCaptionIntent) -> Result { + self.apply_intent(LocalIntent::ReplaceImageCaption(intent)) + } + + pub fn set_image_layout(&self, intent: SetImageLayoutIntent) -> Result { + self.apply_intent(LocalIntent::SetImageLayout(intent)) + } + + pub fn table_op(&self, intent: TableIntent) -> Result { + self.apply_intent(LocalIntent::Table(intent)) + } + + /// Generic dispatch — the fuzz/property harness entry point. + #[hotpath::measure] + pub fn apply_intent(&self, intent: LocalIntent) -> Result { + let started = Instant::now(); + let mut guard = self + .core + .lock(GateHolder::LocalIntent) + .map_err(|_| WriteRejected::GatePoisoned)?; + let mut outcome = apply_local_intent(&mut guard, &intent)?; + self.maybe_release_audit(&mut guard, &intent); + drop(guard); + let hold_micros = u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX); + match &mut outcome { + LocalWriteOutcome::Committed(commit) | LocalWriteOutcome::CommittedWithRebuild { commit, .. } => { + commit.counters.gate_hold_micros = hold_micros; + }, + } + self.intents_committed.fetch_add(1, Ordering::Relaxed); + Ok(outcome) + } + + /// Release-build sampled audit (spec §7). Debug builds audit inside + /// `apply_local_intent` unconditionally. + fn maybe_release_audit(&self, core: &mut CrdtRuntime, intent: &LocalIntent) { + #[cfg(not(debug_assertions))] + { + if let Some(sample) = self.config.release_audit_sample { + let n = self.intents_committed.load(Ordering::Relaxed); + // n starts at 0: auditing on `n % sample == 0` would put the O(doc) + // rebuild-and-compare on the FIRST keystroke of every session (field + // symptom: one high-latency op per fresh peer). Sample the Nth op. + if n % std::num::NonZeroU64::from(sample) == u64::from(sample.get()) - 1 + && let Err(error) = core.audit_projection_against_full_rebuild(intent.class()) + { + tracing::error!(%error, class = intent.class(), "sampled release audit mismatch"); + } + } + } + #[cfg(debug_assertions)] + { + let _ = (core, intent, &self.config); + } + } + + // ---- Undo (spec §10) ------------------------------------------------------ + + /// Execute undo through the Loro `UndoManager`. The manager commits + /// internally (origin `"undo"`); patches are a full projection replace (undo + /// can touch arbitrary spans). + pub fn apply_undo(&self) -> Result { + self.undo_redo(SemanticCommand::Undo) + } + + pub fn apply_redo(&self) -> Result { + self.undo_redo(SemanticCommand::Redo) + } + + fn undo_redo(&self, command: SemanticCommand) -> Result { + let mut guard = self + .core + .lock(GateHolder::UndoRedo) + .map_err(|_| WriteRejected::GatePoisoned)?; + let events = guard + .command(command) + .map_err(|error| WriteRejected::CompensatedFailure { + class: "undo-redo", + diagnostic: format!("{error:#}"), + })?; + let mut applied = false; + let mut selection = None; + let mut publish = Vec::new(); + for event in events { + match event { + // Whether expressed as a replace or incrementally: the projection + // change reaches the editor through the ORDERED stream (already + // queued by the runtime); the outcome only carries the applied + // signal. The former shape cloned the ENTIRE projection here per + // undo — pure waste, the editor never read it. + RuntimeEvent::ProjectionUpdated { .. } | RuntimeEvent::ProjectionPatched { .. } => applied = true, + RuntimeEvent::SelectionRestored { selection: restored } => selection = Some(restored), + RuntimeEvent::RevisionOpened { .. } | RuntimeEvent::RevisionForked { .. } | RuntimeEvent::HistoryRebaseRequired { .. } => {}, + publishable @ (RuntimeEvent::LocalUpdate { .. } | RuntimeEvent::RemoteUpdateApplied { .. }) => publish.push(publishable), + } + } + guard.queue_publish(publish); + drop(guard); + Ok(UndoOutcome { applied, selection }) + } + + /// Begin an undo group (word/burst boundary — spec §10). Fallible by design: + /// a non-disjoint remote import closes the active group underneath us + /// (semantics-audit F3), and a subsequent `group_start` while one is + /// notionally open reports an error. Returns whether a fresh group opened; + /// the editor re-arms at the next boundary either way. + pub fn begin_undo_group(&self) -> Result { + let mut guard = self + .core + .lock(GateHolder::UndoRedo) + .map_err(|_| WriteRejected::GatePoisoned)?; + match guard.undo_manager_mut().group_start() { + Ok(()) => Ok(true), + Err(error) => { + tracing::debug!(%error, "undo group_start declined (group already open or manager not ready); re-arming at next boundary"); + Ok(false) + }, + } + } + + pub fn finish_undo_group(&self) -> Result<(), WriteRejected> { + let mut guard = self + .core + .lock(GateHolder::UndoRedo) + .map_err(|_| WriteRejected::GatePoisoned)?; + guard.undo_manager_mut().group_end(); + drop(guard); + Ok(()) + } +} + +/// The one write path, as injected into the editor (spec invariant 5): solo +/// and collaborative documents receive this same authority object. +impl LocalWriteAuthority for LocalDocHandle { + fn apply(&self, intent: LocalIntent) -> Result { + self.apply_intent(intent) + } + + fn undo(&self) -> Result { + self.apply_undo() + } + + fn redo(&self) -> Result { + self.apply_redo() + } + + fn undo_group_start(&self) -> Result { + self.begin_undo_group() + } + + fn undo_group_end(&self) -> Result<(), WriteRejected> { + self.finish_undo_group() + } + + fn drain_projection_stream(&self) -> Result, WriteRejected> { + let mut guard = self + .core + .lock(GateHolder::LocalIntent) + .map_err(|_| WriteRejected::GatePoisoned)?; + let items = guard.take_editor_stream(); + drop(guard); + Ok(items) + } + + fn canonical_projection(&self) -> Result { + self.projection() + } + + fn rebase_selection( + &self, + selection: &gpui_flowtext::EditorSelection, + before: &DocumentProjection, + editor_frontier: &[u8], + ) -> Option { + let guard = self.core.lock(GateHolder::LocalIntent).ok()?; + let rebased = guard.rebase_selection_across_import(selection, before, editor_frontier); + drop(guard); + rebased + } + + fn encode_selection_anchor(&self, selection: &gpui_flowtext::EditorSelection, editor_frontier: &[u8]) -> Option<(Vec, Vec)> { + let guard = self.core.lock(GateHolder::LocalIntent).ok()?; + // Only encode while the editor is in sync with the core. If a remote import + // has advanced the core but the editor has not yet drained it, the editor's + // offsets are in a stale projection; encoding them here would anchor the + // caret to the wrong body position. Skip → the next drain re-arms (or the + // rebase fork covers this rare window correctly). + if guard.doc().state_frontiers().encode() != editor_frontier { + return None; + } + let encoded = guard.encode_selection_anchor(selection); + drop(guard); + encoded + } + + fn resolve_selection_anchor( + &self, + head_cursor: &[u8], + anchor_cursor: &[u8], + ) -> Option<(flowstate_document::DocumentOffset, flowstate_document::DocumentOffset)> { + let guard = self.core.lock(GateHolder::LocalIntent).ok()?; + let resolved = guard.resolve_selection_anchor(head_cursor, anchor_cursor); + drop(guard); + resolved + } +} diff --git a/crates/flowstate-collab/src/local_write/intents.rs b/crates/flowstate-collab/src/local_write/intents.rs new file mode 100644 index 00000000..4bf47d4f --- /dev/null +++ b/crates/flowstate-collab/src/local_write/intents.rs @@ -0,0 +1,5 @@ +//! The intent vocabulary lives in `gpui_flowtext::local_intents` (the editor +//! constructs intents; crate dependency direction requires the types there). +//! This module re-exports it so the write path's internal imports stay stable. + +pub use gpui_flowtext::local_intents::*; diff --git a/crates/flowstate-collab/src/local_write/mod.rs b/crates/flowstate-collab/src/local_write/mod.rs new file mode 100644 index 00000000..edc2e180 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/mod.rs @@ -0,0 +1,35 @@ +//! Loro-first local write path (spec `flowstate_loro_first_spec.md` §4). +//! +//! The ONE way local editing mutates a document, identical for solo and +//! collaborative documents (invariant 5): +//! +//! ```text +//! editor input → typed intent → [write gate] resolve → mutate → commit +//! → exact projection patches, synchronously +//! ``` +//! +//! Raw projection-space commands do not exist here — the intent types make +//! them unrepresentable (invariant 6). Remote traffic reaches the same doc +//! only through `DocIoService`, which acquires the same gate per import chunk. + +pub mod gate; +pub mod handle; +pub mod intents; + +pub(crate) mod commit; +pub(crate) mod patch_synthesis; +pub(crate) mod recorded_inverse; +pub(crate) mod resolve; + +#[cfg(test)] +mod tests; + +pub use gate::{GateHoldRecord, GateHolder, GateMetrics, GatePoisonedError, WriteGate}; +pub use handle::{LocalDocHandle, LocalWriteConfig}; +pub use intents::{ + CursorEndpoint, DeleteBlocksIntent, DeleteRangeIntent, FragmentBlock, InsertObjectIntent, InsertRichFragmentIntent, InsertTextIntent, + IntentCounters, JoinParagraphsIntent, LocalCommit, LocalIntent, LocalWriteAuthority, LocalWriteOutcome, MoveBlockIntent, ProjectionReplace, + ReplaceEquationSourceRangeIntent, ReplaceImageAltTextIntent, ReplaceImageCaptionIntent, ReplaceMatch, ReplaceMatchesIntent, + ReplaceObjectIntent, SelectionSnapshot, SetImageLayoutIntent, SetMarksIntent, SetParagraphStyleIntent, SetParagraphStylesIntent, + SplitParagraphIntent, TableIntent, TextAnchor, UndoOutcome, WriteRejected, +}; diff --git a/crates/flowstate-collab/src/local_write/patch_synthesis.rs b/crates/flowstate-collab/src/local_write/patch_synthesis.rs new file mode 100644 index 00000000..85490ca4 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/patch_synthesis.rs @@ -0,0 +1,602 @@ +//! Intent-exact projection patch synthesis (spec §7, D4). +//! +//! Called after the intent's mutations have committed, with the core's +//! projection still at its PRE-intent state (patches apply onto it) and the +//! doc at its POST-intent state. Every patch is O(touched text/blocks); the +//! only intent class permitted to fall back to a full rebuild is the compound +//! rich-fragment insert, and that fallback is loud and counted +//! (`full-rebuild-after-local-write`). + +use flowstate_document::{ + DocumentProjection, InputBlock, ParagraphStyle, ProjectionPatch, ProjectionStructuralBlock, input_block_from_block, paragraph_text_len, +}; + +use super::commit::{ResolvedPlan, ResolvedTableOp}; +use super::intents::LocalIntent; +use super::resolve::ResolvedTextPosition; +use crate::crdt_runtime::{ + CrdtRuntime, ProjectionInvalidation, body_input_paragraph_at, object_replacement_patch, paragraph_boundary_loro_unicode_index, + projection_text_delta, text_delta_between, +}; + +/// What the projection should do for a committed intent. +pub(crate) enum PatchPlan { + Patches { + patches: Vec, + invalidation: ProjectionInvalidation, + }, + FullRebuild { + invalidation: ProjectionInvalidation, + reason: &'static str, + }, +} + +#[hotpath::measure] +pub(crate) fn synthesize_patches(core: &CrdtRuntime, intent: &LocalIntent, plan: &ResolvedPlan) -> PatchPlan { + let projection = core.projection_ref(); + let doc = core.doc(); + let frontier_before = projection.frontier.clone(); + let frontier_after = doc.state_frontiers().encode(); + let body_invalidation = |unicode_start: usize, unicode_len: usize| { + ProjectionInvalidation::body_text(frontier_before.clone(), frontier_after.clone(), unicode_start, unicode_len) + }; + let rebuild = |reason: &'static str| PatchPlan::FullRebuild { + invalidation: ProjectionInvalidation::full_rebuild(frontier_before.clone(), frontier_after.clone(), reason), + reason, + }; + + let patches = match plan { + ResolvedPlan::InsertText { at, text, .. } => { + let Some(row) = flowstate_document::block_ix_for_paragraph(projection, at.paragraph_ix) else { + return rebuild("insert-text-block-missing"); + }; + let old_len = projection + .paragraphs + .get(at.paragraph_ix) + .map(paragraph_text_len) + .unwrap_or(0); + let Some((paragraph_start, old_chars)) = resolved_paragraph_span(projection, at) else { + return rebuild("insert-text-position-misaligned"); + }; + let end = paragraph_start + old_chars + text.chars().count(); + let Some(new) = body_input_paragraph_at( + doc, + paragraph_start.saturating_sub(1), + end, + paragraph_style_at(projection, at.paragraph_ix), + ) else { + return rebuild("insert-text-readback-missing"); + }; + Some(( + vec![ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[at.paragraph_ix], + row_hint: row, + new, + delta_utf8: projection_text_delta(at.byte, 0, text.len(), old_len.saturating_sub(at.byte)), + }], + body_invalidation(at.body_unicode, text.chars().count()), + )) + }, + ResolvedPlan::DeleteRange { start, end } => { + let Some(row) = flowstate_document::block_ix_for_paragraph(projection, start.paragraph_ix) else { + return rebuild("delete-range-block-missing"); + }; + let Some((paragraph_start, first_old_chars)) = resolved_paragraph_span(projection, start) else { + return rebuild("delete-range-position-misaligned"); + }; + // Post-delete merged/truncated paragraph end: same-paragraph deletes + // shrink the paragraph by the deleted span; cross-paragraph deletes + // leave the first paragraph's prefix plus the last paragraph's tail. + let range_end = if start.paragraph_ix == end.paragraph_ix { + let deleted = end.body_unicode.saturating_sub(start.body_unicode); + let Some(remaining) = first_old_chars.checked_sub(deleted) else { + return rebuild("delete-range-length-misaligned"); + }; + paragraph_start + remaining + } else { + let last_text = flowstate_document::paragraph_text(projection, end.paragraph_ix); + let Some(tail_chars) = last_text.get(end.byte..).map(|tail| tail.chars().count()) else { + return rebuild("delete-range-tail-misaligned"); + }; + let prefix_chars = start.body_unicode - paragraph_start; + paragraph_start + prefix_chars + tail_chars + }; + let Some(new) = body_input_paragraph_at( + doc, + paragraph_start.saturating_sub(1), + range_end, + paragraph_style_at(projection, start.paragraph_ix), + ) else { + return rebuild("delete-range-readback-missing"); + }; + let old_first_len = projection + .paragraphs + .get(start.paragraph_ix) + .map(paragraph_text_len) + .unwrap_or(0); + let mut patches = Vec::new(); + if start.paragraph_ix == end.paragraph_ix { + patches.push(ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[start.paragraph_ix], + row_hint: row, + new, + delta_utf8: projection_text_delta(start.byte, end.byte.saturating_sub(start.byte), 0, old_first_len.saturating_sub(end.byte)), + }); + } else { + // Cross-paragraph delete: the first paragraph absorbs the tail of the + // last; every whole paragraph/object block strictly inside the range + // is deleted. + let merged_len = input_paragraph_text_len(&new); + patches.push(ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[start.paragraph_ix], + row_hint: row, + new, + delta_utf8: projection_text_delta( + start.byte, + old_first_len.saturating_sub(start.byte), + merged_len.saturating_sub(start.byte), + 0, + ), + }); + let Some(first_block_ix) = flowstate_document::block_ix_for_paragraph(projection, start.paragraph_ix) else { + return rebuild("delete-range-first-block-missing"); + }; + let Some(last_block_ix) = flowstate_document::block_ix_for_paragraph(projection, end.paragraph_ix) else { + return rebuild("delete-range-last-block-missing"); + }; + let removed: Vec<_> = ((first_block_ix + 1)..=last_block_ix) + .filter_map(|ix| projection.ids.block_ids.get(ix).copied()) + .collect(); + if !removed.is_empty() { + patches.push(ProjectionPatch::DeleteBlocks { + block_ids: removed, + row_hint: first_block_ix + 1, + }); + } + } + Some(( + patches, + body_invalidation(start.body_unicode, end.body_unicode.saturating_sub(start.body_unicode)), + )) + }, + ResolvedPlan::SplitParagraph { + at, + inherited_style, + new_paragraph, + new_block, + } => { + let Some(source_block_ix) = flowstate_document::block_ix_for_paragraph(projection, at.paragraph_ix) else { + return rebuild("split-source-block-missing"); + }; + let old_len = projection + .paragraphs + .get(at.paragraph_ix) + .map(paragraph_text_len) + .unwrap_or(0); + let Some((paragraph_start, old_chars)) = resolved_paragraph_span(projection, at) else { + return rebuild("split-position-misaligned"); + }; + // Post-split: `[start, at)` is the truncated source; the inserted `\n` + // at `at.body_unicode` is the tail's leading sentinel (explicitly marked + // with the inherited style by the mutation). + let split_chars = at.body_unicode - paragraph_start; + let source_style = paragraph_style_at(projection, at.paragraph_ix); + let (Some(truncated), Some(tail)) = ( + body_input_paragraph_at(doc, paragraph_start.saturating_sub(1), at.body_unicode, source_style), + body_input_paragraph_at( + doc, + at.body_unicode, + at.body_unicode + 1 + (old_chars.saturating_sub(split_chars)), + *inherited_style, + ), + ) else { + return rebuild("split-readback-missing"); + }; + Some(( + vec![ + ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[source_block_ix], + paragraph_id: projection.ids.paragraph_ids[at.paragraph_ix], + row_hint: source_block_ix, + new: truncated, + delta_utf8: projection_text_delta(at.byte, old_len.saturating_sub(at.byte), 0, 0), + }, + ProjectionPatch::InsertBlocks { + before: projection.ids.block_ids.get(source_block_ix + 1).copied(), + row_hint: source_block_ix + 1, + blocks: vec![ProjectionStructuralBlock { + block_id: *new_block, + paragraph_id: Some(*new_paragraph), + block: InputBlock::Paragraph(tail), + }], + }, + ], + body_invalidation(at.body_unicode, 1), + )) + }, + ResolvedPlan::JoinParagraphs { first, second, first_ix, .. } => { + let Some(first_block_ix) = flowstate_document::block_ix_for_paragraph(projection, *first_ix) else { + return rebuild("join-first-block-missing"); + }; + let Some(second_block_ix) = flowstate_document::block_ix_for_paragraph(projection, first_ix + 1) else { + return rebuild("join-second-block-missing"); + }; + // Post-join merged range: first paragraph's text, any object + // placeholders that sat between the two blocks (folded out of paragraph + // text by the readback), then the second paragraph's text. + let sentinel = paragraph_boundary_loro_unicode_index(doc, projection, *first_ix); + let first_chars = flowstate_document::paragraph_text(projection, *first_ix) + .chars() + .count(); + let second_chars = flowstate_document::paragraph_text(projection, first_ix + 1) + .chars() + .count(); + let objects_between = second_block_ix + .saturating_sub(first_block_ix) + .saturating_sub(1); + let merged_end = sentinel + 1 + first_chars + objects_between + second_chars; + let Some(merged) = body_input_paragraph_at(doc, sentinel, merged_end, paragraph_style_at(projection, *first_ix)) else { + return rebuild("join-readback-missing"); + }; + let old_len = projection + .paragraphs + .get(*first_ix) + .map(paragraph_text_len) + .unwrap_or(0); + let merged_len = input_paragraph_text_len(&merged); + let second_block = projection.ids.block_ids.get(second_block_ix).copied(); + let mut patches = vec![ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[first_block_ix], + paragraph_id: *first, + row_hint: first_block_ix, + new: merged, + delta_utf8: projection_text_delta(old_len, 0, merged_len.saturating_sub(old_len), 0), + }]; + if let Some(second_block) = second_block { + patches.push(ProjectionPatch::DeleteBlocks { + block_ids: vec![second_block], + row_hint: second_block_ix, + }); + } + let _ = second; + Some((patches, body_invalidation(0, 0))) + }, + ResolvedPlan::SetMarks { start, end, .. } => { + // One ParagraphRuns patch per touched paragraph — O(changed range). + // Marks never change text, so paragraph starts chain forward from the + // resolved start position: next start = start + chars + boundary + + // object placeholders between the consecutive paragraph blocks. + let Some((mut paragraph_start, _)) = resolved_paragraph_span(projection, start) else { + return rebuild("set-marks-position-misaligned"); + }; + // Rows for every paragraph in one O(blocks) pass — the loop below needs + // both `row` and the following paragraph's row, and calling + // `block_ix_for_paragraph` twice per iteration is the §perf-heaven T2 + // quadratic on object docs. + let rows = flowstate_document::paragraph_block_rows(projection); + let mut patches = Vec::new(); + for paragraph_ix in start.paragraph_ix..=end.paragraph_ix { + let Some(&row) = rows.get(paragraph_ix) else { + return rebuild("set-marks-block-missing"); + }; + let chars = flowstate_document::paragraph_text(projection, paragraph_ix) + .chars() + .count(); + let Some(new) = body_input_paragraph_at( + doc, + paragraph_start.saturating_sub(1), + paragraph_start + chars, + paragraph_style_at(projection, paragraph_ix), + ) else { + return rebuild("set-marks-readback-missing"); + }; + if paragraph_ix < end.paragraph_ix { + let Some(&next_row) = rows.get(paragraph_ix + 1) else { + return rebuild("set-marks-next-block-missing"); + }; + paragraph_start += chars + 1 + next_row.saturating_sub(row).saturating_sub(1); + } + let runs = flowstate_document::document_from_input_blocks(projection.theme.clone(), vec![InputBlock::Paragraph(new)]) + .paragraphs + .first() + .map(|paragraph| paragraph.runs.clone()); + let Some(runs) = runs else { + return rebuild("set-marks-runs-missing"); + }; + patches.push(ProjectionPatch::ParagraphRuns { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[paragraph_ix], + row_hint: row, + runs, + }); + } + Some(( + patches, + body_invalidation(start.body_unicode, end.body_unicode.saturating_sub(start.body_unicode)), + )) + }, + ResolvedPlan::SetParagraphStyle { + paragraph, + paragraph_ix, + style, + } => { + let Some(row) = flowstate_document::block_ix_for_paragraph(projection, *paragraph_ix) else { + return rebuild("set-paragraph-style-block-missing"); + }; + Some(( + vec![ProjectionPatch::ParagraphStyle { + block_id: projection.ids.block_ids[row], + paragraph_id: *paragraph, + row_hint: row, + style: *style, + }], + body_invalidation(0, 0), + )) + }, + ResolvedPlan::SetParagraphStyles { targets, style } => { + // One exact ParagraphStyle patch per target; style marks never change + // text, so no shift bookkeeping. Rows come from one O(doc) pass, not an + // O(doc) scan per target. + let rows = flowstate_document::paragraph_block_rows(projection); + let mut patches = Vec::with_capacity(targets.len()); + for (paragraph, paragraph_ix, _) in targets { + let Some(&row) = rows.get(*paragraph_ix) else { + return rebuild("set-paragraph-styles-block-missing"); + }; + patches.push(ProjectionPatch::ParagraphStyle { + block_id: projection.ids.block_ids[row], + paragraph_id: *paragraph, + row_hint: row, + style: *style, + }); + } + Some((patches, body_invalidation(0, 0))) + }, + ResolvedPlan::InsertObject { + at, + block_ix, + new_block, + block, + .. + } => { + // Whitelist: the exact patch is only valid for the ONE shape where no + // identity re-derives — a placeholder landing AFTER a non-empty + // paragraph's text (byte == text_len > 0) whose next block is another + // PARAGRAPH (a real `\n` boundary right behind the insertion). Every + // other shape re-derives identities under the materializer law: byte-0 + // and mid-text inserts create boundary-less interstitial rows, inserts + // into empty paragraphs coalesce the row into the object, end-of-doc + // inserts grow a trailing fabricated row, and object-cluster inserts + // shift interstitial anchors. Loud rebuild for all of those (rare, + // explicit UI ops; found by the object-fuzz undo arm). + let text_len = projection + .paragraphs + .get(at.paragraph_ix) + .map(paragraph_text_len) + .unwrap_or(0); + let followed_by_paragraph = flowstate_document::block_ix_for_paragraph(projection, at.paragraph_ix) + .and_then(|row| projection.blocks.get(row + 1)) + .is_some_and(|next| matches!(next, flowstate_document::Block::Paragraph(_))); + if text_len == 0 || at.byte < text_len || !followed_by_paragraph { + return rebuild("insert-object-boundary-adjacent"); + } + Some(( + vec![ProjectionPatch::InsertBlocks { + before: projection.ids.block_ids.get(*block_ix).copied(), + row_hint: (*block_ix).min(projection.blocks.len()), + blocks: vec![ProjectionStructuralBlock { + block_id: *new_block, + paragraph_id: None, + block: block.clone(), + }], + }], + ProjectionInvalidation::body_object(frontier_before.clone(), frontier_after.clone(), at.body_unicode, block_kind(block)), + )) + }, + ResolvedPlan::ReplaceObject { block_ix, after, .. } => { + object_replacement_patch(projection, *block_ix, after.clone()).map(|patches| (patches, body_invalidation(0, 0))) + }, + ResolvedPlan::DeleteBlocks { .. } => { + // Removing a block row changes the boundary geometry around it: a + // paragraph that was interstitial (boundary-less, after an object) + // re-attaches, and a coalesced empty resurrects with its durable + // record's identity — re-derivations only the materializer law can + // produce. Loud rebuild (block deletes are rare, explicit UI ops; + // found by the object-fuzz undo arm as maintained-vs-canonical id + // divergence when deleting a boundary-adjacent object). + return rebuild("delete-blocks-boundary-sensitive"); + }, + ResolvedPlan::MoveBlock { .. } => { + // A move is a boundary-geometry change at BOTH endpoints (see the + // DeleteBlocks rationale) — identities around the source and the + // destination re-derive under the materializer law. Loud rebuild. + return rebuild("move-block-boundary-sensitive"); + }, + ResolvedPlan::InsertRichFragment { .. } => { + // The one op class where a full rebuild is the documented contract + // (compound multi-container splice). Loud + counted by the caller. + return rebuild("insert-rich-fragment"); + }, + ResolvedPlan::ReplaceEquationSourceRange { equation, range, text } => { + let Some(block_ix) = core.projection_index_ref().block_index(*equation) else { + return rebuild("equation-block-missing"); + }; + let Some(InputBlock::Equation(mut equation_input)) = projection.blocks.get(block_ix).map(input_block_from_block) else { + return rebuild("equation-input-missing"); + }; + if range.start > range.end || range.end > equation_input.source.len() { + return rebuild("equation-range-invalid"); + } + equation_input.source.replace_range(range.clone(), text); + object_replacement_patch(projection, block_ix, InputBlock::Equation(equation_input)).map(|patches| (patches, body_invalidation(0, 0))) + }, + ResolvedPlan::ReplaceImageAltText { image, text } => { + image_patch(core, *image, |input| input.alt_text = text.clone()).map(|patches| (patches, body_invalidation(0, 0))) + }, + ResolvedPlan::ReplaceImageCaption { image, caption } => { + image_patch(core, *image, |input| input.caption = caption.clone()).map(|patches| (patches, body_invalidation(0, 0))) + }, + ResolvedPlan::SetImageLayout { image, sizing, alignment } => image_patch(core, *image, |input| { + input.sizing = sizing.clone(); + input.alignment = *alignment; + }) + .map(|patches| (patches, body_invalidation(0, 0))), + ResolvedPlan::Table { table, table_ix, op } => table_patch(core, *table, *table_ix, op).map(|patches| (patches, body_invalidation(0, 0))), + ResolvedPlan::ReplaceMatches { matches, replacement } => { + // Matches are same-paragraph and sorted descending (resolution + // contract); group per paragraph and read each affected paragraph back + // ONCE through the ranged readback — O(matches + affected paragraphs), + // never O(doc), regardless of how many matches the storm carries. + // Groups are processed in ASCENDING position order with a running net + // shift: resolved positions are PRE-commit, but the readback runs + // POST-commit, where every paragraph after an edited one has moved by + // the net length delta of the edits before it. + let replacement_chars = replacement.chars().count(); + let mut groups: Vec<&[(ResolvedTextPosition, ResolvedTextPosition, Option)]> = Vec::new(); + let mut ix = 0; + while ix < matches.len() { + let paragraph_ix = matches[ix].0.paragraph_ix; + let group_len = matches[ix..] + .iter() + .take_while(|(start, ..)| start.paragraph_ix == paragraph_ix) + .count(); + groups.push(&matches[ix..ix + group_len]); + ix += group_len; + } + + // Rows for every paragraph in one O(blocks) pass instead of an O(blocks) + // scan per matched group (§perf-heaven T2 quadratic on object docs). + let rows = flowstate_document::paragraph_block_rows(projection); + let mut patches = Vec::new(); + let mut invalid_lo = usize::MAX; + let mut invalid_hi = 0usize; + let mut shift = 0isize; + for group in groups.into_iter().rev() { + let paragraph_ix = group[0].0.paragraph_ix; + let Some(&row) = rows.get(paragraph_ix) else { + return rebuild("replace-matches-block-missing"); + }; + let Some((paragraph_start, old_chars)) = resolved_paragraph_span(projection, &group[0].0) else { + return rebuild("replace-matches-position-misaligned"); + }; + let removed: usize = group + .iter() + .map(|(start, end, _)| end.body_unicode - start.body_unicode) + .sum(); + let added = replacement_chars * group.len(); + let Some(new_chars) = (old_chars + added).checked_sub(removed) else { + return rebuild("replace-matches-length-misaligned"); + }; + let Some(post_start) = paragraph_start.checked_add_signed(shift) else { + return rebuild("replace-matches-shift-misaligned"); + }; + let Some(new) = body_input_paragraph_at( + doc, + post_start.saturating_sub(1), + post_start + new_chars, + paragraph_style_at(projection, paragraph_ix), + ) else { + return rebuild("replace-matches-readback-missing"); + }; + shift += added as isize - removed as isize; + let old_text = flowstate_document::paragraph_text(projection, paragraph_ix); + let new_text: String = new.runs.iter().map(|run| run.text.as_str()).collect(); + let delta_utf8 = text_delta_between(&old_text, &new_text); + patches.push(ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids[row], + paragraph_id: projection.ids.paragraph_ids[paragraph_ix], + row_hint: row, + new, + delta_utf8, + }); + // Descending within the group: last entry has the lowest start. + invalid_lo = invalid_lo.min(group[group.len() - 1].0.body_unicode); + invalid_hi = invalid_hi.max(group[0].1.body_unicode); + } + Some((patches, body_invalidation(invalid_lo, invalid_hi.saturating_sub(invalid_lo)))) + }, + }; + + match patches { + Some((patches, invalidation)) => PatchPlan::Patches { patches, invalidation }, + None => rebuild(intent_rebuild_reason(intent)), + } +} + +/// UTF-8 byte length of an input paragraph's text. +/// Live-space unicode start and pre-mutation char length of the paragraph a +/// resolved position sits in. Exact for post-mutation reads because text +/// edits at/after the resolved point never move the paragraph's start. +fn resolved_paragraph_span(projection: &DocumentProjection, at: &ResolvedTextPosition) -> Option<(usize, usize)> { + let text = flowstate_document::paragraph_text(projection, at.paragraph_ix); + let within = text.get(..at.byte)?.chars().count(); + let start = at.body_unicode.checked_sub(within)?; + Some((start, text.chars().count())) +} + +fn paragraph_style_at(projection: &DocumentProjection, paragraph_ix: usize) -> ParagraphStyle { + projection + .paragraphs + .get(paragraph_ix) + .map(|paragraph| paragraph.style) + .unwrap_or(ParagraphStyle::Normal) +} + +fn input_paragraph_text_len(paragraph: &flowstate_document::InputParagraph) -> usize { + paragraph.runs.iter().map(|run| run.text.len()).sum() +} + +fn intent_rebuild_reason(intent: &LocalIntent) -> &'static str { + match intent { + LocalIntent::InsertRichFragment(_) => "insert-rich-fragment", + _ => "patch-synthesis-bailout", + } +} + +fn block_kind(block: &InputBlock) -> &'static str { + match block { + InputBlock::Image(_) => "image", + InputBlock::Equation(_) => "equation", + InputBlock::Table(_) => "table", + InputBlock::Paragraph(_) => "paragraph", + } +} + +fn image_patch( + core: &CrdtRuntime, + image: flowstate_document::BlockId, + mutate: impl FnOnce(&mut flowstate_document::InputImageBlock), +) -> Option> { + let projection = core.projection_ref(); + let block_ix = core.projection_index_ref().block_index(image)?; + let InputBlock::Image(mut image_input) = projection + .blocks + .get(block_ix) + .map(input_block_from_block)? + else { + return None; + }; + mutate(&mut image_input); + object_replacement_patch(projection, block_ix, InputBlock::Image(image_input)) +} + +fn table_patch(core: &CrdtRuntime, table: flowstate_document::BlockId, table_ix: usize, op: &ResolvedTableOp) -> Option> { + // §6-R.1: READ the committed table back from canonical state (the same + // one-table materialization law the full rebuild applies) instead of + // simulating the op on the old projected table. Simulation was a second + // doc→projection semantics and diverged under undo-churned histories (the + // table-fuzz undo arm caught stale cell spans). Any defect or a missing + // canonical record falls back to the loud full rebuild (`None`). + let _ = op; + let projection = core.projection_ref(); + let block_ix = core.projection_index_ref().block_index(table)?; + debug_assert_eq!(block_ix, table_ix); + let (table_input, defects) = flowstate_document::materialize_table_block(core.doc(), table.0).ok()?; + if !defects.is_empty() { + return None; + } + object_replacement_patch(projection, block_ix, InputBlock::Table(table_input)) +} diff --git a/crates/flowstate-collab/src/local_write/recorded_inverse.rs b/crates/flowstate-collab/src/local_write/recorded_inverse.rs new file mode 100644 index 00000000..4e27a1e9 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/recorded_inverse.rs @@ -0,0 +1,1700 @@ +//! §act-four M1 (generalizes act-three B.1): the recorded-inverse undo/redo +//! fast path, for every whole-document op class. +//! +//! A qualifying mass op commit records everything needed to invert itself as an +//! ORDINARY forward transaction — a [`RecordedDelta`] for each direction +//! (`undo` / `redo`), holding the doc-side [`RecordedMutation`] plus the +//! intent-exact projection patches. Undo replays the recorded inverse through +//! the same commit/publish ladder as any local write, skipping the +//! `UndoManager`'s checkout-based diff computation entirely; the vendored +//! `external_step_begin`/`external_step_finish` APIs keep the undo/redo stacks +//! bookkeeping-identical to a native `UndoManager::undo`. Redo replays the +//! forward delta; the slot ping-pongs between directions for free after the one +//! capture. +//! +//! The `RecordedDelta` here is the same structure the act-four version graph +//! (M5) records per commit — built flat now, carried into the tree unchanged. +//! +//! Covered op classes (the whole-document ops): +//! - `DeleteRange` (cross-paragraph mass delete) — undo restores the verbatim +//! `slice_delta` + object containers + retired paragraph records (original +//! ids); redo re-deletes. Reprojects via exact pre-recorded patches. +//! - `SetParagraphStyles` (select-all restyle) — each direction re-marks the +//! target boundaries with the corresponding style; position-stable (marks +//! never shift text). Reprojects via exact pre-recorded patches. +//! - `ReplaceMatches` (mass replace-all) — undo restores each match's original +//! rich content (captured pre-op), redo re-applies the replacements (captured +//! post-op); doc-side rich splices with post-position bookkeeping; both +//! directions reproject through the regional derive ladder. +//! +//! Fidelity contract (100% bar): the fast path runs ONLY when the doc frontier +//! still equals the capture frontier (ANY interleaving commit/import/repair/ +//! checkout fails this), the top stack span matches exactly (a group merge +//! changes it → declined), and the stack tracks no interleaved remote diff. +//! Anything else falls through to the checkout-based slow path unchanged. +//! Kill switch: `FLOWSTATE_DISABLE_FAST_UNDO=1`. + +use std::sync::Arc; + +use anyhow::{Context as _, Result}; +use flowstate_document::{ + Block, BlockId, InputBlock, MARK_PARAGRAPH_STYLE, ParagraphId, ProjectionPatch, ProjectionStructuralBlock, block_ix_for_paragraph, + input_block_from_block, input_paragraph_from_document_range, loro_schema::body_text, paragraph_text, paragraph_text_len, +}; +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use loro::{CounterSpan, TextDelta, UndoOrRedo, cursor::PosType}; + +use super::commit::ResolvedPlan; +use super::resolve::ResolvedTextPosition; +use crate::crdt_runtime::{ + CrdtRuntime, ProjectionInvalidation, RuntimeEvent, delete_projection_paragraph_metadata, paragraph_style_value, projection_text_delta, + prune_orphaned_body_object_blocks, repair_paragraph_metadata_after_stable_split, restore_input_object_block_containers, + sentinel_protected_delete_range, +}; + +/// Minimum deleted-range size (unicode chars) worth a `DeleteRange` capture. +const MIN_CAPTURE_CHARS: usize = 2048; + +/// Minimum target count worth a `SetParagraphStyles` capture — a MASS restyle +/// (the select-all class). A handful of paragraphs is already cheap via the +/// checkout path and the capture isn't free. +const MIN_RESTYLE_TARGETS: usize = 8; + +/// Kill switch: `FLOWSTATE_DISABLE_FAST_UNDO=1` reverts every undo/redo to the +/// checkout-based slow path. +fn fast_undo_disabled() -> bool { + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *DISABLED.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_FAST_UNDO").is_some()) +} + +/// A restored paragraph boundary: the absolute body-unicode index of its +/// leading `\n` plus the ORIGINAL durable identities of the paragraph that +/// follows it. +struct RestoredBoundary { + unicode: usize, + paragraph_id: ParagraphId, + block_id: BlockId, +} + +/// A restored object block: the absolute body-unicode index of its U+FFFC +/// placeholder plus the ORIGINAL block id and captured content. +struct RestoredObject { + unicode: usize, + block_id: BlockId, + input: InputBlock, +} + +/// The doc-side replay for one direction of one op — the mutation applied to +/// Loro before commit. +enum RecordedMutation { + /// Restore a previously-deleted body range verbatim (undo of a delete): + /// bulk text+marks delta, object registry containers, retired paragraph + /// records with original ids. + RestoreRange { + clamped_start: usize, + restore_delta: Vec, + boundaries: Vec, + objects: Vec, + }, + /// Delete a body range (redo of a delete): the same mutation sequence as the + /// original `DeleteRange` execution, driven from recorded data. + DeleteRange { + clamped_start: usize, + clamped_len: usize, + prune_objects: bool, + boundaries: Vec, + }, + /// Re-mark paragraph boundaries with the given style values (restyle in + /// either direction). `(boundary_unicode, style_value)`. Position-stable. + MarkParagraphStyles { targets: Vec<(usize, i64)> }, + /// Splice a set of ranges, each replacing `[start, start+delete_len)` with a + /// captured rich delta (text+marks). Processed in DESCENDING start order so + /// no earlier splice shifts a later one. Covers replace-all in either + /// direction (restore originals / re-apply replacements). + SpliceRichRanges { splices: Vec }, + /// §act-twelve A12.2.2: (re)create a paragraph boundary — insert the `\n`, + /// mark its paragraph style, strip absorbed run-style keys from the + /// sentinel, and (re)write the paragraph's durable records with the + /// ORIGINAL ids. The exact `SplitParagraph` executor sequence; also the + /// exact inverse of a `JoinParagraphs`. + CreateBoundary { + at: usize, + style_value: i64, + paragraph_id: ParagraphId, + block_id: BlockId, + }, + /// §act-twelve A12.2.2: re-apply captured run styles per span via the + /// `mark_run_styles` law (clear the four run keys, re-mark the set ones). + /// Mark-only — never touches text identity, so replay cannot interleave + /// with concurrent remote text. Spans are absolute body-unicode. + MarkRunRanges { + spans: Vec<(usize, usize, flowstate_document::RunStyles)>, + }, +} + +/// One rich splice: replace `[start, start+delete_len)` with `insert` (a +/// `slice_delta`-shaped rich content run — text + marks). +struct RichSplice { + start: usize, + delete_len: usize, + insert: Vec, +} + +/// How a direction reprojects after its doc-side mutation. +enum Reproject { + /// Apply these exact intent-synthesized patches to the maintained + /// projection — `O(change)`, no readback. §perf-heaven T8.19: `Arc<[…]>` so the + /// hot undo/redo shares the (possibly mass-op-sized) set with the event. + Patches(Arc<[ProjectionPatch]>), + /// Run the standard post-commit derive ladder (regional / ranged readback + /// off the committed doc). For classes whose exact patches are complex to + /// pre-record; still `O(change)` (regional), just via the proven remote + /// derive path. + Derive, +} + +/// A recorded, replayable mutation for ONE direction (forward or inverse) of a +/// committed op: the doc-side mutation plus how it reprojects and the body +/// invalidation extent for the publish ladder. +struct RecordedDelta { + mutation: RecordedMutation, + reproject: Reproject, + /// §act-ten A10.2: one `(unicode_start, unicode_len)` per touched span. A + /// single covering span made a whole-doc replace-all's undo rematerialize + /// EVERY paragraph between the first and last match; per-span ranges keep the + /// derive ladder proportional to the actual change set. + invalidation_ranges: Vec<(usize, usize)>, +} + +/// Everything needed to replay one committed op in either direction. +pub(crate) struct RecordedInverse { + /// The direction the next fast step serves (flips after every fast step). + direction: UndoOrRedo, + /// Doc frontier that must match EXACTLY at step time (encoded). + expected_frontier: Vec, + /// The top stack item's counter span that must match exactly. + expected_span: CounterSpan, + /// §mass-op collab (oom-leads #9): remote diffs are pending in the Loro undo + /// stacks' buffers, but this entry was REBASED through the import's net + /// delta ([`rebase_recorded_inverse_through_remote`]) — coordinates shifted + /// (content entries: proven hull-disjoint; mark entries: per-coordinate). + /// The fast step may proceed despite `peek_top_span`'s conservative dirty + /// flag: the replay content is coordinate-correct, and the buffered diffs + /// stay valid for any later slow-path pop of deeper (uncaptured) items — + /// kept replays are position-neutral at every buffered position, and vendor + /// patch #22 transforms the buffers through every external-step commit + /// besides (the native `undo_internal` mirror). + survives_pending_remote: bool, + undo: RecordedDelta, + redo: RecordedDelta, +} + +impl std::fmt::Debug for RecordedInverse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let reproject_kind = |delta: &RecordedDelta| match &delta.reproject { + Reproject::Patches(patches) => format!("patches({})", patches.len()), + Reproject::Derive => "derive".to_string(), + }; + f.debug_struct("RecordedInverse") + .field("direction", &self.direction) + .field("expected_span", &self.expected_span) + .field("undo", &reproject_kind(&self.undo)) + .field("redo", &reproject_kind(&self.redo)) + .finish_non_exhaustive() + } +} + +/// Pre-mutation capture: taken between resolve and execute, while the doc and +/// projection are still at their pre-op state. The `undo` delta is fully formed +/// here (it needs pre-op reads); the `redo` mutation is formed here too, but its +/// patches are supplied at [`finalize_capture`] (the op's own synthesized +/// patches are the redo patches). +pub(crate) struct PendingInverseCapture { + undo: RecordedDelta, + redo_mutation: RecordedMutation, + redo_invalidation_ranges: Vec<(usize, usize)>, + /// Redo reprojects via the derive ladder (rather than the op's synthesized + /// patches) — set for classes whose exact patches are complex to pre-record. + redo_derive: bool, + /// For replace-all: the redo splices can only be captured AFTER the op runs + /// (the replacement's applied rich content lives in the post-op doc). Each + /// entry says: redo replaces `[redo_start, redo_start+redo_delete_len)` with + /// the rich content sliced from the post-op doc at `[post_slice_start, + /// post_slice_start+post_slice_len)`. `None` for all other classes. + redo_replace_finalize: Option>, +} + +/// Deferred redo-splice capture for replace-all (see `redo_replace_finalize`). +struct ReplaceFinalize { + redo_start: usize, + redo_delete_len: usize, + post_slice_start: usize, + post_slice_len: usize, +} + +/// Capture the recorded inverse of a qualifying mass op plan. Must run BEFORE +/// the op mutates the doc. Returns `None` (never an error) for anything that +/// doesn't qualify — the op itself proceeds identically either way. +pub(crate) fn capture_before_execute(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + if fast_undo_disabled() { + return None; + } + match plan { + ResolvedPlan::InsertText { .. } => capture_insert_text(plan), + ResolvedPlan::DeleteRange { .. } => capture_delete_range(core, plan), + ResolvedPlan::SetParagraphStyles { .. } => capture_set_paragraph_styles(core, plan), + ResolvedPlan::ReplaceMatches { .. } => capture_replace_matches(core, plan), + // §act-twelve A12.2.2 (capture-coverage completion): the everyday + // structural/mark ops. An uncaptured op both takes the O(history) + // checkout slow path on its own undo AND clears every deeper captured + // fast path behind it — Enter, backspace-join, bold/italic and + // single-paragraph restyle were doing exactly that to typing sessions. + ResolvedPlan::SplitParagraph { .. } => capture_split_paragraph(plan), + ResolvedPlan::JoinParagraphs { .. } => capture_join_paragraphs(core, plan), + ResolvedPlan::SetMarks { .. } => capture_set_marks(core, plan), + ResolvedPlan::SetParagraphStyle { .. } => capture_set_paragraph_style(core, plan), + _ => None, + } +} + +/// §oom-leads #9 (keystroke undo): EVERY text insert records its inverse — the +/// keystroke is the most common op there is, and an uncaptured op both takes +/// the O(history) checkout slow path on its own undo AND clears every deeper +/// captured fast path behind it (a single typed char used to kill a recorded +/// mass-op undo). The record is O(insert) and needs no doc reads at capture: +/// undo = delete the inserted span (derive reprojection — a ranged readback of +/// one paragraph), redo = re-insert the post-op rich slice (captured at +/// finalize, so a `style_override`'s marks ride along verbatim; the op's own +/// synthesized patches replay the projection). +fn capture_insert_text(plan: &ResolvedPlan) -> Option { + let ResolvedPlan::InsertText { at, text, .. } = plan else { + return None; + }; + let len = text.chars().count(); + if len == 0 { + return None; + } + let pos = at.body_unicode; + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::DeleteRange { + clamped_start: pos, + clamped_len: len, + prune_objects: false, + boundaries: Vec::new(), + }, + reproject: Reproject::Derive, + invalidation_ranges: vec![(pos, len)], + }, + // Filled at finalize from the post-op doc (see `redo_replace_finalize`). + redo_mutation: RecordedMutation::SpliceRichRanges { splices: Vec::new() }, + redo_invalidation_ranges: vec![(pos, len)], + redo_derive: false, + redo_replace_finalize: Some(vec![ReplaceFinalize { + redo_start: pos, + redo_delete_len: 0, + post_slice_start: pos, + post_slice_len: len, + }]), + }) +} + +/// §oom-leads #9 (keystroke undo): backspace and small intra-paragraph deletes. +/// A single paragraph's interior can contain neither `\n` nor U+FFFC, so there +/// are no boundaries or objects to retire — the record is a verbatim rich slice +/// (undo re-inserts it) plus a re-delete, both O(len). +fn capture_intra_paragraph_delete( + core: &CrdtRuntime, + start: &ResolvedTextPosition, + end: &ResolvedTextPosition, +) -> Option { + let len = end.body_unicode.saturating_sub(start.body_unicode); + if len == 0 { + return None; + } + let (clamped_start, clamped_len) = sentinel_protected_delete_range(start.body_unicode, len)?; + if (clamped_start, clamped_len) != (start.body_unicode, len) { + return None; + } + let body = body_text(core.doc()); + let restore_delta = match body.slice_delta(clamped_start, clamped_start + clamped_len, PosType::Unicode) { + Ok(delta) => delta, + Err(error) => { + tracing::warn!(%error, "recorded-inverse intra-paragraph capture declined: slice_delta failed"); + return None; + }, + }; + // Defensive: the slice must be pure interior text. A boundary or placeholder + // here means the resolved positions and the projection disagree — decline + // rather than record a structural restore without its metadata. + for segment in &restore_delta { + let TextDelta::Insert { insert, .. } = segment else { + return None; + }; + if insert.contains('\n') || insert.contains(flowstate_document::OBJECT_REPLACEMENT) { + return None; + } + } + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::RestoreRange { + clamped_start, + restore_delta, + boundaries: Vec::new(), + objects: Vec::new(), + }, + reproject: Reproject::Derive, + invalidation_ranges: vec![(clamped_start, clamped_len)], + }, + redo_mutation: RecordedMutation::DeleteRange { + clamped_start, + clamped_len, + prune_objects: false, + boundaries: Vec::new(), + }, + redo_invalidation_ranges: vec![(clamped_start, clamped_len)], + redo_derive: false, + redo_replace_finalize: None, + }) +} + +/// Minimum match count worth a `ReplaceMatches` capture — a MASS replace-all. +const MIN_REPLACE_MATCHES: usize = 8; + +/// Replace-all: undo restores each match's original rich content; redo +/// re-applies the replacement. The doc-side mutation is a set of rich splices; +/// both directions reproject via the derive ladder (their exact patches are the +/// per-paragraph ranged readback the ladder already produces). Positions: +/// matches arrive DESCENDING by start; the op replaces `[s_i, s_i+orig_len_i)` +/// with the replacement at the original `s_i` (descending keeps `s_i` stable). +/// Undo operates on the post-op doc, where match `i`'s replacement sits at +/// `q_i = s_i + Σ_{s_j < s_i}(repl_len − orig_len_j)`. +fn capture_replace_matches(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + let ResolvedPlan::ReplaceMatches { matches, replacement } = plan else { + return None; + }; + if matches.len() < MIN_REPLACE_MATCHES { + return None; + } + let repl_len = replacement.chars().count(); + let body = body_text(core.doc()); + + // Post-position q for each original start, via an ascending cumulative shift. + let mut q_by_start: std::collections::HashMap = std::collections::HashMap::with_capacity(matches.len()); + let mut ascending: Vec<(usize, usize)> = matches + .iter() + .map(|(start, end, _)| (start.body_unicode, end.body_unicode - start.body_unicode)) + .collect(); + ascending.sort_by_key(|(start, _)| *start); + let mut shift: isize = 0; + for (start, orig_len) in &ascending { + let q = usize::try_from(*start as isize + shift).ok()?; + if q_by_start.insert(*start, q).is_some() { + return None; // duplicate start — should be pruned already; decline. + } + shift += repl_len as isize - *orig_len as isize; + } + + let mut undo_splices = Vec::with_capacity(matches.len()); + let mut redo_finalize = Vec::with_capacity(matches.len()); + let mut undo_ranges = Vec::with_capacity(matches.len()); + let mut redo_ranges = Vec::with_capacity(matches.len()); + for (start, end, _styles) in matches { + let s = start.body_unicode; + let orig_len = end.body_unicode - s; + let q = *q_by_start.get(&s)?; + // Original rich content (text + marks), captured pre-op — the undo insert. + let orig_delta = body.slice_delta(s, s + orig_len, PosType::Unicode).ok()?; + undo_splices.push(RichSplice { + start: q, + delete_len: repl_len, + insert: orig_delta, + }); + redo_finalize.push(ReplaceFinalize { + redo_start: s, + redo_delete_len: orig_len, + post_slice_start: q, + post_slice_len: repl_len, + }); + // §act-ten A10.2: one invalidation range PER MATCH — the readback then + // touches only the matched paragraphs, not the whole covered span. + undo_ranges.push((q, repl_len.max(1))); + redo_ranges.push((s, orig_len.max(1))); + } + undo_ranges.sort_unstable(); + redo_ranges.sort_unstable(); + + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::SpliceRichRanges { splices: undo_splices }, + reproject: Reproject::Derive, + invalidation_ranges: undo_ranges, + }, + // Filled at finalize from the post-op doc (see `redo_replace_finalize`). + redo_mutation: RecordedMutation::SpliceRichRanges { splices: Vec::new() }, + redo_invalidation_ranges: redo_ranges, + redo_derive: true, + redo_replace_finalize: Some(redo_finalize), + }) +} + +/// `SetParagraphStyles` (select-all restyle): each direction re-marks the target +/// boundaries. Undo → prior styles; redo → new style. Position-stable. +fn capture_set_paragraph_styles(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + let ResolvedPlan::SetParagraphStyles { targets, style } = plan else { + return None; + }; + if targets.len() < MIN_RESTYLE_TARGETS { + return None; + } + let projection = core.projection_ref(); + let after_value = paragraph_style_value(*style); + let rows = flowstate_document::paragraph_block_rows(projection); + + let mut undo_marks: Vec<(usize, i64)> = Vec::with_capacity(targets.len()); + let mut redo_marks: Vec<(usize, i64)> = Vec::with_capacity(targets.len()); + let mut undo_patches: Vec = Vec::with_capacity(targets.len()); + let mut boundary_ranges: Vec<(usize, usize)> = Vec::with_capacity(targets.len()); + for (paragraph, paragraph_ix, boundary) in targets { + let before_style = projection + .paragraphs + .get(*paragraph_ix) + .map(|paragraph| paragraph.style)?; + let before_value = paragraph_style_value(before_style); + undo_marks.push((*boundary, before_value)); + redo_marks.push((*boundary, after_value)); + let &row = rows.get(*paragraph_ix)?; + undo_patches.push(ProjectionPatch::ParagraphStyle { + block_id: *projection.ids.block_ids.get(row)?, + paragraph_id: *paragraph, + row_hint: row, + style: before_style, + }); + boundary_ranges.push((*boundary, 1)); + } + boundary_ranges.sort_unstable(); + + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::MarkParagraphStyles { targets: undo_marks }, + reproject: Reproject::Patches(undo_patches.into()), + invalidation_ranges: boundary_ranges.clone(), + }, + redo_mutation: RecordedMutation::MarkParagraphStyles { targets: redo_marks }, + redo_invalidation_ranges: boundary_ranges, + redo_derive: false, + redo_replace_finalize: None, + }) +} + +/// §act-twelve A12.2.2: `SplitParagraph` (Enter). The plan carries the +/// PRE-MINTED new paragraph/block ids, so both directions are fully formable +/// before execution: undo deletes the boundary and retires the new records +/// (the exact join sequence); redo re-creates the boundary with the SAME ids +/// (identity preservation — collab cursors and records survive the cycle). +fn capture_split_paragraph(plan: &ResolvedPlan) -> Option { + let ResolvedPlan::SplitParagraph { + at, + inherited_style, + new_paragraph, + new_block, + } = plan + else { + return None; + }; + let pos = at.body_unicode; + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::DeleteRange { + clamped_start: pos, + clamped_len: 1, + prune_objects: false, + boundaries: vec![RestoredBoundary { + unicode: pos, + paragraph_id: *new_paragraph, + block_id: *new_block, + }], + }, + reproject: Reproject::Derive, + invalidation_ranges: vec![(pos, 1)], + }, + redo_mutation: RecordedMutation::CreateBoundary { + at: pos, + style_value: paragraph_style_value(*inherited_style), + paragraph_id: *new_paragraph, + block_id: *new_block, + }, + redo_invalidation_ranges: vec![(pos, 1)], + redo_derive: true, + redo_replace_finalize: None, + }) +} + +/// §act-twelve A12.2.2: `JoinParagraphs` (Backspace at paragraph start). Undo +/// re-creates the boundary with the SECOND paragraph's prior style and its +/// ORIGINAL ids; redo re-deletes it. Mirrors `join_projection_paragraphs`' +/// own guards so a capture never outlives an executor no-op (the executor +/// bails on those shapes, which discards the pending capture anyway). +fn capture_join_paragraphs(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + let ResolvedPlan::JoinParagraphs { second, first_ix, .. } = plan else { + return None; + }; + let projection = core.projection_ref(); + let second_ix = first_ix + 1; + if projection.ids.paragraph_ids.get(second_ix) != Some(second) { + return None; + } + let second_style = projection.paragraphs.get(second_ix)?.style; + let second_block_ix = block_ix_for_paragraph(projection, second_ix)?; + let second_block = projection.ids.block_ids.get(second_block_ix).copied()?; + let boundary = crate::crdt_runtime::paragraph_boundary_loro_unicode_index(core.doc(), projection, second_ix); + if body_text(core.doc()).char_at(boundary) != Ok('\n') { + return None; + } + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::CreateBoundary { + at: boundary, + style_value: paragraph_style_value(second_style), + paragraph_id: *second, + block_id: second_block, + }, + reproject: Reproject::Derive, + invalidation_ranges: vec![(boundary, 1)], + }, + redo_mutation: RecordedMutation::DeleteRange { + clamped_start: boundary, + clamped_len: 1, + prune_objects: false, + boundaries: vec![RestoredBoundary { + unicode: boundary, + paragraph_id: *second, + block_id: second_block, + }], + }, + redo_invalidation_ranges: vec![(boundary, 1)], + redo_derive: true, + redo_replace_finalize: None, + }) +} + +/// §act-twelve A12.2.2: `SetMarks` (bold/italic/highlight over a range). The +/// executor's law REPLACES the four run-style keys over the range +/// (`mark_run_styles`: unmark all four, re-mark the set ones), so undo = +/// re-apply the captured PRIOR per-span styles through the same law, redo = +/// one span with the op's styles. Truth source is `slice_delta` (doc-level, +/// includes boundary chars' actual run keys); non-run attrs are ignored by +/// the mapper, so paragraph-style marks are never touched. +fn capture_set_marks(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + let ResolvedPlan::SetMarks { start, end, styles } = plan else { + return None; + }; + let s = start.body_unicode; + let e = end.body_unicode; + if e <= s { + return None; + } + let body = body_text(core.doc()); + let prior = match body.slice_delta(s, e, PosType::Unicode) { + Ok(delta) => delta, + Err(error) => { + tracing::warn!(%error, "recorded-inverse set-marks capture declined: slice_delta failed"); + return None; + }, + }; + let mut spans: Vec<(usize, usize, flowstate_document::RunStyles)> = Vec::new(); + let mut cursor = s; + for segment in &prior { + let TextDelta::Insert { insert, attributes } = segment else { + return None; + }; + let len = insert.chars().count(); + if len == 0 { + continue; + } + let span_styles = run_styles_from_mark_attrs(attributes.as_ref()); + match spans.last_mut() { + Some((last_start, last_len, last_styles)) if *last_styles == span_styles && *last_start + *last_len == cursor => { + *last_len += len; + }, + _ => spans.push((cursor, len, span_styles)), + } + cursor += len; + } + if spans.is_empty() { + return None; + } + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::MarkRunRanges { spans }, + reproject: Reproject::Derive, + invalidation_ranges: vec![(s, e - s)], + }, + redo_mutation: RecordedMutation::MarkRunRanges { + spans: vec![(s, e - s, *styles)], + }, + redo_invalidation_ranges: vec![(s, e - s)], + redo_derive: true, + redo_replace_finalize: None, + }) +} + +/// The four run-style keys `mark_run_styles` manages, mapped back from Loro +/// mark attributes — the exact inverse of that writer (other keys, e.g. +/// paragraph style or vert-align, are deliberately ignored: `SetMarks` never +/// touches them, so neither may its undo). +fn run_styles_from_mark_attrs( + attrs: Option<&std::collections::HashMap>, +) -> flowstate_document::RunStyles { + use flowstate_document::loro_schema::{MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH}; + let mut styles = flowstate_document::RunStyles::default(); + let Some(attrs) = attrs else { + return styles; + }; + if let Some(loro::LoroValue::I64(slot)) = attrs.get(MARK_RUN_SEMANTIC_STYLE) + && let Ok(slot) = u8::try_from(*slot) + { + styles.semantic = flowstate_document::RunSemanticStyle::Custom(slot); + } + if let Some(loro::LoroValue::I64(slot)) = attrs.get(MARK_HIGHLIGHT_STYLE) + && let Ok(slot) = u8::try_from(*slot) + { + styles.highlight = Some(flowstate_document::HighlightStyle::Custom(slot)); + } + if matches!(attrs.get(MARK_DIRECT_UNDERLINE), Some(loro::LoroValue::Bool(true))) { + styles.direct_underline = true; + } + if matches!(attrs.get(MARK_STRIKETHROUGH), Some(loro::LoroValue::Bool(true))) { + styles.strikethrough = true; + } + styles +} + +/// §act-twelve A12.2.2: single-paragraph `SetParagraphStyle` — the plural +/// capture's law with one target (no minimum gate: the capture is O(1)). +fn capture_set_paragraph_style(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + let ResolvedPlan::SetParagraphStyle { + paragraph, + paragraph_ix, + style, + } = plan + else { + return None; + }; + let projection = core.projection_ref(); + if projection.ids.paragraph_ids.get(*paragraph_ix) != Some(paragraph) { + return None; + } + let before_style = projection.paragraphs.get(*paragraph_ix)?.style; + let boundary = crate::crdt_runtime::paragraph_boundary_loro_unicode_index(core.doc(), projection, *paragraph_ix); + let rows = flowstate_document::paragraph_block_rows(projection); + let &row = rows.get(*paragraph_ix)?; + let undo_patch = ProjectionPatch::ParagraphStyle { + block_id: *projection.ids.block_ids.get(row)?, + paragraph_id: *paragraph, + row_hint: row, + style: before_style, + }; + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::MarkParagraphStyles { + targets: vec![(boundary, paragraph_style_value(before_style))], + }, + reproject: Reproject::Patches(vec![undo_patch].into()), + invalidation_ranges: vec![(boundary, 1)], + }, + redo_mutation: RecordedMutation::MarkParagraphStyles { + targets: vec![(boundary, paragraph_style_value(*style))], + }, + redo_invalidation_ranges: vec![(boundary, 1)], + redo_derive: false, + redo_replace_finalize: None, + }) +} + +/// Cross-paragraph mass `DeleteRange`: undo restores verbatim, redo re-deletes. +fn capture_delete_range(core: &CrdtRuntime, plan: &ResolvedPlan) -> Option { + let ResolvedPlan::DeleteRange { start, end } = plan else { + return None; + }; + let len = end.body_unicode.saturating_sub(start.body_unicode); + // §oom-leads #9: intra-paragraph deletes (backspace, word/selection deletes) + // capture at ANY size — the record is O(len) with no structural bookkeeping. + if start.paragraph_ix == end.paragraph_ix { + return capture_intra_paragraph_delete(core, start, end); + } + if len < MIN_CAPTURE_CHARS { + return None; + } + let (clamped_start, clamped_len) = sentinel_protected_delete_range(start.body_unicode, len)?; + if (clamped_start, clamped_len) != (start.body_unicode, len) { + // Sentinel-clamped deletes shift the whole geometry; rare — slow path. + return None; + } + let projection = core.projection_ref(); + let first_block_ix = block_ix_for_paragraph(projection, start.paragraph_ix)?; + let last_block_ix = block_ix_for_paragraph(projection, end.paragraph_ix)?; + + // The rows the delete removes: everything strictly after the first + // paragraph's row through the last paragraph's row (which merges into the + // first). Captured with their ORIGINAL ids so undo restores identity. + let mut removed_rows = Vec::with_capacity(last_block_ix - first_block_ix); + let mut removed_paragraphs: Vec<(ParagraphId, BlockId)> = Vec::new(); + let mut removed_objects: Vec<(BlockId, InputBlock)> = Vec::new(); + let mut paragraph_ix = start.paragraph_ix; + for block_ix in (first_block_ix + 1)..=last_block_ix { + let block = projection.blocks.get(block_ix)?; + let block_id = projection.ids.block_ids.get(block_ix).copied()?; + match block { + Block::Table(_) => { + // Durable row/column/cell ids cannot be re-minted losslessly here. + return None; + }, + Block::Paragraph(_) => { + paragraph_ix += 1; + let paragraph_id = projection.ids.paragraph_ids.get(paragraph_ix).copied()?; + removed_paragraphs.push((paragraph_id, block_id)); + removed_rows.push(ProjectionStructuralBlock { + block_id, + paragraph_id: Some(paragraph_id), + block: InputBlock::Paragraph(input_paragraph_from_document_range(projection, paragraph_ix, 0..usize::MAX)), + }); + }, + Block::Image(_) | Block::Equation(_) => { + let input = input_block_from_block(block); + removed_objects.push((block_id, input.clone())); + removed_rows.push(ProjectionStructuralBlock { + block_id, + paragraph_id: None, + block: input, + }); + }, + } + } + if paragraph_ix != end.paragraph_ix { + tracing::warn!( + start = start.paragraph_ix, + end = end.paragraph_ix, + walked = paragraph_ix, + "recorded-inverse capture declined: paragraph row walk mismatch" + ); + return None; + } + + // Verbatim capture of the deleted range straight from Loro. + let body = body_text(core.doc()); + let restore_delta = match body.slice_delta(clamped_start, clamped_start + clamped_len, PosType::Unicode) { + Ok(delta) => delta, + Err(error) => { + tracing::warn!(%error, "recorded-inverse capture declined: slice_delta failed"); + return None; + }, + }; + + // Pair each deleted `\n`/U+FFFC with the captured rows, in document order. + let mut boundaries = Vec::with_capacity(removed_paragraphs.len()); + let mut objects = Vec::with_capacity(removed_objects.len()); + { + let mut paragraph_iter = removed_paragraphs.into_iter(); + let mut object_iter = removed_objects.into_iter(); + let mut offset = clamped_start; + for segment in &restore_delta { + let TextDelta::Insert { insert, .. } = segment else { + tracing::warn!("recorded-inverse capture declined: non-insert slice segment"); + return None; + }; + for ch in insert.chars() { + match ch { + '\n' => { + let Some((paragraph_id, block_id)) = paragraph_iter.next() else { + tracing::warn!("recorded-inverse capture declined: more boundaries than retired paragraphs"); + return None; + }; + boundaries.push(RestoredBoundary { + unicode: offset, + paragraph_id, + block_id, + }); + }, + flowstate_document::OBJECT_REPLACEMENT => { + let Some((block_id, input)) = object_iter.next() else { + tracing::warn!("recorded-inverse capture declined: more placeholders than captured objects"); + return None; + }; + objects.push(RestoredObject { + unicode: offset, + block_id, + input, + }); + }, + _ => {}, + } + offset += 1; + } + } + if paragraph_iter.next().is_some() || object_iter.next().is_some() { + tracing::warn!("recorded-inverse capture declined: retired rows not covered by deleted text"); + return None; + } + } + + // Undo-direction patches: restore the first paragraph's original content and + // re-insert the removed rows (original ids) before the surviving successor. + let first_para_text = paragraph_text(projection, start.paragraph_ix); + if first_para_text.len() < start.byte { + return None; + } + let last_para_len = projection + .paragraphs + .get(end.paragraph_ix) + .map(paragraph_text_len)?; + if last_para_len < end.byte { + return None; + } + let merged_len = start.byte + (last_para_len - end.byte); + let mut undo_patches = vec![ProjectionPatch::ParagraphText { + block_id: projection.ids.block_ids.get(first_block_ix).copied()?, + paragraph_id: projection + .ids + .paragraph_ids + .get(start.paragraph_ix) + .copied()?, + row_hint: first_block_ix, + new: input_paragraph_from_document_range(projection, start.paragraph_ix, 0..usize::MAX), + delta_utf8: projection_text_delta(start.byte, merged_len - start.byte, first_para_text.len() - start.byte, 0), + }]; + let redo_boundaries = boundaries + .iter() + .map(|b| RestoredBoundary { + unicode: b.unicode, + paragraph_id: b.paragraph_id, + block_id: b.block_id, + }) + .collect(); + let prune_objects = !objects.is_empty(); + if !removed_rows.is_empty() { + undo_patches.push(ProjectionPatch::InsertBlocks { + before: projection.ids.block_ids.get(last_block_ix + 1).copied(), + row_hint: first_block_ix + 1, + blocks: removed_rows, + }); + } + + Some(PendingInverseCapture { + undo: RecordedDelta { + mutation: RecordedMutation::RestoreRange { + clamped_start, + restore_delta, + boundaries, + objects, + }, + reproject: Reproject::Patches(undo_patches.into()), + invalidation_ranges: vec![(clamped_start, clamped_len)], + }, + redo_mutation: RecordedMutation::DeleteRange { + clamped_start, + clamped_len, + prune_objects, + boundaries: redo_boundaries, + }, + redo_invalidation_ranges: vec![(clamped_start, clamped_len)], + redo_derive: false, + redo_replace_finalize: None, + }) +} + +/// Post-commit half of the capture: arm the slot with the committed op's +/// counter span, the post-commit frontier, and the op's own synthesized patches +/// (the verbatim redo patches). +pub(crate) fn finalize_capture(core: &mut CrdtRuntime, pending: PendingInverseCapture, op_span: CounterSpan, redo_patches: &[ProjectionPatch]) { + let expected_frontier = core.doc().state_frontiers().encode(); + let redo_reproject = if pending.redo_derive { + Reproject::Derive + } else { + Reproject::Patches(Arc::from(redo_patches)) + }; + // Replace-all: the redo splices can only be captured now, from the POST-op + // doc where the replacements' applied rich content lives. If any slice fails, + // drop the whole record (undo/redo fall back to the checkout path) rather + // than arm a partial one. + let redo_mutation = match pending.redo_replace_finalize { + Some(finalize) => { + let body = body_text(core.doc()); + let mut splices = Vec::with_capacity(finalize.len()); + let mut ok = true; + for entry in finalize { + match body.slice_delta(entry.post_slice_start, entry.post_slice_start + entry.post_slice_len, PosType::Unicode) { + Ok(insert) => splices.push(RichSplice { + start: entry.redo_start, + delete_len: entry.redo_delete_len, + insert, + }), + Err(error) => { + tracing::warn!(%error, "recorded-inverse replace-all redo finalize declined: slice_delta failed"); + ok = false; + break; + }, + } + } + if !ok { + // A partial capture is worse than none: this forward edit is NOT + // fast-undoable, and any older stacked inverse is now unreachable behind + // it (the Loro top span no longer matches), so drop the whole cache. + core.clear_recorded_inverse(); + return; + } + RecordedMutation::SpliceRichRanges { splices } + }, + None => pending.redo_mutation, + }; + // §act-five P3-deep: a forward edit PUSHES its inverse onto the undo stack and + // invalidates the redo stack (the redo timeline is now unreachable). Consecutive + // pushes let a run of undos each replay `O(change)`. + core.recorded_redo_stack().clear(); + core.recorded_undo_stack().push(RecordedInverse { + direction: UndoOrRedo::Undo, + expected_frontier, + expected_span: op_span, + survives_pending_remote: false, + undo: pending.undo, + redo: RecordedDelta { + mutation: redo_mutation, + reproject: redo_reproject, + invalidation_ranges: pending.redo_invalidation_ranges, + }, + }); + // §oom-leads #9: keystrokes capture too now, so a long typing session grows + // the stack one entry per char. Bound it: undos deeper than the cap fall to + // the slow path exactly as an uncaptured op would (the Loro stack has its + // own `max_stack_size` for the same reason). + const MAX_RECORDED_ENTRIES: usize = 512; + let stack = core.recorded_undo_stack(); + if stack.len() > MAX_RECORDED_ENTRIES { + let excess = stack.len() - MAX_RECORDED_ENTRIES; + stack.drain(..excess); + } +} + +/// §act-ten A10.2: re-arm the recorded fast-path stacks across a known-INERT +/// frontier advance (a "meta"-origin metadata/revision commit, e.g. the +/// autosave checkpoint). Those commits touch only meta containers — the body +/// state the recorded inverses replay against is unchanged — but they advance +/// the frontier, which the strict `expected_frontier` equality at step time +/// would read as "concurrent change, drop everything": the exact bug that made +/// fast undo dead in the field (anyone pausing >900ms before Ctrl-Z hit the +/// O(doc) slow path). Only the TOPS need re-arming — the step-time check reads +/// the active top, and each fast step re-arms the next entry as it goes. The +/// `expected_span` fail-safe still validates against the Loro `UndoManager`'s +/// real top (meta commits are origin-excluded there, so it is unchanged). +pub(crate) fn rearm_recorded_inverse_frontier(core: &mut CrdtRuntime) { + let frontier = core.doc().state_frontiers().encode(); + if let Some(top) = core.recorded_undo_stack().last_mut() { + top.expected_frontier.clone_from(&frontier); + } + if let Some(top) = core.recorded_redo_stack().last_mut() { + top.expected_frontier = frontier; + } +} + +/// §mass-op collab (oom-leads #9): the lowest body-unicode coordinate a +/// recorded direction addresses — mutation positions plus invalidation-range +/// starts. `None` for a direction with no coordinates at all (nothing to +/// anchor a disjointness proof to — callers must clear). +fn delta_min_coordinate(delta: &RecordedDelta) -> Option { + let mut min: Option = None; + let mut fold = |value: usize| min = Some(min.map_or(value, |current| current.min(value))); + match &delta.mutation { + RecordedMutation::RestoreRange { + clamped_start, + boundaries, + objects, + .. + } => { + fold(*clamped_start); + for boundary in boundaries { + fold(boundary.unicode); + } + for object in objects { + fold(object.unicode); + } + }, + RecordedMutation::DeleteRange { + clamped_start, boundaries, .. + } => { + fold(*clamped_start); + for boundary in boundaries { + fold(boundary.unicode); + } + }, + RecordedMutation::MarkParagraphStyles { targets } => { + for (unicode, _) in targets { + fold(*unicode); + } + }, + RecordedMutation::SpliceRichRanges { splices } => { + for splice in splices { + fold(splice.start); + } + }, + RecordedMutation::CreateBoundary { at, .. } => fold(*at), + RecordedMutation::MarkRunRanges { spans } => { + for (start, _, _) in spans { + fold(*start); + } + }, + } + for (start, _) in &delta.invalidation_ranges { + fold(*start); + } + min +} + +/// Shift every body-unicode coordinate in one recorded direction by `by`. +/// Returns `false` on arithmetic failure (callers clear the cache). +fn shift_delta_coordinates(delta: &mut RecordedDelta, by: isize) -> bool { + let shift = |value: &mut usize| -> bool { + match value.checked_add_signed(by) { + Some(next) => { + *value = next; + true + }, + None => false, + } + }; + let mutation_ok = match &mut delta.mutation { + RecordedMutation::RestoreRange { + clamped_start, + boundaries, + objects, + .. + } => shift(clamped_start) && boundaries.iter_mut().all(|b| shift(&mut b.unicode)) && objects.iter_mut().all(|o| shift(&mut o.unicode)), + RecordedMutation::DeleteRange { + clamped_start, boundaries, .. + } => shift(clamped_start) && boundaries.iter_mut().all(|b| shift(&mut b.unicode)), + RecordedMutation::MarkParagraphStyles { targets } => targets.iter_mut().all(|(unicode, _)| shift(unicode)), + RecordedMutation::SpliceRichRanges { splices } => splices.iter_mut().all(|splice| shift(&mut splice.start)), + RecordedMutation::CreateBoundary { at, .. } => shift(at), + RecordedMutation::MarkRunRanges { spans } => spans.iter_mut().all(|(start, _, _)| shift(start)), + }; + mutation_ok + && delta + .invalidation_ranges + .iter_mut() + .all(|(start, _)| shift(start)) +} + +/// The import's shape, precomputed once for the per-entry rebase laws. +struct RemoteNetLaw { + /// Highest PRE-space position any net insert lands at. + max_insert_pos: Option, + /// Highest PRE-space position any net delete ends at. + max_delete_end: Option, + /// Net length change (Σ inserts − Σ deletes) — the coordinate shift for an + /// entry whose hull sits above every remote op. + total_delta: isize, + /// Deleted PRE-space ranges, merged and ordered (for per-target drops). + deleted_ranges: Vec<(usize, usize)>, + /// End of the last `changed_text_ranges` entry (POST space) — remote + /// rich/mark changes ride the invalidation without touching the net, and a + /// CONTENT replay (restore/splice re-applies captured text+marks) would + /// clobber one; they must end at/before a content entry's hull. Mark-only + /// replays are exempt: re-marking a surviving boundary char with the + /// recorded value is exactly what the native slow path replays too. + max_changed_end_post: Option, +} + +impl RemoteNetLaw { + fn from_net(net: &crate::crdt_runtime::import_delta::NetBodyDelta, changed_body_ranges_post: &[(usize, usize)]) -> Self { + use crate::crdt_runtime::import_delta::NetOp; + let mut max_insert_pos = None; + let mut max_delete_end = None; + let mut total_delta = 0_isize; + let mut old_pos = 0_usize; + for op in &net.ops { + match op { + NetOp::Retain(n) => old_pos += n, + NetOp::Insert { len, .. } => { + max_insert_pos = Some(max_insert_pos.map_or(old_pos, |current: usize| current.max(old_pos))); + total_delta += *len as isize; + }, + NetOp::Delete(n) => { + old_pos += n; + max_delete_end = Some(max_delete_end.map_or(old_pos, |current: usize| current.max(old_pos))); + total_delta -= *n as isize; + }, + } + } + Self { + max_insert_pos, + max_delete_end, + total_delta, + deleted_ranges: net.deleted_pre_ranges(), + max_changed_end_post: changed_body_ranges_post + .iter() + .map(|(start, len)| start + len) + .max(), + } + } + + /// The v1 hull law against ONE entry's hull floor `lo` (PRE space): every + /// remote insert strictly before it (an insert AT the floor is ambiguous — + /// the CRDT parks a remote edit whose original neighbors were locally + /// deleted at the deletion point, i.e. exactly where a recorded restore + /// would replay, and the native undo would interleave it INSIDE the restored + /// range); deletes and rich changes may END exactly at the floor. + fn before_hull(&self, lo: usize) -> bool { + if !(self.max_insert_pos.is_none_or(|pos| pos < lo) && self.max_delete_end.is_none_or(|end| end <= lo)) { + return false; + } + let Some(lo_post) = lo.checked_add_signed(self.total_delta) else { + return false; + }; + self.max_changed_end_post.is_none_or(|end| end <= lo_post) + } + + /// Whether PRE-space `position`'s char was deleted by the import. + fn position_deleted(&self, position: usize) -> bool { + let ix = self + .deleted_ranges + .partition_point(|(start, _)| *start <= position); + ix > 0 && { + let (start, len) = self.deleted_ranges[ix - 1]; + position < start + len + } + } +} + +/// Per-coordinate rebase of one mark-only direction: `shift_positions` tracks +/// each targeted boundary char exactly (net-delta space math — survivors never +/// land inside inserted runs); a target whose boundary char the import DELETED +/// is dropped, which is what the native slow path's remote-diff transform +/// would do. (Re-marking a KEPT boundary with the recorded value even when a +/// remote rich change touched it is also native semantics — the slow path's +/// transform shifts inverse-mark positions, never removes them.) Returns +/// `(ok, any_dropped)`; `ok == false` truncates the entry — unsorted targets +/// (never produced by capture) or all targets dropped (an empty replay can't +/// drive the external-step protocol). +fn rebase_mark_delta(delta: &mut RecordedDelta, net: &crate::crdt_runtime::import_delta::NetBodyDelta, law: &RemoteNetLaw) -> (bool, bool) { + let RecordedMutation::MarkParagraphStyles { targets } = &mut delta.mutation else { + return (false, false); + }; + if !targets.windows(2).all(|pair| pair[0].0 <= pair[1].0) { + return (false, false); + } + let positions: Vec = targets.iter().map(|(unicode, _)| *unicode).collect(); + let shifted = net.shift_positions(&positions); + let mut kept: Vec<(usize, i64)> = Vec::with_capacity(targets.len()); + for (ix, (position, value)) in targets.iter().enumerate() { + if law.position_deleted(*position) { + continue; + } + kept.push((shifted[ix], *value)); + } + if kept.is_empty() { + return (false, false); + } + let dropped = kept.len() < targets.len(); + // Mark captures record one `(boundary, 1)` invalidation range per target — + // rebuild the set from the kept, shifted boundaries. + delta.invalidation_ranges = kept.iter().map(|(unicode, _)| (*unicode, 1)).collect(); + *targets = kept; + (true, dropped) +} + +/// Rebase ONE entry through the import: `true` keeps it (coordinates now in +/// POST space); `false` means the entry — and everything DEEPER in its stack — +/// must be truncated. +fn rebase_entry( + entry: &mut RecordedInverse, + net: &crate::crdt_runtime::import_delta::NetBodyDelta, + law: &RemoteNetLaw, + structure_neutral: bool, +) -> bool { + let mark_only = matches!(entry.undo.mutation, RecordedMutation::MarkParagraphStyles { .. }) + && matches!(entry.redo.mutation, RecordedMutation::MarkParagraphStyles { .. }); + let keep_patches = if mark_only { + let (undo_ok, undo_dropped) = rebase_mark_delta(&mut entry.undo, net, law); + if !undo_ok { + return false; + } + let (redo_ok, redo_dropped) = rebase_mark_delta(&mut entry.redo, net, law); + if !redo_ok { + return false; + } + // A dropped target means the import deleted a boundary — the projection's + // row set changed, so pre-recorded row-addressed patches are stale. + // (`structure_neutral` is false in that case too; the explicit check keeps + // the coupling local.) + structure_neutral && !undo_dropped && !redo_dropped + } else { + // Content entries (restore/delete/splice) keep the v1 law against their + // OWN hull: all remote ops proven before it, so every recorded coordinate + // shifts by the net's total length delta. + let lo = match (delta_min_coordinate(&entry.undo), delta_min_coordinate(&entry.redo)) { + (Some(a), Some(b)) => a.min(b), + (Some(a), None) => a, + (None, Some(b)) => b, + (None, None) => return false, + }; + if !law.before_hull(lo) { + return false; + } + if law.total_delta != 0 + && !(shift_delta_coordinates(&mut entry.undo, law.total_delta) && shift_delta_coordinates(&mut entry.redo, law.total_delta)) + { + return false; + } + structure_neutral + }; + // Pre-recorded projection patches survive a remote change that neither adds + // nor removes rows: patches are row/id-addressed, and a nonstructural + // character shift touches neither the patched rows' indices nor their + // contents. A STRUCTURAL remote change (paragraph boundary added/removed) + // shifts row indices — those entries downgrade to the derive ladder, which + // recomputes patches at O(change). + if !(keep_patches && patches_survive_rebase(&entry.undo)) { + entry.undo.reproject = Reproject::Derive; + } + if !(keep_patches && patches_survive_rebase(&entry.redo)) { + entry.redo.reproject = Reproject::Derive; + } + true +} + +/// §mass-op collab v2 (oom-leads #9): REBASE the recorded fast-path stacks +/// through a remote import instead of dropping them. Each stack is walked from +/// its TOP (the next entry a fast step replays) downward, deciding PER ENTRY: +/// +/// * Mark-only entries (select-all restyle — both directions re-mark boundary +/// chars, a position-neutral replay) rebase PER-COORDINATE via +/// [`rebase_mark_delta`]: remote edits may land anywhere, including inside +/// the marked span. +/// * Content entries keep the v1 hull law against their OWN hull +/// ([`RemoteNetLaw::before_hull`]) and shift by the net length delta. +/// * The first entry that fails truncates ITSELF AND EVERYTHING DEEPER; the +/// kept suffix stays aligned with the Loro stack tops (once the recorded +/// stack runs dry, `try_fast_step` returns `None` and the slow path serves +/// the deeper items). +/// +/// Space soundness (why no cross-entry position mapping is needed): a deeper +/// entry replays only AFTER the entries above it, so its coordinates live in a +/// space that differs from the current doc by those replays — but every KEPT +/// entry's replay is position-IDENTITY at every remote-op position (mark +/// replays move nothing anywhere; a kept content entry touches only positions +/// at/above its hull, strictly above every remote op). Inductively the remote +/// positions mean the same thing in every kept entry's space, so per-entry +/// checks and shifts against the raw net are exact — and the first entry where +/// the induction would break is exactly where the walk truncates. +/// +/// The Loro stacks' buffered remote diffs stay valid across kept fast steps +/// for the same reason (kept replays are position-neutral at every buffered +/// position) — and vendor patch #22 additionally transforms them through +/// every external-step inverse commit, mirroring the native +/// `undo_internal` path, so a later slow-path pop of a deeper item is +/// coordinate-correct unconditionally. +/// +/// Returns `false` when both stacks ended up empty (the old "cleared" signal). +pub(crate) fn rebase_recorded_inverse_through_remote( + core: &mut CrdtRuntime, + net: &crate::crdt_runtime::import_delta::NetBodyDelta, + changed_body_ranges_post: &[(usize, usize)], + structure_neutral: bool, +) -> bool { + if core.recorded_undo_stack().is_empty() && core.recorded_redo_stack().is_empty() { + return true; + } + let law = RemoteNetLaw::from_net(net, changed_body_ranges_post); + static REBASE_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + let debug = *REBASE_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()); + let frontier = core.doc().state_frontiers().encode(); + let mut any_kept = false; + for kind in [UndoOrRedo::Undo, UndoOrRedo::Redo] { + let stack = active_stack(core, kind); + let before = stack.len(); + let mut cut = None; + for ix in (0..stack.len()).rev() { + if !rebase_entry(&mut stack[ix], net, &law, structure_neutral) { + cut = Some(ix); + break; + } + } + if let Some(ix) = cut { + stack.drain(..=ix); + } + for entry in stack.iter_mut() { + entry.survives_pending_remote = true; + } + if let Some(top) = stack.last_mut() { + top.expected_frontier.clone_from(&frontier); + any_kept = true; + } + if debug { + let kept = active_stack(core, kind).len(); + eprintln!( + "rebase[recorded-inverse]: {kind:?} kept {kept}/{before} pre_span={:?} changed_post={changed_body_ranges_post:?}", + net.pre_change_span() + ); + } + } + any_kept +} + +/// §mass-op collab (oom-leads #9, soundness): whether a pre-recorded patch set +/// stays exact through a (structure-neutral, before-hull) rebase. The hull only +/// bounds the MUTATION coordinates — but `ParagraphText.new`/`delta_utf8` and +/// `ParagraphRuns.runs` snapshot the WHOLE patched paragraph, which extends +/// back before the mutation start (to the paragraph's first char), and +/// `InsertBlocks`/`ReplaceObjectBlock` carry block content snapshots. A remote +/// edit that is hull-disjoint can still land INSIDE the first patched +/// paragraph, ahead of the mutation start; replaying the stale snapshot would +/// clobber it in the projection (the doc-side mutation is position-shifted and +/// stays correct — the divergence is projection-only, which is worse: silent). +/// Only id-addressed, content-free patches survive; everything else downgrades +/// to the derive ladder (still O(change), recomputed from the true doc). +fn patches_survive_rebase(delta: &RecordedDelta) -> bool { + match &delta.reproject { + Reproject::Derive => true, + Reproject::Patches(patches) => patches.iter().all(|patch| { + matches!( + patch, + ProjectionPatch::ParagraphStyle { .. } + | ProjectionPatch::DeleteBlocks { .. } + | ProjectionPatch::MoveBlock { .. } + | ProjectionPatch::AssetArrived { .. } + ) + }), + } +} + +/// Current local-peer counter end (oplog space). +fn counter_end(core: &CrdtRuntime) -> loro::Counter { + let peer = core.doc().peer_id(); + core.doc().oplog_vv().get(&peer).copied().unwrap_or(0) +} + +/// The active fast-path stack for `kind` (undo stack for Undo, redo for Redo) — +/// its TOP is the entry a `kind` step would replay. +fn active_stack(core: &mut CrdtRuntime, kind: UndoOrRedo) -> &mut Vec { + match kind { + UndoOrRedo::Undo => core.recorded_undo_stack(), + UndoOrRedo::Redo => core.recorded_redo_stack(), + } +} + +/// The stack a replayed `kind` entry moves to (so the reverse step replays it). +fn opposite_stack(core: &mut CrdtRuntime, kind: UndoOrRedo) -> &mut Vec { + match kind { + UndoOrRedo::Undo => core.recorded_redo_stack(), + UndoOrRedo::Redo => core.recorded_undo_stack(), + } +} + +/// The undo/redo fast path. Returns `Ok(Some(events))` when the recorded +/// inverse was replayed, `Ok(None)` when the slow path must run instead. Never +/// leaves partial state behind: a replay failure compensates via `revert_to`. +/// +/// §act-five P3-deep: STACK-based — the top of `kind`'s stack is the active +/// entry; on success it moves to the opposite stack, so a RUN of consecutive +/// undos each replays `O(change)`. Fail-safe: every step is validated against +/// the Loro `UndoManager`'s real top span (`peek_top_span`), so a stale entry +/// bails to the correct slow path and the whole cache is dropped. +pub(crate) fn try_fast_step(core: &mut CrdtRuntime, kind: UndoOrRedo) -> Result>> { + if fast_undo_disabled() { + return Ok(None); + } + let expected_frontier = core.doc().state_frontiers().encode(); + let (expected_span, matches, survives_pending_remote) = match active_stack(core, kind).last() { + Some(inverse) => ( + inverse.expected_span, + inverse.expected_frontier == expected_frontier, + inverse.survives_pending_remote, + ), + None => return Ok(None), + }; + if !matches { + // The top entry is stale (a concurrent change moved the frontier). The whole + // local fast-path timeline is now unreachable — drop it; the slow path runs. + core.clear_recorded_inverse(); + return Ok(None); + } + // §oom-leads #9: `clean == false` means remote diffs are buffered in the Loro + // stacks. Normally that kills the fast path (the recorded coordinates would + // be stale), but the rebase proved THIS entry's replay is coordinate-correct, + // and the buffered diffs survive the inverse commit (kept replays are + // position-neutral at every buffered position; vendor patch #22 transforms + // the buffers through the commit besides) — proceed. + match core.undo_manager_mut().peek_top_span(kind) { + Some((span, clean)) if (clean || survives_pending_remote) && span == expected_span => {}, + _ => return Ok(None), + } + + let from_frontier = core.doc().state_frontiers(); + let from_vv = core.doc().state_vv(); + if !core + .undo_manager_mut() + .external_step_begin(kind, expected_span) + { + return Ok(None); + } + + // ---- replay the recorded transaction ------------------------------------ + let counter_before = counter_end(core); + let replayed = hotpath::measure_block!("recorded_inverse_replay", apply_recorded_mutation(core, kind)); + if let Err(error) = replayed { + // I-10-style compensation. + tracing::error!(%error, ?kind, "recorded-inverse replay failed mid-apply; compensating via revert_to"); + core.doc().set_next_commit_origin("repair"); + core + .doc() + .set_next_commit_message("recorded-inverse-compensation"); + if let Err(revert_error) = core.doc().revert_to(&from_frontier) { + core.undo_manager_mut().external_step_abort(kind); + core.clear_recorded_inverse(); + return Err(anyhow::anyhow!( + "recorded-inverse replay failed ({error:#}) and revert_to failed ({revert_error}); runtime must be reloaded" + )); + } + core.doc().set_next_commit_origin("repair"); + core + .doc() + .set_next_commit_message("recorded-inverse-compensation-inverse"); + core.doc().commit(); + core.undo_manager_mut().external_step_abort(kind); + core.clear_recorded_inverse(); + let mut invalidation = ProjectionInvalidation::full_rebuild( + from_frontier.encode(), + core.doc().state_frontiers().encode(), + "recorded-inverse-compensation", + ); + core.merge_subscription_invalidation(&mut invalidation); + let events = core.events_after_local_change(from_frontier, from_vv, invalidation, false)?; + core.queue_publish(events); + return Ok(None); + } + + core.doc().set_next_commit_origin("undo"); + core.doc().set_next_commit_message(match kind { + UndoOrRedo::Undo => "recorded-inverse-undo", + UndoOrRedo::Redo => "recorded-inverse-redo", + }); + core.doc().commit(); + let counter_after = counter_end(core); + if counter_after == counter_before { + tracing::error!(?kind, "recorded-inverse replay committed no ops; falling back"); + core.undo_manager_mut().external_step_abort(kind); + core.clear_recorded_inverse(); + return Ok(None); + } + let inverse_span = CounterSpan::new(counter_before, counter_after); + core + .undo_manager_mut() + .external_step_finish(kind, inverse_span); + + // ---- projection + publish (same ladder as any local write) -------------- + let context = match kind { + UndoOrRedo::Undo => "recorded-inverse-undo", + UndoOrRedo::Redo => "recorded-inverse-redo", + }; + let (recorded_patches, inval_ranges) = { + let inverse = active_stack(core, kind) + .last() + .expect("active entry survives replay"); + let delta = match kind { + UndoOrRedo::Undo => &inverse.undo, + UndoOrRedo::Redo => &inverse.redo, + }; + let patches = match &delta.reproject { + Reproject::Patches(patches) => Some(patches.clone()), + Reproject::Derive => None, + }; + (patches, delta.invalidation_ranges.clone()) + }; + // Style-only and text deltas build the same range-carrying invalidation + // (`body_style` has always delegated to `body_text`), so one multi-range + // constructor serves both directions. + let mut invalidation = ProjectionInvalidation::body_text_ranges(from_frontier.encode(), core.doc().state_frontiers().encode(), &inval_ranges); + let drained = core.merge_subscription_invalidation(&mut invalidation); + let mut events = core.events_after_local_change(from_frontier.clone(), from_vv, invalidation.clone(), false)?; + match recorded_patches { + Some(patches) => { + // Exact pre-recorded patches — O(change), no readback. + core.apply_projection_patch_set(&patches); + core.set_projection_frontier(core.doc().state_frontiers().encode()); + super::commit::audit_patched_projection(core, context); + events.push(core.projection_patched_event(patches, invalidation)); + }, + None => { + // Derive ladder: regional / ranged readback off the committed doc (the + // same path remote imports and semantic commands use). + core.derive_body_projection_events(invalidation, &drained, context, &mut events)?; + super::commit::audit_patched_projection(core, context); + }, + } + if let Some(snapshot) = core.take_restored_undo_selection() { + if let Some(selection) = core.resolve_undo_selection(&snapshot) { + events.push(RuntimeEvent::SelectionRestored { selection }); + } else { + core.restore_undo_selection_later(snapshot); + } + } + + fidelity::event(FidelityClass::Undo, "recorded-inverse", || { + format!( + "kind={kind:?} span={expected_span:?} inverse={inverse_span:?} frontier {:?} -> {:?}", + from_frontier.encode(), + core.doc().state_frontiers().encode() + ) + }); + + // MOVE the replayed entry to the OPPOSITE stack, re-armed for the reverse + // step: the transaction we just committed is exactly what the reverse step + // replays. The entries BELOW it on the active stack keep their arming, so the + // NEXT same-direction step (the Ctrl-Z mash) replays for free too. + let new_frontier = core.doc().state_frontiers().encode(); + let mut entry = active_stack(core, kind) + .pop() + .expect("active entry survives replay"); + entry.direction = match kind { + UndoOrRedo::Undo => UndoOrRedo::Redo, + UndoOrRedo::Redo => UndoOrRedo::Undo, + }; + entry.expected_frontier = new_frontier.clone(); + entry.expected_span = inverse_span; + opposite_stack(core, kind).push(entry); + // §act-five P3-deep: replaying the top committed a NEW inverse op, so the + // frontier advanced. The entry now exposed at the top of the active stack was + // recorded at its OWN commit frontier, which no longer matches — but it IS + // replayable at THIS new frontier (we just moved to it). Re-arm it so the next + // chained step's frontier check passes. Its `expected_span` still matches the + // Loro `UndoManager`'s now-top (external_step popped exactly this direction), so + // only the frontier needs refreshing. + if let Some(next) = active_stack(core, kind).last_mut() { + next.expected_frontier = new_frontier; + } + + Ok(Some(events)) +} + +/// Apply the recorded doc-side mutation for `kind`'s direction to the live doc. +fn apply_recorded_mutation(core: &mut CrdtRuntime, kind: UndoOrRedo) -> Result<()> { + let doc = core.doc().clone(); + let body = body_text(&doc); + // The active entry is the top of `kind`'s stack (validated by the caller). + let inverse = active_stack(core, kind) + .last() + .expect("caller checked the active stack"); + match &(match kind { + UndoOrRedo::Undo => &inverse.undo, + UndoOrRedo::Redo => &inverse.redo, + }) + .mutation + { + RecordedMutation::RestoreRange { + clamped_start, + restore_delta, + boundaries, + objects, + } => { + let mut delta = Vec::with_capacity(restore_delta.len() + 1); + if *clamped_start > 0 { + delta.push(TextDelta::Retain { + retain: *clamped_start, + attributes: None, + }); + } + delta.extend(restore_delta.iter().cloned()); + body + .apply_delta(&delta) + .context("applying recorded-inverse restore delta")?; + for object in objects { + restore_input_object_block_containers(&doc, object.unicode, object.block_id, &object.input) + .context("restoring object block containers for recorded-inverse undo")?; + } + for boundary in boundaries { + repair_paragraph_metadata_after_stable_split( + &doc, + &body, + boundary.unicode, + boundary.paragraph_id, + boundary.block_id, + "recorded_inverse_undo", + ) + .context("restoring paragraph records for recorded-inverse undo")?; + } + }, + RecordedMutation::DeleteRange { + clamped_start, + clamped_len, + prune_objects, + boundaries, + } => { + body + .delete(*clamped_start, *clamped_len) + .context("re-deleting range for recorded-inverse redo")?; + if *prune_objects { + prune_orphaned_body_object_blocks(&doc, &body).context("pruning object blocks for recorded-inverse redo")?; + } + for boundary in boundaries { + delete_projection_paragraph_metadata(&doc, boundary.paragraph_id, boundary.block_id) + .context("retiring paragraph records for recorded-inverse redo")?; + } + }, + RecordedMutation::MarkParagraphStyles { targets } => { + for (boundary, value) in targets { + body + .mark(*boundary..*boundary + 1, MARK_PARAGRAPH_STYLE, *value) + .context("re-marking paragraph style for recorded-inverse step")?; + } + }, + RecordedMutation::CreateBoundary { + at, + style_value, + paragraph_id, + block_id, + } => { + body + .insert(*at, "\n") + .context("re-creating paragraph boundary for recorded-inverse step")?; + body + .mark(*at..*at + 1, MARK_PARAGRAPH_STYLE, *style_value) + .context("marking re-created boundary's paragraph style")?; + // Sentinel hygiene, mirroring the SplitParagraph executor: expand-After + // run marks absorb the inserted newline; strip the run keys so styling + // never bleeds across the boundary. `mark_run_styles` with default + // styles IS the four-key unmark. + crate::crdt_runtime::mark_run_styles(&body, *at..*at + 1, flowstate_document::RunStyles::default()) + .context("stripping run keys from re-created boundary")?; + repair_paragraph_metadata_after_stable_split(&doc, &body, *at, *paragraph_id, *block_id, "recorded_inverse_boundary") + .context("re-writing paragraph records for recorded-inverse boundary")?; + }, + RecordedMutation::MarkRunRanges { spans } => { + for (start, len, styles) in spans { + crate::crdt_runtime::mark_run_styles(&body, *start..*start + *len, *styles) + .context("re-applying run styles for recorded-inverse step")?; + } + }, + RecordedMutation::SpliceRichRanges { splices } => { + // Splices are recorded in DESCENDING start order; applying them in that + // order keeps every earlier splice's position valid. Each range is + // replaced by its captured rich content via one `apply_delta` + // (Retain → Delete → Insert…), preserving marks exactly. + for splice in splices { + let mut delta = Vec::with_capacity(splice.insert.len() + 2); + if splice.start > 0 { + delta.push(TextDelta::Retain { + retain: splice.start, + attributes: None, + }); + } + if splice.delete_len > 0 { + delta.push(TextDelta::Delete { delete: splice.delete_len }); + } + delta.extend(splice.insert.iter().cloned()); + body + .apply_delta(&delta) + .context("applying recorded-inverse rich splice")?; + } + }, + } + Ok(()) +} diff --git a/crates/flowstate-collab/src/local_write/resolve.rs b/crates/flowstate-collab/src/local_write/resolve.rs new file mode 100644 index 00000000..386cbbe8 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/resolve.rs @@ -0,0 +1,174 @@ +//! The resolution law (spec §4): identity + cursor over hint, reject before +//! mutation. +//! +//! Runs strictly inside the write gate, where the maintained projection and +//! index are in-sync with the live doc by construction — so identity lookups +//! through them are lookups against canonical Loro state, not against a +//! possibly-stale render model. Every resolution is O(touched paragraph), never +//! O(document): paragraph starts come from the durable paragraph-record cursor +//! (Loro space) or the maintained index, and the only per-call scan is the +//! byte→codepoint conversion within the one anchored paragraph (spec I-8). +//! +//! Resolution order for a [`TextAnchor`]: +//! 1. If an encoded cursor is present and resolves inside the body flow, the +//! cursor wins — it is finer-grained identity than the paragraph id and +//! survives structural moves (e.g. a remote split relocating the anchored +//! character into a new paragraph). +//! 2. Otherwise the paragraph id resolves through the maintained index and the +//! byte hint is clamped into that paragraph's current text on a char +//! boundary. +//! 3. Otherwise: reject, before any mutation (spec I-15). +//! +//! Cursor caveats baked in from the semantics audit: `get_cursor_pos` on a +//! deleted anchor is a commit barrier + `DiffCalculator` run — legal here (we +//! hold the gate; no pending foreign txn exists inside an intent before its +//! first mutation), but never legal inside a subscription callback (I-9b). +//! Degraded `id: None` cursors resolve to 0 or the whole-body end and must be +//! clamped back into a paragraph (audit F7). + +use flowstate_document::DocumentProjection; +use gpui_flowtext::{DocumentOffset, ParagraphId}; +use loro::cursor::Cursor; +use loro::{ContainerTrait as _, LoroDoc}; + +use super::intents::{TextAnchor, WriteRejected}; +use crate::crdt_runtime::ProjectionRuntimeIndex; + +/// A text anchor resolved against live Loro state. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ResolvedTextPosition { + /// Paragraph index in the in-sync projection (render space). + pub paragraph_ix: usize, + /// UTF-8 byte offset within that paragraph's text, clamped to a char + /// boundary. + pub byte: usize, + /// Unicode-codepoint index in the LIVE Loro body flow (mutation space). + pub body_unicode: usize, +} + +/// Resolve a [`TextAnchor`] per the resolution law. +pub(crate) fn resolve_text_anchor( + doc: &LoroDoc, + projection: &DocumentProjection, + index: &ProjectionRuntimeIndex, + anchor: &TextAnchor, +) -> Result { + // 1. Cursor basis, when present and resolvable inside the body flow. + if let Some(encoded) = &anchor.cursor + && let Some(position) = resolve_body_cursor(doc, projection, index, encoded) + { + return Ok(position); + } + + // 2. Identity basis: paragraph id through the maintained index, byte hint + // clamped into the paragraph. + resolve_paragraph_position(doc, projection, index, anchor.paragraph, anchor.byte_hint) + .ok_or(WriteRejected::UnresolvedParagraph(anchor.paragraph)) +} + +/// Resolve a paragraph identity + byte hint to a live body position. +pub(crate) fn resolve_paragraph_position( + doc: &LoroDoc, + projection: &DocumentProjection, + index: &ProjectionRuntimeIndex, + paragraph: ParagraphId, + byte_hint: usize, +) -> Option { + let paragraph_ix = index.paragraph_index(paragraph)?; + // Defense-in-depth (I-2): the index and projection are in-sync inside the + // gate, but the identity check is cheap and a mismatch here would mean a + // bookkeeping bug about to become a wrong-position op. Reject loudly. + if projection.ids.paragraph_ids.get(paragraph_ix).copied() != Some(paragraph) { + tracing::error!( + paragraph = paragraph.0, + paragraph_ix, + "projection index/id desync detected during resolution; rejecting intent" + ); + return None; + } + let byte = clamp_byte_to_char_boundary(projection, paragraph_ix, byte_hint); + let body_unicode = index.body_unicode_for_offset_in_loro( + doc, + projection, + DocumentOffset { + paragraph: paragraph_ix, + byte, + }, + )?; + Some(ResolvedTextPosition { + paragraph_ix, + byte, + body_unicode, + }) +} + +/// Resolve an encoded body cursor to a live position, mapping it back into +/// render space. Returns `None` (identity fallback) rather than erroring: a +/// cursor is an accelerator, and the paragraph identity is the required basis. +fn resolve_body_cursor( + doc: &LoroDoc, + projection: &DocumentProjection, + index: &ProjectionRuntimeIndex, + encoded: &[u8], +) -> Option { + let cursor = Cursor::decode(encoded).ok()?; + // The cursor must address the body flow; cursors into cell flows or foreign + // containers fall back to identity resolution. + if cursor.container != flowstate_document::loro_schema::body_text(doc).id() { + return None; + } + let position = doc.get_cursor_pos(&cursor).ok()?; + let body_unicode = clamp_degraded_cursor_position(projection, index, &cursor, position.current.pos); + // Map to render space through the maintained index; a position we cannot map + // is not a safe mutation target. + let offset = index.offset_for_body_unicode(projection, body_unicode)?; + Some(ResolvedTextPosition { + paragraph_ix: offset.paragraph, + byte: offset.byte, + body_unicode, + }) +} + +/// Audit F7: a degraded (`id: None`) cursor resolves to 0 or the WHOLE-BODY +/// end, not a paragraph-relative position. Clamp such results into the last +/// valid body position so a degraded cursor can never target the wrong end of +/// a large document silently. +fn clamp_degraded_cursor_position(projection: &DocumentProjection, index: &ProjectionRuntimeIndex, cursor: &Cursor, pos: usize) -> usize { + if cursor.id.is_some() { + return pos; + } + tracing::warn!(pos, "degraded (id-less) cursor resolved; clamping into body range"); + let _ = projection; + index.clamp_body_unicode(pos) +} + +/// Clamp a byte hint into the paragraph's current text on a UTF-8 char +/// boundary. +pub(crate) fn clamp_byte_to_char_boundary(projection: &DocumentProjection, paragraph_ix: usize, byte: usize) -> usize { + let Some(paragraph) = projection.paragraphs.get(paragraph_ix) else { + return 0; + }; + let text = flowstate_document::paragraph_text(projection, paragraph_ix); + let mut byte = byte.min(flowstate_document::paragraph_text_len(paragraph)); + while byte > 0 && !text.is_char_boundary(byte) { + byte -= 1; + } + byte +} + +/// Resolve an ordered (start, end) anchored range. Start and end resolve +/// independently; a range that resolves inverted (concurrent edits moved the +/// endpoints past each other) is normalized rather than rejected — both +/// positions are individually valid, and a normalized range preserves the +/// user's intent of "delete what sits between my endpoints". +pub(crate) fn resolve_text_range( + doc: &LoroDoc, + projection: &DocumentProjection, + index: &ProjectionRuntimeIndex, + start: &TextAnchor, + end: &TextAnchor, +) -> Result<(ResolvedTextPosition, ResolvedTextPosition), WriteRejected> { + let a = resolve_text_anchor(doc, projection, index, start)?; + let b = resolve_text_anchor(doc, projection, index, end)?; + if a.body_unicode <= b.body_unicode { Ok((a, b)) } else { Ok((b, a)) } +} diff --git a/crates/flowstate-collab/src/local_write/tests.rs b/crates/flowstate-collab/src/local_write/tests.rs new file mode 100644 index 00000000..2a559155 --- /dev/null +++ b/crates/flowstate-collab/src/local_write/tests.rs @@ -0,0 +1,2900 @@ +//! Stage-1 architecture proofs for the Loro-first local write path +//! (spec §13.3–13.6, 13.10 at this layer). + +use std::sync::Arc; + +use flowstate_document::{DocumentProjection, ParagraphId, ParagraphStyle, paragraph_text}; + +use super::commit::INJECT_FRAGMENT_FAULT; +use super::gate::{GateHolder, WriteGate}; +use super::handle::{LocalDocHandle, LocalWriteConfig}; +use super::intents::{ + DeleteRangeIntent, FragmentBlock, InsertRichFragmentIntent, InsertTextIntent, SetMarksIntent, SetParagraphStylesIntent, SplitParagraphIntent, + TextAnchor, WriteRejected, +}; +use crate::crdt_runtime::{CrdtRuntime, RuntimeEvent}; + +fn new_handle(title: &str) -> (LocalDocHandle, Arc>) { + let core = CrdtRuntime::new_empty(title).expect("empty runtime"); + LocalDocHandle::new(core, LocalWriteConfig::default()) +} + +fn first_paragraph(projection: &DocumentProjection) -> ParagraphId { + projection.ids.paragraph_ids[0] +} + +fn body_string(gate: &Arc>) -> String { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() +} + +fn frontier(gate: &Arc>) -> Vec { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard.doc().state_frontiers().encode() +} + +/// §13.4: local insert commits synchronously and returns an exact patch; the +/// debug-build audit (compare vs full rebuild) runs inside the same call. +#[test] +fn local_insert_commits_synchronously_and_returns_exact_patch() { + let (handle, gate) = new_handle("insert"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + let before = frontier(&gate); + + let outcome = handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "hello".into(), + style_override: None, + }) + .expect("insert commits"); + let commit = outcome.commit(); + assert!(!commit.patches.patches.is_empty(), "insert must return patches"); + assert_ne!(commit.frontier, before, "commit must advance the frontier"); + assert!(commit.counters.loro_ops >= 1, "ops counter must be recorded"); + assert!(!commit.counters.full_rebuild, "a plain insert must never full-rebuild (I-14)"); + assert_eq!(body_string(&gate), "\nhello"); + + // Appending resolves through identity + hint (clamped) — still exact. + let outcome = handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: " world".into(), + style_override: None, + }) + .expect("append commits"); + assert!(!outcome.commit().counters.full_rebuild); + assert_eq!(body_string(&gate), "\nhello world"); +} + +/// §13.5 / I-15: unresolvable identity rejects BEFORE any mutation. +#[test] +fn resolution_failure_rejects_before_mutation() { + let (handle, gate) = new_handle("reject"); + let before = frontier(&gate); + let result = handle.insert_text(InsertTextIntent { + at: TextAnchor::new(ParagraphId(0xDEAD_BEEF), 0), + text: "x".into(), + style_override: None, + }); + assert!( + matches!(result, Err(WriteRejected::UnresolvedParagraph(_))), + "unknown identity must reject" + ); + assert_eq!(frontier(&gate), before, "rejection must not mutate the doc"); + assert_eq!(body_string(&gate), "\n"); +} + +/// I-2/I-8: structure never enters through a text insert. +#[test] +fn insert_text_rejects_structural_payload() { + let (handle, gate) = new_handle("structure"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + let before = frontier(&gate); + let result = handle.insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "a\nb".into(), + style_override: None, + }); + assert!(matches!(result, Err(WriteRejected::StructureViolation(_)))); + assert_eq!(frontier(&gate), before); +} + +/// Split commits through Loro paragraph-boundary state and patches exactly. +#[test] +fn split_paragraph_commits_and_patches() { + let (handle, gate) = new_handle("split"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "aabb".into(), + style_override: None, + }) + .expect("seed text"); + let outcome = handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, 2), + inherited_style: ParagraphStyle::Normal, + }) + .expect("split commits"); + assert!(!outcome.commit().counters.full_rebuild, "split must patch, not rebuild"); + assert_eq!(body_string(&gate), "\naa\nbb"); + + let projection = handle.projection().expect("projection after split"); + assert_eq!(projection.paragraphs.len(), 2); + assert_eq!(paragraph_text(&projection, 0), "aa"); + assert_eq!(paragraph_text(&projection, 1), "bb"); +} + +/// Cross-paragraph delete merges paragraphs and retires records + blocks. +#[test] +fn delete_range_across_paragraphs_merges() { + let (handle, gate) = new_handle("merge"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "aabb".into(), + style_override: None, + }) + .expect("seed"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, 2), + inherited_style: ParagraphStyle::Normal, + }) + .expect("split"); + let projection = handle.projection().expect("projection"); + let second = projection.ids.paragraph_ids[1]; + + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(paragraph, 1), + end: TextAnchor::new(second, 1), + }) + .expect("cross-paragraph delete commits"); + assert_eq!(body_string(&gate), "\nab"); + let projection = handle.projection().expect("projection after merge"); + assert_eq!(projection.paragraphs.len(), 1); + assert_eq!(paragraph_text(&projection, 0), "ab"); +} + +/// §13.6: compound-intent failure atomicity. A deterministic mid-apply fault +/// must (a) converge the doc back to export-equivalent pre-intent state, +/// (b) let no projection patch escape, (c) publish partial + inverse as ONE +/// payload a peer can import without ever observing partial state. +#[test] +fn compound_intent_failure_compensates_atomically() { + let (handle, gate) = new_handle("compensate"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "base".into(), + style_override: None, + }) + .expect("seed"); + // Drain the seed's publish traffic so the compensation payload is isolated. + let _ = gate + .lock(GateHolder::Test) + .expect("gate healthy") + .take_pending_publish(); + let text_before = body_string(&gate); + let paragraphs_before = handle.projection().expect("projection").paragraphs.len(); + + INJECT_FRAGMENT_FAULT.with(|fault| fault.set(true)); + let result = handle.insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, 4), + blocks: vec![ + FragmentBlock::Paragraph(flowstate_document::InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![flowstate_document::InputRun { + text: "partial".into(), + styles: flowstate_document::RunStyles::default(), + }], + }), + FragmentBlock::Paragraph(flowstate_document::InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![flowstate_document::InputRun { + text: "never".into(), + styles: flowstate_document::RunStyles::default(), + }], + }), + ], + }); + INJECT_FRAGMENT_FAULT.with(|fault| fault.set(false)); + + // (a) rejected as compensated; doc state export-equivalent to pre-intent. + assert!( + matches!(result, Err(WriteRejected::CompensatedFailure { .. })), + "fault must surface as compensated failure: {result:?}" + ); + assert_eq!(body_string(&gate), text_before, "doc must converge back to pre-intent state"); + let projection_after = handle.projection().expect("projection"); + assert_eq!(projection_after.paragraphs.len(), paragraphs_before); + assert_eq!(paragraph_text(&projection_after, 0), "base"); + + // (b)/(c): the compensation left exactly one atomic publish payload whose + // import converges a fresh peer to the same (pre-intent-equivalent) state. + let events = gate + .lock(GateHolder::Test) + .expect("gate healthy") + .take_pending_publish(); + let payload_count = events + .iter() + .filter(|event| matches!(event, RuntimeEvent::LocalUpdate { .. })) + .count(); + assert_eq!(payload_count, 1, "partial + inverse must publish as ONE atomic payload"); + + // Convergence bar: a fresh peer importing our FULL history (which contains + // the compensated pair) must agree with us and must never see the partial + // content. (Both replicas seed their own sentinel, so we assert convergence + // and partial-invisibility rather than byte equality with our replica.) + let mut peer = CrdtRuntime::new_empty("compensate").expect("peer"); + let full_history = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export full updates") + }; + peer + .import_remote_update(&full_history) + .expect("peer imports"); + let peer_history = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_update(&peer_history) + .expect("local imports peer seed"); + drop(guard); + let peer_text = flowstate_document::loro_schema::body_text(peer.doc()).to_string(); + let local_text = body_string(&gate); + assert_eq!(peer_text, local_text, "replicas must converge after full exchange"); + assert!( + !peer_text.contains("partial"), + "no replica may ever observe the compensated partial content" + ); + assert!(!peer_text.contains("never"), "the never-applied second block must not exist anywhere"); + assert!(peer_text.contains("base"), "the pre-intent content survives"); +} + +/// §13.3 gate atomicity: a competing import thread hammering remote updates +/// can never interleave inside an intent — every committed insert lands where +/// its identity resolved, and both peers converge. +#[test] +fn concurrent_imports_never_interleave_inside_an_intent() { + let (handle, gate) = new_handle("gate-atomicity"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + + // Peer B edits its own replica. Converge the two mergeable seeds FIRST so + // the race runs over one shared document state (each replica seeds its own + // sentinel; the exchange merges them identically on both sides). + let mut peer = CrdtRuntime::new_empty("gate-atomicity").expect("peer"); + let local_seed = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer + .import_remote_update(&local_seed) + .expect("peer imports seed"); + let peer_seed = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_update(&peer_seed) + .expect("local imports peer seed"); + drop(guard); + let peer_paragraph = peer.projection_ref().ids.paragraph_ids[0]; + assert_eq!( + paragraph, peer_paragraph, + "mergeable seed must give both replicas the same initial paragraph identity" + ); + + let gate_for_imports = Arc::clone(&gate); + let (updates_tx, updates_rx) = std::sync::mpsc::channel::>(); + let importer = std::thread::spawn(move || { + for update in updates_rx { + let mut guard = gate_for_imports + .lock(GateHolder::ImportChunk) + .expect("gate healthy"); + guard.import_remote_update(&update).expect("import applies"); + } + }); + + const LOCAL_CHARS: usize = 40; + let mut peer_vv = peer.doc().state_vv(); + for i in 0..LOCAL_CHARS { + // Peer B prepends into the shared paragraph and ships the update. + let peer_body = flowstate_document::loro_schema::body_text(peer.doc()); + peer_body.insert(1, "R").expect("peer insert"); + peer.doc().commit(); + let update = peer + .doc() + .export(loro::ExportMode::updates(&peer_vv)) + .expect("peer export"); + peer_vv = peer.doc().state_vv(); + updates_tx.send(update).expect("send update"); + + // Local user appends at the paragraph end through the intent API while + // imports race on the other side of the gate. + let digit = char::from(b'a' + (i % 26) as u8); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: digit.to_string(), + style_override: None, + }) + .expect("local insert commits"); + } + drop(updates_tx); + importer.join().expect("importer thread"); + + // Full exchange: peer receives everything local, local received everything + // remote (already, via the importer thread). + let local_update = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer + .import_remote_update(&local_update) + .expect("peer imports local history"); + let local_text = body_string(&gate); + let peer_text = flowstate_document::loro_schema::body_text(peer.doc()).to_string(); + assert_eq!(local_text, peer_text, "replicas must converge"); + + // The local keystrokes were appends resolved by identity inside the gate: + // they must appear in order, uninterleaved by the remote prepends. + let expected: String = (0..LOCAL_CHARS) + .map(|i| char::from(b'a' + (i % 26) as u8)) + .collect(); + let locals: String = local_text + .chars() + .filter(|c| c.is_ascii_lowercase()) + .collect(); + assert_eq!(locals, expected, "local appends must retain their intended order and placement"); + assert_eq!(local_text.chars().filter(|c| *c == 'R').count(), LOCAL_CHARS); +} + +/// Undo executes through the Loro `UndoManager` and restores prior content. +#[test] +fn undo_redo_round_trip_through_loro() { + let (handle, gate) = new_handle("undo"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "abc".into(), + style_override: None, + }) + .expect("insert"); + assert_eq!(body_string(&gate), "\nabc"); + + let outcome = handle.apply_undo().expect("undo runs"); + assert!(outcome.applied, "undo must apply a projection change"); + assert_eq!(body_string(&gate), "\n", "undo must restore pre-insert content"); + + let outcome = handle.apply_redo().expect("redo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), "\nabc", "redo must restore the insert"); +} + +/// Spec §13.9 (rewritten per the Loro semantics audit): a NON-DISJOINT remote +/// import closes the active undo group — that is Loro's collaborative-undo +/// safety behavior, asserted rather than fought. Commits before the import +/// undo as one unit; commits after it are separate items; the editor re-arms +/// grouping at the next boundary. +#[test] +fn remote_import_mid_group_closes_the_group() { + let (handle, gate) = new_handle("undo-groups"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + + // A converged peer (seed exchange first, as in the other harnesses). + let mut peer = CrdtRuntime::new_empty("undo-groups").expect("peer"); + let local_seed = { + let guard = gate.lock(GateHolder::Test).expect("gate"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer + .import_remote_update(&local_seed) + .expect("peer imports seed"); + let peer_seed = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate"); + guard + .import_remote_update(&peer_seed) + .expect("local imports peer seed"); + drop(guard); + let peer_vv = peer.doc().state_vv(); + + // Open a group and commit two intents inside it. + assert!(handle.begin_undo_group().expect("gate healthy"), "group opens"); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: "one".into(), + style_override: None, + }) + .expect("first grouped insert"); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: "two".into(), + style_override: None, + }) + .expect("second grouped insert"); + + // Remote peer edits the SAME body container; importing it mid-group is + // non-disjoint and closes the group underneath us. + let peer_body = flowstate_document::loro_schema::body_text(peer.doc()); + peer_body.insert(1, "R").expect("peer insert"); + peer.doc().commit(); + let update = peer + .doc() + .export(loro::ExportMode::updates(&peer_vv)) + .expect("export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate"); + guard.import_remote_update(&update).expect("import applies"); + drop(guard); + + // Post-import commit: with merge_interval(0) and the group closed, this is + // its own undo item. + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: "three".into(), + style_override: None, + }) + .expect("post-import insert"); + // end_undo_group is a no-op on the already-closed group; the editor re-arms. + handle.finish_undo_group().expect("gate healthy"); + + let full_text = body_string(&gate); + assert!( + full_text.contains("onetwo") && full_text.contains("three"), + "all three inserts landed (remote R may interleave): {full_text:?}" + ); + + // First undo removes ONLY the post-import commit ("three") — the group did + // not swallow it. + handle.apply_undo().expect("undo runs"); + let after_first_undo = body_string(&gate); + assert!( + after_first_undo.contains("onetwo") && !after_first_undo.contains("three"), + "post-import commit undoes alone: {after_first_undo:?}" + ); + assert!(after_first_undo.contains('R'), "remote content is never undone locally"); + + // Second undo removes the grouped pre-import commits as one unit. + handle.apply_undo().expect("undo runs"); + let after_second_undo = body_string(&gate); + assert!( + !after_second_undo.contains("one") && !after_second_undo.contains("two"), + "the pre-import group undoes as one unit: {after_second_undo:?}" + ); + assert!(after_second_undo.contains('R'), "remote content survives all local undo"); +} + +/// Replace-all (find & replace) through the ONE write path: same-paragraph +/// matches ride one compound intent = one commit = one undo member; canonical +/// state carries the replacements (the pre-intent editor-side mutation lost +/// them — the 2026-07-07 replace-all data-loss class). +#[test] +fn replace_matches_commits_canonically_and_undoes_as_one_unit() { + let (handle, gate) = new_handle("replace"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "foo alpha foo beta foo".into(), + style_override: None, + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + let second = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("second paragraph"); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(second, 0), + text: "foo gamma".into(), + style_override: None, + }) + .expect("seed second paragraph"); + + // Matches across two paragraphs, in ascending order (the write path sorts + // and applies back-to-front itself). + let matches = vec![ + super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 0), + end: TextAnchor::new(paragraph, 3), + styles: None, + }, + super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 10), + end: TextAnchor::new(paragraph, 13), + styles: None, + }, + super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 19), + end: TextAnchor::new(paragraph, 22), + styles: None, + }, + super::intents::ReplaceMatch { + start: TextAnchor::new(second, 0), + end: TextAnchor::new(second, 3), + styles: None, + }, + ]; + let outcome = handle + .replace_matches(super::intents::ReplaceMatchesIntent { + matches, + replacement: "QUUX".into(), + }) + .expect("replace commits"); + let commit = outcome.commit(); + assert!(!commit.counters.full_rebuild, "replace-matches must patch, not rebuild"); + assert_eq!(body_string(&gate), "\nQUUX alpha QUUX beta QUUX\nQUUX gamma"); + + // ONE undo restores every match — the storm was one commit. + handle.apply_undo().expect("undo runs"); + assert_eq!(body_string(&gate), "\nfoo alpha foo beta foo\nfoo gamma"); + handle.apply_redo().expect("redo runs"); + assert_eq!(body_string(&gate), "\nQUUX alpha QUUX beta QUUX\nQUUX gamma"); +} + +/// Replace with the empty string deletes matches; collapsed and overlapping +/// ranges are pruned rather than double-edited; a structural replacement is +/// rejected before mutation. +#[test] +fn replace_matches_edge_cases() { + let (handle, gate) = new_handle("replace-edges"); + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "xxABxxCDxx".into(), + style_override: None, + }) + .expect("seed insert"); + + // Overlapping second range (2..6 vs 4..8) is pruned; collapsed (8..8) is + // skipped; the empty replacement deletes what survives. + let outcome = handle.replace_matches(super::intents::ReplaceMatchesIntent { + matches: vec![ + super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 2), + end: TextAnchor::new(paragraph, 6), + styles: None, + }, + super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 4), + end: TextAnchor::new(paragraph, 8), + styles: None, + }, + super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 8), + end: TextAnchor::new(paragraph, 8), + styles: None, + }, + ], + replacement: String::new(), + }); + outcome.expect("empty replacement deletes matches"); + assert_eq!(body_string(&gate), "\nxxCDxx"); + + // Structural replacement text is rejected before any mutation. + let rejected = handle.replace_matches(super::intents::ReplaceMatchesIntent { + matches: vec![super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 0), + end: TextAnchor::new(paragraph, 2), + styles: None, + }], + replacement: "a\nb".into(), + }); + assert!(matches!(rejected, Err(WriteRejected::StructureViolation(_)))); + assert_eq!(body_string(&gate), "\nxxCDxx", "rejected intent must not mutate"); + + // All matches collapsed/skipped ⇒ EmptyIntent. + let empty = handle.replace_matches(super::intents::ReplaceMatchesIntent { + matches: vec![super::intents::ReplaceMatch { + start: TextAnchor::new(paragraph, 3), + end: TextAnchor::new(paragraph, 3), + styles: None, + }], + replacement: "y".into(), + }); + assert!(matches!(empty, Err(WriteRejected::EmptyIntent))); +} + +// --------------------------------------------------------------------------- +// §act-three B.1: recorded-inverse undo/redo fast path +// --------------------------------------------------------------------------- + +/// Build a mass multi-paragraph document (large enough to qualify for the +/// recorded-inverse capture) with a styled run per paragraph so mark restore +/// is exercised, plus optional object rows. +fn seed_mass_fragment(handle: &LocalDocHandle, paragraphs: usize, with_equation_at: Option) { + let projection = handle.projection().expect("projection"); + let paragraph = first_paragraph(&projection); + let mut blocks = Vec::with_capacity(paragraphs + 1); + for ix in 0..paragraphs { + if Some(ix) == with_equation_at { + blocks.push(FragmentBlock::Object(flowstate_document::InputBlock::Equation( + flowstate_document::InputEquationBlock { + source: "e = mc^2".into(), + syntax: flowstate_document::InputEquationSyntax::Latex, + display: flowstate_document::InputEquationDisplay::Display, + }, + ))); + } + let styled = flowstate_document::RunStyles { + direct_underline: ix % 3 == 0, + strikethrough: ix % 5 == 0, + ..Default::default() + }; + blocks.push(FragmentBlock::Paragraph(flowstate_document::InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![ + flowstate_document::InputRun { + text: format!("paragraph {ix:03} plain lead-in "), + styles: flowstate_document::RunStyles::default(), + }, + flowstate_document::InputRun { + text: format!("styled body text for row {ix:03} padding padding padding padding"), + styles: styled, + }, + ], + })); + } + handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, 0), + blocks, + }) + .expect("mass fragment seeds"); +} + +/// `(undo_stack.len(), redo_stack.len())` — the fast-path stack depths. Because +/// every fast step MOVES one entry between the stacks (and the slow path clears +/// both), these depths prove exactly how many consecutive steps stayed fast. +fn stack_depths(gate: &Arc>) -> (usize, usize) { + let mut guard = gate.lock(GateHolder::Test).expect("gate healthy"); + (guard.recorded_undo_stack().len(), guard.recorded_redo_stack().len()) +} + +/// §act-five P3-deep (fail-loud): a RUN of consecutive local undos must EACH +/// replay via the fast path — not just the first. On the old single-slot design +/// undo #2+ fell to the O(doc/history) Loro `UndoManager`; the stack keeps them +/// all O(change). `stack_depths` is the proof: three undos move all three +/// inverses undo→redo, which the single slot could not do. +#[test] +fn recorded_inverse_stack_chains_consecutive_fast_undos() { + let (handle, gate) = new_handle("recorded-inverse-chain"); + seed_mass_fragment(&handle, 40, None); + // Restyle every paragraph EXCEPT the first: boundary 0 (`ROOT_FIRST_PARAGRAPH`) + // has a pre-existing live-vs-canonical style quirk orthogonal to this change, + // so excluding it keeps the convergence assertion a clean P3-deep signal. + let ids: Vec = paragraph_ids(&handle).into_iter().skip(1).collect(); + let styles_of = |handle: &LocalDocHandle| -> Vec { + handle + .projection() + .expect("projection") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + let original = styles_of(&handle); + + // THREE consecutive mass restyles — each arms its own recorded inverse. + for slot in [1_u8, 2, 3] { + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids.clone(), + style: ParagraphStyle::Custom(slot), + }) + .expect("restyle commits"); + } + let after_three = styles_of(&handle); + assert!( + after_three + .iter() + .skip(1) + .all(|style| *style == ParagraphStyle::Custom(3)), + "third restyle wins for all restyled paragraphs" + ); + assert_eq!(stack_depths(&gate), (3, 0), "three forward edits stack three undoable inverses"); + + // THREE undos — EACH must use the fast path (all three move undo→redo). + for _ in 0..3 { + assert!(handle.apply_undo().expect("undo runs").applied); + } + assert_eq!( + stack_depths(&gate), + (0, 3), + "P3-deep: all three consecutive undos replayed via the fast path" + ); + assert_eq!(styles_of(&handle), original, "chained undo restores the original styles"); + + // THREE redos — each fast, back to the third restyle. + for _ in 0..3 { + assert!(handle.apply_redo().expect("redo runs").applied); + } + assert_eq!( + stack_depths(&gate), + (3, 0), + "P3-deep: all three consecutive redos replayed via the fast path" + ); + assert_eq!(styles_of(&handle), after_three, "chained redo re-applies every restyle"); + + // The fast-path chain must produce the SAME canonical Loro state the slow path + // would: the maintained (live) projection equals a fresh materialization of the + // committed doc (P3-deep changes only WHICH replay path runs, not the ops). + let local_canonical: Vec = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()) + .expect("materializes") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + assert_eq!(styles_of(&handle), local_canonical, "the fast-path chain leaves live == canonical"); +} + +/// §act-ten A10.2 (the field killer): the 900ms autosave commits metadata + +/// revision ops, advancing the frontier — on the old strict `expected_frontier` +/// equality this cleared the recorded stacks, so anyone pausing before Ctrl-Z +/// fell to the O(doc) slow path. The "meta"-origin commit is undo-INERT (it +/// never touches the body), so the stacks must survive it. `stack_depths` is +/// the proof the step stayed fast (a slow-path step clears both stacks). +#[test] +fn recorded_inverse_survives_inert_checkpoint_commits() { + let (handle, gate) = new_handle("checkpoint-inert"); + seed_mass_fragment(&handle, 12, None); + let ids: Vec = paragraph_ids(&handle).into_iter().skip(1).collect(); + let styles_of = |handle: &LocalDocHandle| -> Vec { + handle + .projection() + .expect("projection") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + let original = styles_of(&handle); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids, + style: ParagraphStyle::Custom(4), + }) + .expect("restyle commits"); + assert_eq!(stack_depths(&gate), (1, 0), "forward edit arms one recorded inverse"); + + // The autosave: metadata + revision commits (the in-place checkpoint path). + let dir = std::env::temp_dir().join(format!("flowstate-checkpoint-inert-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("autosave.db8"); + { + let mut guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .checkpoint_package("Autosave", Some(path.clone())) + .expect("checkpoint runs") + }; + + // Undo AFTER the autosave must still take the fast path. + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "undo stayed FAST across the inert autosave commit"); + assert_eq!(styles_of(&handle), original, "undo restored the pre-restyle styles"); + + // And redo across ANOTHER autosave. + { + let mut guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .checkpoint_package("Autosave", Some(path)) + .expect("second checkpoint runs") + }; + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "redo stayed FAST across the inert autosave commit"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// §act-ten A10.2: a fully-pending (out-of-order) remote import applies NOTHING +/// — the frontier is unchanged, so the recorded fast-undo timelines are exactly +/// as valid as before. The old eager clear-at-entry destroyed them on every +/// drip, which under live collab meant fast undo almost never survived. +#[test] +fn recorded_inverse_survives_noop_pending_import() { + let (handle, gate) = new_handle("noop-import"); + seed_mass_fragment(&handle, 12, None); + let ids: Vec = paragraph_ids(&handle).into_iter().skip(1).collect(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids, + style: ParagraphStyle::Custom(4), + }) + .expect("restyle commits"); + assert_eq!(stack_depths(&gate), (1, 0)); + + // Build an out-of-order update on a peer: two edits, export ONLY the second + // (its dependency is missing on the receiver ⇒ fully pending, applies nothing). + let (peer, peer_gate) = new_handle("noop-import-peer"); + let peer_paragraph = first_paragraph(&peer.projection().expect("projection")); + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraph, 0), + text: "first".into(), + style_override: None, + }) + .expect("peer edit 1"); + let vv_after_first = { + let guard = peer_gate.lock(GateHolder::Test).expect("gate healthy"); + guard.doc().state_vv() + }; + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraph, 0), + text: "second".into(), + style_override: None, + }) + .expect("peer edit 2"); + let second_only = { + let guard = peer_gate.lock(GateHolder::Test).expect("gate healthy"); + let export = guard + .export_updates_for(&vv_after_first) + .expect("delta export"); + drop(guard); + export + }; + + // Import the dependency-less blob: fully pending, frontier unchanged. + let mut guard = gate.lock(GateHolder::Test).expect("gate healthy"); + let frontier_before = guard.doc().state_frontiers(); + guard + .import_remote_update(&second_only) + .expect("pending import is accepted"); + let frontier_after = guard.doc().state_frontiers(); + drop(guard); + assert_eq!(frontier_after, frontier_before, "fully-pending import applied nothing"); + assert_eq!(stack_depths(&gate), (1, 0), "no-op import kept the recorded timeline"); + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "undo stayed FAST after the no-op import"); +} + +/// §oom-leads #9 (mass-op collab): a remote import whose changes land strictly +/// BEFORE every recorded coordinate must REBASE the fast-path stacks — shift +/// the recorded coordinates through the import's net delta and keep the fast +/// step armed — instead of the old unconditional clear (which made fast undo +/// effectively dead under live collab: any partner keystroke between an edit +/// and Ctrl-Z forced the O(doc/history) slow path). A remote change AT/INSIDE +/// the recorded hull must still clear: the recorded replay content would +/// clobber it, and mixed-space coordinate math is unsound there. +#[test] +fn recorded_inverse_rebases_across_hull_disjoint_remote_import() { + let (handle, gate) = new_handle("rebase-disjoint"); + seed_mass_fragment(&handle, 24, None); + // Bidirectional seed exchange so later peer deltas import cleanly (both + // sides hold both `new_empty` sentinels). + let (peer, peer_gate) = new_handle("rebase-disjoint-peer"); + let local_history = { + let g = gate.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + { + let mut g = peer_gate.lock(GateHolder::ImportChunk).expect("peer gate"); + g.import_remote_update(&local_history) + .expect("peer imports local history") + }; + let peer_history = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&peer_history) + .expect("local imports peer seed") + }; + let styles_of = |handle: &LocalDocHandle| -> Vec { + handle + .projection() + .expect("projection") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + let original = styles_of(&handle); + + // A restyles the BOTTOM half — the recorded hull sits far above position 0. + let ids: Vec = paragraph_ids(&handle).into_iter().skip(12).collect(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids, + style: ParagraphStyle::Custom(5), + }) + .expect("restyle commits"); + assert_eq!(stack_depths(&gate), (1, 0), "forward edit arms one recorded inverse"); + + // B types near the TOP of the document — strictly before the hull. + let peer_paragraphs = paragraph_ids(&peer); + let vv_before_chatter = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc().state_vv() + }; + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraphs[1], 0), + text: "typed above the restyled region ".into(), + style_override: None, + }) + .expect("peer chatter above the hull"); + let chatter = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.export_updates_for(&vv_before_chatter) + .expect("delta export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&chatter).expect("chatter imports") + }; + assert_eq!( + stack_depths(&gate), + (1, 0), + "hull-disjoint remote import REBASED the recorded inverse instead of clearing it" + ); + + // Undo/redo AFTER the remote chatter must both stay on the fast path AND be + // content-correct (the coordinates were shifted through the net delta). + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "undo stayed FAST across the rebased remote import"); + assert_eq!( + styles_of(&handle), + original, + "undo restored the pre-restyle styles at the shifted positions" + ); + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "redo stayed FAST across the rebased remote import"); + + // The fast-path replay must leave live == canonical (same bar as P3-deep). + let canonical: Vec = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()) + .expect("materializes") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + assert_eq!(styles_of(&handle), canonical, "rebased fast-path replay leaves live == canonical"); + + // §v2: chatter INSIDE the hull used to clear (v1's global-hull law) — a + // MARK-ONLY entry now rebases per-coordinate instead (its replay is + // position-neutral, so in-hull chatter is sound). The residual clear cases + // live in `remote_insert_at_recorded_coordinate_clears_instead_of_rebasing` + // (content entry, at-coordinate) and the truncation half of + // `mark_top_survives_while_deeper_keystroke_truncates_then_slow_path_converges`. + let vv_before_inside = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc().state_vv() + }; + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraphs[20], 0), + text: "typed inside the restyled region ".into(), + style_override: None, + }) + .expect("peer chatter inside the hull"); + let inside = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.export_updates_for(&vv_before_inside) + .expect("delta export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&inside) + .expect("in-hull chatter imports") + }; + assert_eq!( + stack_depths(&gate), + (1, 0), + "in-hull chatter REBASES a mark-only entry per-coordinate (v2)" + ); + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "undo stayed FAST across the in-hull rebase"); + assert_eq!( + styles_of(&handle), + original, + "fast undo after the in-hull rebase still restores the styles" + ); + let canonical_after: Vec = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()) + .expect("materializes") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + assert_eq!(styles_of(&handle), canonical_after, "in-hull rebased replay leaves live == canonical"); +} + +/// §oom-leads #9 (keystroke undo): EVERY `InsertText` and intra-paragraph +/// `DeleteRange` records its inverse, so the most common ops — typing and +/// backspace — undo/redo O(change) with no checkout, AND no longer clear a +/// deeper captured fast path (a single typed char used to kill a recorded +/// mass-op inverse behind it). `stack_depths` is the fast-path proof: the slow +/// path clears both stacks; every fast step moves one entry. +#[test] +fn recorded_inverse_captures_keystrokes_and_backspace() { + let (handle, gate) = new_handle("keystroke-capture"); + seed_mass_fragment(&handle, 12, None); + let target = paragraph_ids(&handle)[3]; + let original = body_string(&gate); + + // Three keystrokes, one styled — three armed inverses. + for (ch, style) in [ + ("a", None), + ("b", None), + ( + "c", + Some(flowstate_document::RunStyles { + direct_underline: true, + ..Default::default() + }), + ), + ] { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(target, 0), + text: ch.to_string(), + style_override: style, + }) + .expect("keystroke commits"); + } + assert_eq!(stack_depths(&gate), (3, 0), "every keystroke arms a recorded inverse"); + + // Backspace: a 1-char intra-paragraph delete — captured too. + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(target, 0), + end: TextAnchor::new(target, 1), + }) + .expect("backspace commits"); + assert_eq!(stack_depths(&gate), (4, 0), "an intra-paragraph delete arms a recorded inverse"); + let after_edits = body_string(&gate); + + // Four undos — ALL fast (undo→redo moves, no clear), back to the original. + for _ in 0..4 { + assert!(handle.apply_undo().expect("undo runs").applied); + } + assert_eq!(stack_depths(&gate), (0, 4), "all four keystroke undos replayed via the fast path"); + assert_eq!(body_string(&gate), original, "chained keystroke undo restores the original body"); + + // Four redos — all fast, back to the edited body (styled char included). + for _ in 0..4 { + assert!(handle.apply_redo().expect("redo runs").applied); + } + assert_eq!(stack_depths(&gate), (4, 0), "all four keystroke redos replayed via the fast path"); + assert_eq!(body_string(&gate), after_edits, "chained keystroke redo re-applies every edit"); + + // Live projection == canonical materialization (the same bar as P3-deep). + let live = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + let canonical = { + let document = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("materializes") + }; + (0..document.paragraphs.len()) + .map(|ix| paragraph_text(&document, ix)) + .collect::>() + }; + assert_eq!(live, canonical, "keystroke fast-path replay leaves live == canonical"); +} + +/// §oom-leads #9 (keystroke undo × rebase): the field's most common freeze — +/// type, partner types ABOVE you, Ctrl-Z — must stay on the fast path: the +/// keystroke's recorded inverse rebases through the hull-disjoint remote import +/// (coordinates shift by the net delta) instead of clearing. +#[test] +fn recorded_inverse_keystroke_survives_hull_disjoint_chatter() { + let (handle, gate) = new_handle("keystroke-rebase"); + seed_mass_fragment(&handle, 24, None); + let (peer, peer_gate) = new_handle("keystroke-rebase-peer"); + let local_history = { + let g = gate.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + { + let mut g = peer_gate.lock(GateHolder::ImportChunk).expect("peer gate"); + g.import_remote_update(&local_history) + .expect("peer imports local history") + }; + let peer_history = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&peer_history) + .expect("local imports peer seed") + }; + + // A types DEEP in the document — the recorded hull sits far above position 0. + let deep = paragraph_ids(&handle)[16]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(deep, 4), + text: "X".into(), + style_override: None, + }) + .expect("local keystroke commits"); + assert_eq!(stack_depths(&gate), (1, 0), "the keystroke armed a recorded inverse"); + let after_keystroke_projection = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + + // B types near the TOP — strictly before the hull. + let peer_paragraphs = paragraph_ids(&peer); + let vv_before_chatter = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc().state_vv() + }; + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraphs[1], 0), + text: "partner typed above ".into(), + style_override: None, + }) + .expect("peer chatter above the hull"); + let chatter = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.export_updates_for(&vv_before_chatter) + .expect("delta export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&chatter).expect("chatter imports") + }; + assert_eq!( + stack_depths(&gate), + (1, 0), + "hull-disjoint chatter REBASED the keystroke inverse instead of clearing it" + ); + + // Ctrl-Z after the partner's edit: fast, and the deep 'X' is gone while the + // partner's text survives. + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "keystroke undo stayed FAST across the rebased remote import"); + let live = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + assert!( + live + .iter() + .any(|text| text.contains("partner typed above ")), + "the partner's chatter survives the undo" + ); + assert!(!live.iter().any(|text| text.contains('X')), "the undone keystroke is gone"); + let canonical = { + let document = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("materializes") + }; + (0..document.paragraphs.len()) + .map(|ix| paragraph_text(&document, ix)) + .collect::>() + }; + assert_eq!(live, canonical, "rebased keystroke undo leaves live == canonical"); + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "keystroke redo stayed FAST too"); + let live_after_redo = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + assert!( + live_after_redo.iter().any(|text| text.contains('X')), + "redo re-applied the keystroke at the shifted position" + ); + // The projection must equal the pre-chatter shape plus the chatter. + assert_ne!(live_after_redo, after_keystroke_projection, "the chatter is part of the final projection"); +} + +/// §oom-leads #9 (rebase soundness, v1 gap): a mass delete's pre-recorded UNDO +/// PATCHES snapshot the WHOLE first paragraph — including its prefix BEFORE the +/// delete start, which is OUTSIDE the coordinate hull. A hull-disjoint remote +/// edit landing in that prefix passes the rebase, and replaying the stale +/// snapshot would silently clobber it in the projection (the doc-side mutation +/// is shifted and stays correct — live and canonical diverge). The rebase must +/// downgrade content-carrying patches to the derive ladder. +#[test] +fn rebase_downgrades_content_patches_when_remote_edits_first_patched_paragraph() { + let (handle, gate) = new_handle("rebase-patch-content"); + seed_mass_fragment(&handle, 40, None); + let (peer, peer_gate) = new_handle("rebase-patch-content-peer"); + let local_history = { + let g = gate.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + { + let mut g = peer_gate.lock(GateHolder::ImportChunk).expect("peer gate"); + g.import_remote_update(&local_history) + .expect("peer imports local history") + }; + let peer_history = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&peer_history) + .expect("local imports peer seed") + }; + + // A mass-deletes paragraphs 12..38 starting MID-paragraph-12: the first + // patched paragraph keeps a prefix BEFORE the recorded hull. + let ids = paragraph_ids(&handle); + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(ids[12], 9), + end: TextAnchor::new(ids[38], 12), + }) + .expect("mass delete commits"); + assert_eq!(stack_depths(&gate), (1, 0), "the mass delete armed a recorded inverse"); + + // B types INSIDE paragraph 12's surviving prefix — before the hull, but + // inside the first patched paragraph's snapshot. + let peer_paragraphs = paragraph_ids(&peer); + let vv_before = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc().state_vv() + }; + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraphs[12], 2), + text: "REMOTE".into(), + style_override: None, + }) + .expect("peer edits the first patched paragraph's prefix"); + let chatter = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.export_updates_for(&vv_before).expect("delta export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&chatter).expect("chatter imports") + }; + assert_eq!(stack_depths(&gate), (1, 0), "the prefix edit is hull-disjoint — the inverse rebased"); + + // Undo: the restore must NOT clobber B's prefix edit in the projection. + assert!(handle.apply_undo().expect("undo runs").applied); + let live = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + let canonical = { + let document = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("materializes") + }; + (0..document.paragraphs.len()) + .map(|ix| paragraph_text(&document, ix)) + .collect::>() + }; + assert_eq!( + live, canonical, + "post-rebase undo leaves live == canonical (stale content patches must downgrade to derive)" + ); + assert!( + live.iter().any(|text| text.contains("REMOTE")), + "the remote prefix edit survives the undo replay" + ); +} + +/// Bidirectionally seed `peer_gate` with `gate`'s history (both end holding +/// both sentinels) — the standard two-peer test opening. +fn exchange_full_histories(gate: &Arc>, peer_gate: &Arc>) { + let local_history = { + let g = gate.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + { + let mut g = peer_gate.lock(GateHolder::ImportChunk).expect("peer gate"); + g.import_remote_update(&local_history) + .expect("peer imports local history") + }; + let peer_history = { + let g = peer_gate.lock(GateHolder::Test).expect("peer gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export") + }; + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&peer_history) + .expect("local imports peer seed") + }; +} + +fn gate_state_vv(gate: &Arc>) -> loro::VersionVector { + let g = gate.lock(GateHolder::Test).expect("gate"); + g.doc().state_vv() +} + +/// Export `from`'s ops since `vv` and import them into `to`. +fn sync_delta(from: &Arc>, to: &Arc>, vv: &loro::VersionVector) { + let delta = { + let g = from.lock(GateHolder::Test).expect("gate"); + g.export_updates_for(vv).expect("delta export") + }; + let mut g = to.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&delta).expect("delta imports"); +} + +/// The live-projection audit: text AND styles must equal an independent full +/// materialization (the styles half is what makes the mark-rebase tests bite). +fn assert_live_equals_canonical(handle: &LocalDocHandle, gate: &Arc>, context: &str) { + let (live_text, live_styles) = { + let projection = handle.projection().expect("projection"); + ( + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>(), + projection + .paragraphs + .iter() + .map(|p| p.style) + .collect::>(), + ) + }; + let (canonical_text, canonical_styles) = { + let document = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("materializes") + }; + ( + (0..document.paragraphs.len()) + .map(|ix| paragraph_text(&document, ix)) + .collect::>(), + document + .paragraphs + .iter() + .map(|p| p.style) + .collect::>(), + ) + }; + assert_eq!(live_text, canonical_text, "{context}: live text == canonical"); + assert_eq!(live_styles, canonical_styles, "{context}: live styles == canonical"); +} + +/// §oom-leads #9 v2 (select-all rebase): a mass restyle's recorded inverse is +/// MARK-ONLY — its replay is position-neutral — so remote chatter INSIDE the +/// restyled span rebases per-coordinate instead of clearing. (v1 demanded the +/// chatter sit strictly before the recorded hull, which a select-all pins to +/// the doc start: any chatter anywhere killed the fast path — the bench's +/// UNDO-AFTER-REMOTE slow-path cliff.) Covers both stacks: a second chatter +/// burst lands while the entry sits on the REDO side. +#[test] +fn select_all_restyle_rebases_per_coordinate_through_remote_chatter() { + let (handle, gate) = new_handle("select-all-rebase"); + seed_mass_fragment(&handle, 24, None); + let (peer, peer_gate) = new_handle("select-all-rebase-peer"); + exchange_full_histories(&gate, &peer_gate); + + let styles_before: Vec = handle + .projection() + .expect("projection") + .paragraphs + .iter() + .map(|p| p.style) + .collect(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: paragraph_ids(&handle), + style: ParagraphStyle::Custom(4), + }) + .expect("select-all restyle"); + assert_eq!(stack_depths(&gate), (1, 0), "the restyle armed a recorded inverse"); + + // B types in the MIDDLE of the restyled span — inside the v1 hull. + let vv = gate_state_vv(&peer_gate); + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&peer)[12], 0), + text: "partner typed inside ".into(), + style_override: None, + }) + .expect("peer chatter inside the span"); + sync_delta(&peer_gate, &gate, &vv); + assert_eq!( + stack_depths(&gate), + (1, 0), + "chatter inside the restyled span REBASED the mark entry per-coordinate" + ); + + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "select-all undo stayed FAST across the rebased import"); + let after_undo = handle.projection().expect("projection"); + let styles: Vec = after_undo.paragraphs.iter().map(|p| p.style).collect(); + assert_eq!(styles, styles_before, "undo restored every paragraph's prior style"); + let text = (0..after_undo.paragraphs.len()) + .map(|ix| paragraph_text(&after_undo, ix)) + .collect::>(); + assert!( + text.iter().any(|t| t.contains("partner typed inside ")), + "the partner's chatter survives the undo" + ); + assert_live_equals_canonical(&handle, &gate, "select-all undo after mid-span chatter"); + + // Another chatter burst while the entry sits on the REDO stack. + let vv = gate_state_vv(&peer_gate); + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&peer)[3], 2), + text: "more partner text ".into(), + style_override: None, + }) + .expect("peer chatter while entry is on the redo stack"); + sync_delta(&peer_gate, &gate, &vv); + assert_eq!(stack_depths(&gate), (0, 1), "redo-stack mark entries rebase per-coordinate too"); + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "select-all redo stayed FAST"); + let after_redo = handle.projection().expect("projection"); + assert!( + after_redo + .paragraphs + .iter() + .all(|p| p.style == ParagraphStyle::Custom(4)), + "redo re-applied the restyle to every paragraph" + ); + assert_live_equals_canonical(&handle, &gate, "select-all redo after second chatter"); +} + +/// §oom-leads #9 v2: a remote delete that REMOVES a targeted boundary (merges +/// two paragraphs) drops that TARGET, not the whole entry — the native slow +/// path's remote-diff transform would exclude the dead boundary the same way. +#[test] +fn select_all_rebase_drops_targets_whose_boundary_the_remote_deleted() { + let (handle, gate) = new_handle("select-all-drop-target"); + seed_mass_fragment(&handle, 24, None); + let (peer, peer_gate) = new_handle("select-all-drop-target-peer"); + exchange_full_histories(&gate, &peer_gate); + + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: paragraph_ids(&handle), + style: ParagraphStyle::Custom(4), + }) + .expect("select-all restyle"); + assert_eq!(stack_depths(&gate), (1, 0), "the restyle armed a recorded inverse"); + + // B deletes across the paragraph 9/10 boundary — that boundary char dies. + let vv = gate_state_vv(&peer_gate); + let peer_ids = paragraph_ids(&peer); + peer + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(peer_ids[9], 4), + end: TextAnchor::new(peer_ids[10], 2), + }) + .expect("peer merges two paragraphs"); + sync_delta(&peer_gate, &gate, &vv); + assert_eq!( + stack_depths(&gate), + (1, 0), + "the mark entry survived with the dead boundary's target dropped" + ); + + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "undo stayed FAST despite the dropped target"); + let after_undo = handle.projection().expect("projection"); + assert!( + after_undo + .paragraphs + .iter() + .all(|p| p.style == ParagraphStyle::Normal), + "undo reverted every surviving paragraph (the merged one rides its surviving boundary)" + ); + assert_live_equals_canonical(&handle, &gate, "select-all undo after remote boundary delete"); + + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "redo stayed FAST"); + let after_redo = handle.projection().expect("projection"); + assert!( + after_redo + .paragraphs + .iter() + .all(|p| p.style == ParagraphStyle::Custom(4)), + "redo re-restyled every surviving paragraph" + ); + assert_live_equals_canonical(&handle, &gate, "select-all redo after remote boundary delete"); +} + +/// §oom-leads #9 v2 (suffix truncation + the slow-path handoff): remote chatter +/// AFTER a deep keystroke's hull truncates the keystroke entry, but the +/// mark-only select-all entry ABOVE it survives per-coordinate. Undo #1 replays +/// fast; undo #2 finds the recorded stack dry and takes the native slow path, +/// which must transform through the buffered remote diff correctly (the vendor +/// patch #22 window: the fast step's inverse commit already transformed the +/// buffers, native-style). +#[test] +fn mark_top_survives_while_deeper_keystroke_truncates_then_slow_path_converges() { + let (handle, gate) = new_handle("mark-top-truncate"); + seed_mass_fragment(&handle, 24, None); + let (peer, peer_gate) = new_handle("mark-top-truncate-peer"); + exchange_full_histories(&gate, &peer_gate); + + // A types at paragraph 8 (captured content entry, hull deep in the doc)... + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&handle)[8], 4), + text: "X".into(), + style_override: None, + }) + .expect("local keystroke commits"); + // ...then select-all restyles (mark entry on top). + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: paragraph_ids(&handle), + style: ParagraphStyle::Custom(6), + }) + .expect("select-all restyle"); + assert_eq!(stack_depths(&gate), (2, 0), "keystroke + restyle both recorded"); + + // B types at paragraph 20 — AFTER the keystroke's hull, inside the mark span. + let vv = gate_state_vv(&peer_gate); + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&peer)[20], 0), + text: "partner deep chatter ".into(), + style_override: None, + }) + .expect("peer chatter after the keystroke hull"); + sync_delta(&peer_gate, &gate, &vv); + assert_eq!( + stack_depths(&gate), + (1, 0), + "the mark top survived per-coordinate; the keystroke entry (chatter after its hull) truncated" + ); + + // Undo #1: the mark entry, fast. + assert!(handle.apply_undo().expect("undo #1 runs").applied); + let after_first_undo = handle.projection().expect("projection"); + assert!( + after_first_undo + .paragraphs + .iter() + .all(|p| p.style == ParagraphStyle::Normal), + "undo #1 reverted the restyle" + ); + // Undo #2: recorded stack dry — the native slow path pops the keystroke item + // and must land exactly on the keystroke (partner text intact). + assert!(handle.apply_undo().expect("undo #2 runs").applied); + let live = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + assert!(!live.iter().any(|text| text.contains('X')), "the slow-path undo removed the keystroke"); + assert!( + live + .iter() + .any(|text| text.contains("partner deep chatter ")), + "the partner's chatter survives both undos" + ); + assert_live_equals_canonical(&handle, &gate, "fast mark undo then slow keystroke undo"); +} + +/// §oom-leads #9 v2 (structural rebase): a remote paragraph SPLIT strictly +/// before a captured keystroke's hull shifts the recorded coordinates like any +/// other insert — the entry survives (patches downgrade to the derive ladder; +/// the projection row set changed) instead of clearing. +#[test] +fn keystroke_rebase_survives_remote_structural_insert_before_hull() { + let (handle, gate) = new_handle("keystroke-structural-rebase"); + seed_mass_fragment(&handle, 24, None); + let (peer, peer_gate) = new_handle("keystroke-structural-rebase-peer"); + exchange_full_histories(&gate, &peer_gate); + + let paragraph_count_before = handle.projection().expect("projection").paragraphs.len(); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&handle)[16], 4), + text: "X".into(), + style_override: None, + }) + .expect("local keystroke commits"); + assert_eq!(stack_depths(&gate), (1, 0), "the keystroke armed a recorded inverse"); + + // B splits paragraph 2 — a structural `\n` insert strictly before the hull. + let vv = gate_state_vv(&peer_gate); + peer + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph_ids(&peer)[2], 3), + inherited_style: ParagraphStyle::Normal, + }) + .expect("peer splits a paragraph above the hull"); + sync_delta(&peer_gate, &gate, &vv); + assert_eq!( + stack_depths(&gate), + (1, 0), + "structural chatter above the hull REBASED the keystroke entry (derive), not cleared" + ); + + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "keystroke undo stayed FAST across the structural rebase"); + let after_undo = handle.projection().expect("projection"); + assert_eq!( + after_undo.paragraphs.len(), + paragraph_count_before + 1, + "the remote split survives the undo" + ); + let undo_text = (0..after_undo.paragraphs.len()) + .map(|ix| paragraph_text(&after_undo, ix)) + .collect::>(); + assert!(!undo_text.iter().any(|t| t.contains('X')), "the undone keystroke is gone"); + assert_live_equals_canonical(&handle, &gate, "keystroke undo after remote structural insert"); + + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "keystroke redo stayed FAST"); + let after_redo = handle.projection().expect("projection"); + let redo_text = (0..after_redo.paragraphs.len()) + .map(|ix| paragraph_text(&after_redo, ix)) + .collect::>(); + assert!( + redo_text.iter().any(|t| t.contains('X')), + "redo re-applied the keystroke at the shifted position" + ); + assert_live_equals_canonical(&handle, &gate, "keystroke redo after remote structural insert"); +} + +/// §oom-leads #9 v2 (the at-coordinate trap stays closed): a remote insert +/// EXACTLY AT a recorded coordinate is ambiguous — the CRDT parks remote edits +/// whose neighbors were locally touched at that spot — so the entry must CLEAR +/// (slow path), never shift-and-replay (which would delete the partner's char). +#[test] +fn remote_insert_at_recorded_coordinate_clears_instead_of_rebasing() { + let (handle, gate) = new_handle("keystroke-at-coordinate"); + seed_mass_fragment(&handle, 24, None); + let (peer, peer_gate) = new_handle("keystroke-at-coordinate-peer"); + exchange_full_histories(&gate, &peer_gate); + + let vv_peer_baseline = gate_state_vv(&gate); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&handle)[16], 4), + text: "X".into(), + style_override: None, + }) + .expect("local keystroke commits"); + assert_eq!(stack_depths(&gate), (1, 0), "the keystroke armed a recorded inverse"); + + // B sees the keystroke, then types at the SAME anchor — its char lands + // exactly at the recorded delete coordinate. + sync_delta(&gate, &peer_gate, &vv_peer_baseline); + let vv = gate_state_vv(&peer_gate); + peer + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph_ids(&peer)[16], 4), + text: "Y".into(), + style_override: None, + }) + .expect("peer types at the recorded coordinate"); + sync_delta(&peer_gate, &gate, &vv); + assert_eq!( + stack_depths(&gate), + (0, 0), + "an insert AT the recorded coordinate cleared the entry (ambiguous parking spot)" + ); + + // The slow path still serves the undo, removing exactly the local 'X'. + assert!(handle.apply_undo().expect("undo runs").applied); + let live = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + assert!(!live.iter().any(|text| text.contains('X')), "the undone keystroke is gone"); + assert!(live.iter().any(|text| text.contains('Y')), "the partner's at-coordinate char survives"); + assert_live_equals_canonical(&handle, &gate, "slow-path undo after at-coordinate chatter"); +} + +/// §act-five P3-deep convergence: a restyle→undo→redo run must converge with a +/// peer after a BIDIRECTIONAL exchange. (An earlier one-way variant "diverged" +/// only because the fresh peer seeds its OWN sentinel paragraph — `new_empty` +/// mints ~84 init ops incl a `\n` boundary — so a one-way import leaves the peer +/// with an extra sentinel; the real bar is convergence after both import each +/// other, per the existing round-trip tests.) +#[test] +fn recorded_inverse_stack_restyle_undo_redo_converges_bidirectionally() { + let (handle, gate) = new_handle("chain-converge"); + seed_mass_fragment(&handle, 12, None); + let ids: Vec = paragraph_ids(&handle).into_iter().skip(1).collect(); + for slot in [1_u8, 2, 3] { + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids.clone(), + style: ParagraphStyle::Custom(slot), + }) + .expect("restyle"); + } + for _ in 0..3 { + handle.apply_undo().expect("undo"); + } + for _ in 0..3 { + handle.apply_redo().expect("redo"); + } + let styles = |doc: &loro::LoroDoc| -> Vec { + flowstate_document::document_from_loro(doc) + .expect("mat") + .paragraphs + .iter() + .map(|p| p.style) + .collect() + }; + // Bidirectional exchange: peer takes local's whole history, local takes the + // peer's seed back — now both hold both sentinels and must converge. + let local_history = { + let g = gate.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + let mut peer = CrdtRuntime::new_empty("chain-peer").expect("peer"); + peer + .import_remote_update(&local_history) + .expect("peer imports local"); + let peer_seed = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + { + let mut g = gate.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&peer_seed) + .expect("local imports peer seed") + }; + let local_after = { + let g = gate.lock(GateHolder::Test).expect("gate"); + styles(g.doc()) + }; + assert_eq!( + styles(peer.doc()), + local_after, + "restyle→undo→redo converges after a bidirectional exchange" + ); +} + +/// §act-five P1.B: a peer importing a LONG run of sequential remote updates +/// (each advancing the RETAINED `import_diff_calculator`) must converge to the +/// sender. Exercises the reuse path repeatedly — a bug in the retained-tracker +/// advance would surface as body divergence here (and the retained calc uses +/// Loro's general Checkout apply, which must stay bit-correct across imports). +#[test] +fn retained_import_calculator_converges_over_many_sequential_imports() { + let (handle_a, gate_a) = new_handle("p1b-a"); + let a_init = { + let g = gate_a.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export A init") + }; + let mut peer = CrdtRuntime::new_empty("p1b-b").expect("peer B"); + peer + .import_remote_update(&a_init) + .expect("B imports A init"); + + let first = first_paragraph(&handle_a.projection().expect("projection")); + for i in 0..24 { + handle_a + .insert_text(InsertTextIntent { + at: TextAnchor::new(first, 0), + text: format!("x{i} "), + style_override: None, + }) + .expect("A edit commits"); + // B imports A's whole history each round; only the new op applies, exercising + // the retained calculator once more (idempotent for already-present ops). + let full = { + let g = gate_a.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export A") + }; + peer.import_remote_update(&full).expect("B imports A"); + } + // Bidirectional close: A takes B's seed back so both hold both sentinels; then + // the two canonical bodies must be identical (the real convergence bar). + let b_seed = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export B"); + { + let mut g = gate_a.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&b_seed).expect("A imports B seed") + }; + let a_body = { + let g = gate_a.lock(GateHolder::Test).expect("gate"); + flowstate_document::loro_schema::body_text(g.doc()).to_string() + }; + let b_body = flowstate_document::loro_schema::body_text(peer.doc()).to_string(); + assert_eq!(b_body, a_body, "peer converges after 24 sequential retained-calculator imports"); +} + +/// §oom-leads #9 P1.C (vendor patch #21, meet-clamped diff feed): the RECEIVER +/// keeps making local edits between sequential imports from a peer that never +/// syncs back — the bench's REMOTE-STRUCT-BARE shape and the deep-LCA +/// pathology's trigger. The retained import calculator is never fed local +/// commits, so its tracker has a coverage GAP each round; the clamp must +/// re-feed exactly that gap (a clamped-away hole corrupts the import diff's +/// retain offsets and diverges the body). +#[test] +fn meet_clamped_import_feed_converges_with_interleaved_local_edits() { + let (handle_a, gate_a) = new_handle("p21-clamp-a"); + seed_mass_fragment(&handle_a, 8, None); + let (handle_b, gate_b) = new_handle("p21-clamp-b"); + let a_init = { + let g = gate_a.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export A init") + }; + { + let mut g = gate_b.lock(GateHolder::ImportChunk).expect("B gate"); + g.import_remote_update(&a_init).expect("B imports A init") + }; + let b_init = { + let g = gate_b.lock(GateHolder::Test).expect("B gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export B init") + }; + { + let mut g = gate_a.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&b_init).expect("A imports B init") + }; + + let mut b_exported_vv = { + let g = gate_b.lock(GateHolder::Test).expect("B gate"); + g.doc().state_vv() + }; + for i in 0..16 { + // B edits its own copy — a growing chain A must import; B NEVER imports + // A's edits, so every delta is maximally concurrent with A's local work + // (the deep-dominator shape). + let b_target = paragraph_ids(&handle_b)[1]; + handle_b + .insert_text(InsertTextIntent { + at: TextAnchor::new(b_target, 0), + text: format!("b{i} "), + style_override: None, + }) + .expect("B edit commits"); + // A makes a LOCAL edit — the retained import calculator's coverage gap. + let a_target = paragraph_ids(&handle_a)[3]; + handle_a + .insert_text(InsertTextIntent { + at: TextAnchor::new(a_target, 0), + text: format!("a{i} "), + style_override: None, + }) + .expect("A edit commits"); + let delta = { + let g = gate_b.lock(GateHolder::Test).expect("B gate"); + let delta = g + .export_updates_for(&b_exported_vv) + .expect("B delta export"); + b_exported_vv = g.doc().state_vv(); + delta + }; + let mut g = gate_a.lock(GateHolder::ImportChunk).expect("gate"); + g.import_remote_update(&delta).expect("A imports B delta"); + } + + // Close the loop: B takes A's full history; both canonical bodies must match. + let a_full = { + let g = gate_a.lock(GateHolder::Test).expect("gate"); + g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export A full") + }; + { + let mut g = gate_b.lock(GateHolder::ImportChunk).expect("B gate"); + g.import_remote_update(&a_full).expect("B imports A full") + }; + let a_body = { + let g = gate_a.lock(GateHolder::Test).expect("gate"); + flowstate_document::loro_schema::body_text(g.doc()).to_string() + }; + let b_body = { + let g = gate_b.lock(GateHolder::Test).expect("B gate"); + flowstate_document::loro_schema::body_text(g.doc()).to_string() + }; + assert_eq!( + b_body, a_body, + "interleaved local edits + one-way sequential imports converge (gap re-feed is complete)" + ); + // And A's LIVE projection must equal its canonical materialization — a + // clamped-away tracker hole shows up as a projection/doc offset mismatch. + let live = { + let projection = handle_a.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + let canonical = { + let document = { + let guard = gate_a.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("materializes") + }; + (0..document.paragraphs.len()) + .map(|ix| paragraph_text(&document, ix)) + .collect::>() + }; + assert_eq!(live, canonical, "A's live projection == canonical after clamped imports"); +} + +fn slot_direction(gate: &Arc>) -> Option { + // §act-five P3-deep: the fast path is now two stacks. A pending REDO (top of + // the redo stack) means the last fast step was an undo; else a pending UNDO. + let mut guard = gate.lock(GateHolder::Test).expect("gate healthy"); + if !guard.recorded_redo_stack().is_empty() { + Some(loro::UndoOrRedo::Redo) + } else if !guard.recorded_undo_stack().is_empty() { + Some(loro::UndoOrRedo::Undo) + } else { + None + } +} + +fn paragraph_ids(handle: &LocalDocHandle) -> Vec { + handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .to_vec() +} + +fn block_ids(handle: &LocalDocHandle) -> Vec { + handle + .projection() + .expect("projection") + .ids + .block_ids + .to_vec() +} + +/// The core round trip: a qualifying mass delete arms the slot; undo replays +/// the recorded inverse (no checkout), restoring text, marks, and the ORIGINAL +/// paragraph/block identities; redo replays the delete; ping-pong stays fast. +/// The debug-build audit inside every step compares the patched projection +/// against a full rematerialization. +#[test] +fn recorded_inverse_fast_undo_round_trips_mass_delete() { + let (handle, gate) = new_handle("recorded-inverse"); + seed_mass_fragment(&handle, 40, None); + + let body_before = body_string(&gate); + let paragraph_ids_before = paragraph_ids(&handle); + let block_ids_before = block_ids(&handle); + + // Cross-paragraph mass delete: from inside paragraph 2 to inside paragraph 37. + let start = paragraph_ids_before[2]; + let end = paragraph_ids_before[37]; + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(start, 9), + end: TextAnchor::new(end, 12), + }) + .expect("mass delete commits"); + let body_deleted = body_string(&gate); + assert!( + body_deleted.len() + 2048 < body_before.len(), + "delete must remove a qualifying mass range" + ); + assert_eq!( + slot_direction(&gate), + Some(loro::UndoOrRedo::Undo), + "mass delete must arm the recorded inverse" + ); + + // Undo: fast path (flips the slot; the slow path never touches it). + let outcome = handle.apply_undo().expect("undo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), body_before, "undo must restore the exact body text + boundaries"); + assert_eq!( + slot_direction(&gate), + Some(loro::UndoOrRedo::Redo), + "fast undo must flip the slot to redo" + ); + assert_eq!( + paragraph_ids(&handle), + paragraph_ids_before, + "undo must restore ORIGINAL paragraph identities" + ); + assert_eq!(block_ids(&handle), block_ids_before, "undo must restore ORIGINAL block identities"); + + // Redo: fast replay of the recorded delete. + let outcome = handle.apply_redo().expect("redo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), body_deleted, "redo must reproduce the deleted state"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Undo), "fast redo must flip the slot back"); + + // Ping-pong once more. + let outcome = handle.apply_undo().expect("second undo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), body_before); + assert_eq!(paragraph_ids(&handle), paragraph_ids_before); + + // Convergence bar: a fresh peer importing the full history (delete + fast + // inverse + fast redelete + fast inverse) must agree byte-for-byte. + let mut peer = CrdtRuntime::new_empty("recorded-inverse").expect("peer"); + let full_history = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export full updates") + }; + peer + .import_remote_update(&full_history) + .expect("peer imports"); + let peer_text = flowstate_document::loro_schema::body_text(peer.doc()).to_string(); + let peer_history = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_update(&peer_history) + .expect("local imports peer seed"); + drop(guard); + assert_eq!(body_string(&gate), peer_text, "replicas must converge on the fast-path history"); +} + +/// Object rows (equation) inside the deleted range restore with their ORIGINAL +/// block ids and content. +#[test] +fn recorded_inverse_restores_object_blocks_with_original_ids() { + let (handle, gate) = new_handle("recorded-inverse-objects"); + seed_mass_fragment(&handle, 36, Some(18)); + + let body_before = body_string(&gate); + let block_ids_before = block_ids(&handle); + let ids = paragraph_ids(&handle); + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(ids[3], 4), + end: TextAnchor::new(ids[33], 6), + }) + .expect("mass delete across the equation commits"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Undo), "object range must still qualify"); + + let outcome = handle.apply_undo().expect("undo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), body_before); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Redo), "fast path must have run"); + assert_eq!(block_ids(&handle), block_ids_before, "equation block id must be restored, not re-minted"); + let projection = handle.projection().expect("projection"); + let equation = projection + .blocks + .iter() + .find_map(|block| match block { + flowstate_document::Block::Equation(equation) => Some(equation.source.to_string()), + _ => None, + }) + .expect("equation row restored"); + assert_eq!(equation, "e = mc^2"); +} + +/// A remote import whose change lands INSIDE the recorded hull kills the fast +/// path (stacks cleared) — undo still works through the checkout-based slow +/// path and keeps BOTH the restored content and the remote edit. (§oom-leads +/// #9: a hull-DISJOINT import now rebases instead — see +/// `recorded_inverse_rebases_across_hull_disjoint_remote_import`.) +#[test] +fn recorded_inverse_declines_after_remote_import() { + let (handle, gate) = new_handle("recorded-inverse-import"); + seed_mass_fragment(&handle, 40, None); + let ids = paragraph_ids(&handle); + + // Converged peer first. + let mut peer = CrdtRuntime::new_empty("recorded-inverse-import").expect("peer"); + let seed = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer.import_remote_update(&seed).expect("peer imports seed"); + let peer_seed = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_update(&peer_seed) + .expect("local imports peer seed"); + drop(guard); + let peer_vv = peer.doc().state_vv(); + + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(ids[2], 0), + end: TextAnchor::new(ids[38], 3), + }) + .expect("mass delete commits"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Undo)); + + // Remote edit lands between delete and undo, INSIDE the deleted range's + // hull (the delete spans paragraphs 2..38 — body position 500 is well + // within it), so the rebase is unsound and the stacks must clear. + let peer_body = flowstate_document::loro_schema::body_text(peer.doc()); + peer_body.insert(500, "REMOTE").expect("peer insert"); + peer.doc().commit(); + let update = peer + .doc() + .export(loro::ExportMode::updates(&peer_vv)) + .expect("export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard.import_remote_update(&update).expect("import applies"); + drop(guard); + assert_eq!(slot_direction(&gate), None, "an in-hull import must clear the recorded inverse"); + + let outcome = handle + .apply_undo() + .expect("undo still runs via the slow path"); + assert!(outcome.applied); + let body = body_string(&gate); + assert!(body.contains("REMOTE"), "slow-path undo must keep the remote edit"); + assert!(body.contains("paragraph 020"), "slow-path undo must restore the deleted content"); +} + +/// Small or same-paragraph deletes never arm the slot; a table row inside the +/// range gates the capture out (durable table ids cannot be re-minted). +#[test] +fn recorded_inverse_gating() { + let (handle, gate) = new_handle("recorded-inverse-gating"); + seed_mass_fragment(&handle, 8, None); + let ids = paragraph_ids(&handle); + + // Small cross-paragraph delete: below the capture threshold. + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(ids[1], 0), + end: TextAnchor::new(ids[2], 4), + }) + .expect("small delete commits"); + assert_eq!(slot_direction(&gate), None, "sub-threshold deletes must not arm the slot"); + + // Table inside the range: gated out. + let (handle, gate) = new_handle("recorded-inverse-table-gate"); + seed_mass_fragment(&handle, 40, None); + let projection = handle.projection().expect("projection"); + let anchor = projection.ids.paragraph_ids[20]; + let row_id = gpui_flowtext::RowId(1); + let column_id = gpui_flowtext::ColumnId(2); + handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(anchor, 0), + blocks: vec![FragmentBlock::Object(flowstate_document::InputBlock::Table( + flowstate_document::InputTableBlock { + rows: vec![flowstate_document::InputTableRow { + id: row_id, + cells: vec![flowstate_document::InputTableCell { + id: gpui_flowtext::CellId(3), + row_id, + column_id, + blocks: Vec::new(), + row_span: 1, + col_span: 1, + }], + }], + columns: vec![flowstate_document::InputTableColumn { + id: column_id, + width: flowstate_document::InputTableColumnWidth::Auto, + }], + style: flowstate_document::InputTableStyle { header_row: false }, + }, + ))], + }) + .expect("table inserts"); + let ids = paragraph_ids(&handle); + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(ids[2], 0), + end: TextAnchor::new(ids[38], 3), + }) + .expect("mass delete across the table commits"); + // The one guarantee B.1 owns here: a durable-id table in range gates the + // recorded-inverse capture OUT (its row/column/cell ids cannot be re-minted + // losslessly), so undo takes the checkout-based slow path. (Whether that + // slow path perfectly restores an object placeholder deleted by a + // cross-paragraph range is a separate, pre-existing Loro-undo concern — + // demonstrated by `slow_undo_drops_object_placeholder_pre_existing` — that + // this fast path is not responsible for, and in fact improves upon.) + assert_eq!(slot_direction(&gate), None, "a table in range must gate the capture out"); + let outcome = handle.apply_undo().expect("slow undo runs"); + assert!(outcome.applied, "slow-path undo of the gated delete still applies"); +} + +/// Characterization of a PRE-EXISTING slow-path (checkout-based Loro undo) +/// limitation, kept as a guardrail: undoing a cross-paragraph delete that +/// removed an object placeholder restores all the TEXT but DROPS the U+FFFC +/// object placeholder (the block record then projects as an unresolved anchor +/// and is repaired away). This is independent of B.1 — the recorded-inverse +/// fast path (`recorded_inverse_restores_object_blocks_with_original_ids`) +/// restores the object correctly, so it is strictly more faithful than the +/// slow path it replaces for qualifying deletes. If a Loro upgrade ever fixes +/// the underlying checkout-undo behavior, this test will flip and flag it. +#[test] +fn slow_undo_drops_object_placeholder_pre_existing() { + let (handle, gate) = new_handle("slow-undo-object"); + seed_mass_fragment(&handle, 4, Some(2)); + let ids = paragraph_ids(&handle); + assert_eq!( + body_string(&gate) + .chars() + .filter(|ch| *ch == '\u{FFFC}') + .count(), + 1 + ); + + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(ids[1], 0), + end: TextAnchor::new(ids[3], 4), + }) + .expect("small delete across the equation commits"); + assert_eq!(slot_direction(&gate), None, "sub-threshold: no recorded-inverse capture"); + assert_eq!( + body_string(&gate) + .chars() + .filter(|ch| *ch == '\u{FFFC}') + .count(), + 0 + ); + + let outcome = handle.apply_undo().expect("slow undo runs"); + assert!(outcome.applied); + // Pre-existing: the placeholder is NOT restored by the checkout-based undo. + assert_eq!( + body_string(&gate) + .chars() + .filter(|ch| *ch == '\u{FFFC}') + .count(), + 0, + "documents the pre-existing slow-path object-placeholder drop (see doc comment)" + ); +} + +/// §act-four M1: a mass select-all restyle records its inverse; undo reverts +/// every paragraph's style, redo re-applies — via the fast path (no checkout), +/// with the debug audit inside each step verifying patch-vs-full-rebuild. +#[test] +fn recorded_inverse_fast_undo_round_trips_mass_restyle() { + let (handle, gate) = new_handle("recorded-inverse-restyle"); + seed_mass_fragment(&handle, 40, None); + let ids = paragraph_ids(&handle); + + let styles_of = |handle: &LocalDocHandle| -> Vec { + handle + .projection() + .expect("projection") + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + let before = styles_of(&handle); + assert!( + before.iter().all(|style| *style == ParagraphStyle::Normal), + "seed paragraphs start Normal" + ); + + // Select-all restyle to a custom style. + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids.clone(), + style: ParagraphStyle::Custom(3), + }) + .expect("mass restyle commits"); + let after = styles_of(&handle); + assert!( + after + .iter() + .all(|style| *style == ParagraphStyle::Custom(3)), + "restyle sets every paragraph" + ); + assert_eq!( + slot_direction(&gate), + Some(loro::UndoOrRedo::Undo), + "mass restyle must arm the recorded inverse" + ); + + // Undo: fast path reverts every style. + let outcome = handle.apply_undo().expect("undo runs"); + assert!(outcome.applied); + assert_eq!(styles_of(&handle), before, "undo must restore every paragraph's prior style"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Redo), "fast undo flips the slot to redo"); + + // Redo: fast path re-applies. + let outcome = handle.apply_redo().expect("redo runs"); + assert!(outcome.applied); + assert_eq!(styles_of(&handle), after, "redo must re-apply the restyle"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Undo), "fast redo flips the slot back"); + + // Ping-pong once more + convergence: a fresh peer receives the full history + // (restyle + fast inverse + fast redo + fast inverse), and after a + // BIDIRECTIONAL exchange both replicas materialize the same converged + // document (each seeds its own sentinel, so convergence — not identity with a + // pre-sync snapshot — is the bar). + handle.apply_undo().expect("second undo runs"); + assert_eq!(styles_of(&handle), before); + let mut peer = CrdtRuntime::new_empty("recorded-inverse-restyle").expect("peer"); + let full_history = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer + .import_remote_update(&full_history) + .expect("peer imports local history"); + let peer_history = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_update(&peer_history) + .expect("local imports peer seed"); + drop(guard); + let styles = |doc: &DocumentProjection| -> Vec { + doc + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect() + }; + let local_fresh = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("local materializes") + }; + let peer_fresh = flowstate_document::document_from_loro(peer.doc()).expect("peer materializes"); + assert_eq!( + styles(&peer_fresh), + styles(&local_fresh), + "replicas converge on the same styles after the fast-path history" + ); + assert!( + styles(&local_fresh) + .iter() + .all(|style| *style == ParagraphStyle::Normal), + "converged on the undone (Normal) styles" + ); +} + +/// §fidelity (field bug 2026-07-08, two-peer hotpath): a LOCAL undo must revert +/// ONLY the local peer's op — NEVER a concurrent REMOTE op. Repro from the field: +/// peer A highlights the whole body (a run-style mark), peer B concurrently tags +/// every paragraph (a paragraph style); A imports B's tag, then A Ctrl-Z's its +/// highlight. The observed bug: BOTH the highlight AND the remote tag reverted. +/// +/// This asserts the invariant against the LIVE maintained projection (what the UI +/// paints) AND a fresh canonical materialization — the pair localizes whether the +/// fault is the CRDT undo-exclusion or the post-undo reprojection. +#[test] +fn local_undo_of_highlight_preserves_concurrent_remote_paragraph_style() { + let (handle_a, gate_a) = new_handle("undo-isolation-a"); + seed_mass_fragment(&handle_a, 8, None); + + // Peer B starts from A's exact initial state (shared paragraph ids), then edits + // concurrently (neither peer has seen the other's edit). + let a_initial = { + let guard = gate_a.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export A initial") + }; + let mut core_b = CrdtRuntime::new_empty("undo-isolation-b").expect("peer B runtime"); + core_b + .import_remote_update(&a_initial) + .expect("B imports A initial state"); + let (handle_b, gate_b) = LocalDocHandle::new(core_b, LocalWriteConfig::default()); + + let ids = paragraph_ids(&handle_a); + let tag = ParagraphStyle::Custom(7); + + // Peer B: tag every paragraph (LOCAL to B). + handle_b + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids.clone(), + style: tag, + }) + .expect("B tags every paragraph"); + + // Peer A: highlight the whole body (LOCAL to A) — a run-style mark. + let projection_a = handle_a.projection().expect("projection A"); + let first = projection_a.ids.paragraph_ids[0]; + let last = *projection_a + .ids + .paragraph_ids + .last() + .expect("last paragraph id"); + let last_len = paragraph_text(&projection_a, projection_a.paragraphs.len() - 1).len(); + let highlight = flowstate_document::RunStyles { + highlight: Some(flowstate_document::HighlightStyle::Custom(3)), + ..Default::default() + }; + handle_a + .set_marks(SetMarksIntent { + start: TextAnchor::new(first, 0), + end: TextAnchor::new(last, last_len), + styles: highlight, + }) + .expect("A highlights the whole body"); + + // A imports B's concurrent tag (origin "remote" ⇒ excluded from A's undo stack). + let b_update = { + let guard = gate_b.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export B") + }; + { + let mut guard = gate_a.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_updates(&[&b_update]) + .expect("A imports B's tag") + }; + + // Read (styles, any-highlight) from the LIVE projection and from a fresh + // canonical materialization — divergence between them localizes the fault. + let live = |handle: &LocalDocHandle| { + let doc = handle.projection().expect("live projection"); + let styles: Vec = doc + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect(); + let any_highlight = doc.paragraphs.iter().any(|paragraph| { + paragraph + .runs + .iter() + .any(|run| run.styles.highlight.is_some()) + }); + (styles, any_highlight) + }; + let canonical = |gate: &Arc>| { + let doc = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + flowstate_document::document_from_loro(guard.doc()).expect("materialize canonical") + }; + let styles: Vec = doc + .paragraphs + .iter() + .map(|paragraph| paragraph.style) + .collect(); + let any_highlight = doc.paragraphs.iter().any(|paragraph| { + paragraph + .runs + .iter() + .any(|run| run.styles.highlight.is_some()) + }); + (styles, any_highlight) + }; + + // The remote tag defines the EXPECTED paragraph-style vector: A's local undo + // touches only its run-style highlight, so paragraph styles must be identical + // to B's tag result before undo, after undo, and after convergence. + let expected_styles = canonical(&gate_b).0; + assert!( + expected_styles.contains(&tag), + "B's tag must land on the durable paragraphs: {expected_styles:?}" + ); + + // Before undo A already reflects BOTH edits. + let (styles_before, hl_before) = live(&handle_a); + assert_eq!(styles_before, expected_styles, "A must reflect the remote tag before undo"); + assert!(hl_before, "A must see its own highlight before undo"); + + // A UNDOES its highlight. + assert!(handle_a.apply_undo().expect("A undo runs").applied, "undo must apply"); + + // THE INVARIANT: highlight gone, paragraph styles UNCHANGED — live AND canonical. + let (live_styles, live_hl) = live(&handle_a); + let (canon_styles, canon_hl) = canonical(&gate_a); + assert!(!live_hl, "undo must remove A's own highlight (live projection)"); + assert!(!canon_hl, "undo must remove A's own highlight (canonical)"); + assert_eq!( + canon_styles, expected_styles, + "CRDT undo-exclusion bug: local undo changed paragraph styles in canonical state (reverted the concurrent REMOTE tag)" + ); + assert_eq!( + live_styles, expected_styles, + "reprojection bug: post-undo LIVE projection dropped the concurrent REMOTE tag" + ); + + // Convergence: B receives A's post-undo history; both replicas agree, tag intact. + let a_after = { + let guard = gate_a.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export A after undo") + }; + { + let mut guard = gate_b.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_updates(&[&a_after]) + .expect("B imports A's post-undo history") + }; + let (styles_b, hl_b) = canonical(&gate_b); + assert!(!hl_b, "peer B converges on highlight-removed"); + assert_eq!(styles_b, expected_styles, "peer B keeps the remote tag after convergence"); +} + +/// §act-four M1: a mass replace-all records its inverse; undo restores every +/// original match, redo re-applies — via the fast path (no checkout), the +/// projection side deriving through the regional ladder. Convergence verified +/// against a peer receiving the whole fast-path history. +#[test] +fn recorded_inverse_fast_undo_round_trips_mass_replace() { + let (handle, gate) = new_handle("recorded-inverse-replace"); + seed_mass_fragment(&handle, 40, None); + + let body_before = body_string(&gate); + // Every seeded paragraph contains "row" exactly once (…"for row NNN"…). + let projection = handle.projection().expect("projection"); + let ranges = flowstate_document::find_text_ranges(&projection, "row"); + assert!(ranges.len() >= 8, "need a mass replace (got {} matches)", ranges.len()); + let matches: Vec = ranges + .iter() + .map(|range| super::intents::ReplaceMatch { + start: TextAnchor::new(projection.ids.paragraph_ids[range.start.paragraph], range.start.byte), + end: TextAnchor::new(projection.ids.paragraph_ids[range.end.paragraph], range.end.byte), + styles: None, + }) + .collect(); + + handle + .replace_matches(super::intents::ReplaceMatchesIntent { + matches, + replacement: "COLUMN".into(), + }) + .expect("mass replace commits"); + let body_replaced = body_string(&gate); + assert!(!body_replaced.contains("row"), "replace removed every 'row'"); + assert!(body_replaced.contains("COLUMN"), "replace inserted 'COLUMN'"); + assert_eq!( + slot_direction(&gate), + Some(loro::UndoOrRedo::Undo), + "mass replace must arm the recorded inverse" + ); + + // Undo: fast path restores the originals. + let outcome = handle.apply_undo().expect("undo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), body_before, "undo must restore every original match"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Redo)); + + // Redo: fast path re-applies the replacement. + let outcome = handle.apply_redo().expect("redo runs"); + assert!(outcome.applied); + assert_eq!(body_string(&gate), body_replaced, "redo must reproduce the replaced state"); + assert_eq!(slot_direction(&gate), Some(loro::UndoOrRedo::Undo)); + + // Ping-pong + bidirectional convergence. + handle.apply_undo().expect("second undo runs"); + assert_eq!(body_string(&gate), body_before); + let mut peer = CrdtRuntime::new_empty("recorded-inverse-replace").expect("peer"); + let full_history = { + let guard = gate.lock(GateHolder::Test).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer + .import_remote_update(&full_history) + .expect("peer imports local history"); + let peer_history = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard + .import_remote_update(&peer_history) + .expect("local imports peer seed"); + drop(guard); + let local_text = body_string(&gate); + let peer_text = flowstate_document::loro_schema::body_text(peer.doc()).to_string(); + assert_eq!(local_text, peer_text, "replicas converge on the replace-all fast-path history"); + assert!(!peer_text.contains("COLUMN"), "converged on the undone (original) content"); +} + +/// §act-twelve A12.2.2: Enter (`SplitParagraph`) is captured — undo merges the +/// split back (retiring the new records), redo re-splits with the SAME +/// paragraph id (identity preservation), and none of it clears deeper +/// captured entries. +#[test] +fn recorded_inverse_captures_split_paragraph() { + let (handle, gate) = new_handle("split-capture"); + seed_mass_fragment(&handle, 8, None); + let target = paragraph_ids(&handle)[3]; + let count_before = handle.projection().expect("projection").paragraphs.len(); + + let outcome = handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(target, 5), + inherited_style: ParagraphStyle::Normal, + }) + .expect("split commits"); + let _ = outcome; + assert_eq!(stack_depths(&gate), (1, 0), "the split armed a recorded inverse"); + let split_ids: Vec = handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .to_vec(); + assert_eq!(split_ids.len(), count_before + 1); + + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "split undo stayed FAST"); + let after_undo = handle.projection().expect("projection"); + assert_eq!(after_undo.paragraphs.len(), count_before, "undo merged the split back"); + assert_live_equals_canonical(&handle, &gate, "split undo"); + + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "split redo stayed FAST"); + let after_redo = handle.projection().expect("projection"); + assert_eq!( + after_redo.ids.paragraph_ids.to_vec(), + split_ids, + "redo re-created the boundary with the SAME paragraph identity" + ); + assert_live_equals_canonical(&handle, &gate, "split redo"); +} + +/// §act-twelve A12.2.2: Backspace at paragraph start (`JoinParagraphs`) is +/// captured — undo re-creates the boundary with the second paragraph's +/// ORIGINAL id and prior style; redo re-joins. +#[test] +fn recorded_inverse_captures_join_paragraphs() { + let (handle, gate) = new_handle("join-capture"); + seed_mass_fragment(&handle, 8, None); + let ids_before: Vec = handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .to_vec(); + let (first, second) = (ids_before[3], ids_before[4]); + // Give the second paragraph a distinct style so undo must restore it. + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .to_vec(), + style: ParagraphStyle::Custom(2), + }) + .expect("restyle commits"); + + handle + .join_paragraphs(crate::local_write::JoinParagraphsIntent { first, second }) + .expect("join commits"); + assert_eq!(stack_depths(&gate), (2, 0), "restyle + join both recorded"); + + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (1, 1), "join undo stayed FAST"); + let after_undo = handle.projection().expect("projection"); + assert_eq!( + after_undo.ids.paragraph_ids.to_vec(), + ids_before, + "undo restored the second paragraph's ORIGINAL id" + ); + let second_ix = after_undo + .ids + .paragraph_ids + .iter() + .position(|id| *id == second) + .expect("second restored"); + assert_eq!( + after_undo.paragraphs[second_ix].style, + ParagraphStyle::Custom(2), + "undo restored the second paragraph's prior style" + ); + assert_live_equals_canonical(&handle, &gate, "join undo"); + + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (2, 0), "join redo stayed FAST"); + assert_eq!( + handle.projection().expect("projection").paragraphs.len(), + ids_before.len() - 1, + "redo re-joined the paragraphs" + ); + assert_live_equals_canonical(&handle, &gate, "join redo"); +} + +/// §act-twelve A12.2.2: `SetMarks` (bold/italic class) is captured — undo +/// restores the EXACT prior per-span run styles (the seeded content mixes +/// underline/strikethrough runs), redo re-applies the uniform mark. +#[test] +fn recorded_inverse_captures_set_marks() { + let (handle, gate) = new_handle("marks-capture"); + seed_mass_fragment(&handle, 8, None); + let target = paragraph_ids(&handle)[3]; + let runs_before = format!("{:?}", handle.projection().expect("projection").paragraphs[3].runs); + + handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(target, 2), + end: TextAnchor::new(target, 40), + styles: flowstate_document::RunStyles { + strikethrough: true, + ..Default::default() + }, + }) + .expect("marks commit"); + assert_eq!(stack_depths(&gate), (1, 0), "the mark op armed a recorded inverse"); + let runs_marked = format!("{:?}", handle.projection().expect("projection").paragraphs[3].runs); + assert_ne!(runs_before, runs_marked, "the mark changed the runs"); + + assert!(handle.apply_undo().expect("undo runs").applied); + assert_eq!(stack_depths(&gate), (0, 1), "marks undo stayed FAST"); + assert_eq!( + format!("{:?}", handle.projection().expect("projection").paragraphs[3].runs), + runs_before, + "undo restored the exact prior run styles" + ); + assert_live_equals_canonical(&handle, &gate, "marks undo"); + + assert!(handle.apply_redo().expect("redo runs").applied); + assert_eq!(stack_depths(&gate), (1, 0), "marks redo stayed FAST"); + assert_eq!( + format!("{:?}", handle.projection().expect("projection").paragraphs[3].runs), + runs_marked, + "redo re-applied the mark" + ); + assert_live_equals_canonical(&handle, &gate, "marks redo"); +} + +/// §act-twelve A12.2.2: the typing-session shape — keystrokes, Enter, marks, +/// restyle, backspace-join chained; EVERY undo and redo must stay on the +/// fast path (the stacks prove it: each fast step moves one entry). +#[test] +fn recorded_inverse_chains_full_typing_session() { + let (handle, gate) = new_handle("session-capture"); + seed_mass_fragment(&handle, 8, None); + let target = paragraph_ids(&handle)[2]; + let original_text = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(target, 0), + text: "abc".into(), + style_override: None, + }) + .expect("keystrokes"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(target, 3), + inherited_style: ParagraphStyle::Normal, + }) + .expect("enter"); + handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(target, 0), + end: TextAnchor::new(target, 3), + styles: flowstate_document::RunStyles { + direct_underline: true, + ..Default::default() + }, + }) + .expect("bold-ish"); + handle + .set_paragraph_style(crate::local_write::SetParagraphStyleIntent { + paragraph: target, + style: ParagraphStyle::Custom(3), + }) + .expect("restyle one"); + assert_eq!(stack_depths(&gate), (4, 0), "all four ops recorded"); + + for step in 0..4 { + assert!(handle.apply_undo().expect("undo runs").applied, "undo {step} applied"); + } + assert_eq!(stack_depths(&gate), (0, 4), "ALL undos stayed on the fast path"); + let live = { + let projection = handle.projection().expect("projection"); + (0..projection.paragraphs.len()) + .map(|ix| paragraph_text(&projection, ix)) + .collect::>() + }; + assert_eq!(live, original_text, "the session fully unwound"); + assert_live_equals_canonical(&handle, &gate, "session unwind"); + + for step in 0..4 { + assert!(handle.apply_redo().expect("redo runs").applied, "redo {step} applied"); + } + assert_eq!(stack_depths(&gate), (4, 0), "ALL redos stayed on the fast path"); + assert_live_equals_canonical(&handle, &gate, "session replay"); +} diff --git a/crates/flowstate-collab/src/net/anti_entropy.rs b/crates/flowstate-collab/src/net/anti_entropy.rs new file mode 100644 index 00000000..ae40036c --- /dev/null +++ b/crates/flowstate-collab/src/net/anti_entropy.rs @@ -0,0 +1,189 @@ +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; + +use iroh::EndpointId; + +use crate::ids::SessionId; + +pub const DIGEST_INTERVAL: Duration = Duration::from_secs(10); +pub const DIGEST_JITTER_PERCENT: u32 = 20; +/// FS-077: safety net for a wedged direct pull. A per-peer in-flight pull that +/// is never finished (e.g. the runtime reply is dropped) is force-expired after +/// this deadline, so one stuck peer can never block anti-entropy pulls to every +/// other peer. +pub const PULL_IN_FLIGHT_DEADLINE: Duration = Duration::from_secs(60); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum VersionVectorRelation { + Equal, + SenderHasMissingOps, + WeHaveMissingOps, + Concurrent, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum GapAction { + None, + Pull { from: EndpointId, our_vv: Vec }, + LineageMismatch { from: EndpointId, expected: SessionId, got: SessionId }, +} + +#[derive(Clone, Debug)] +pub struct AntiEntropyState { + session: SessionId, + next_digest: Instant, + // FS-077: per-peer in-flight pulls keyed by peer, valued by an expiry + // deadline. Replaces a single global `pull_in_flight` flag so one wedged peer + // cannot starve pulls to the rest of the swarm, while still deduplicating a + // second pull for the same peer while one is outstanding. + pulls_in_flight: HashMap, +} + +impl AntiEntropyState { + #[must_use] + pub fn new(session: SessionId, now: Instant) -> Self { + Self { + session, + next_digest: next_jittered_deadline(now), + pulls_in_flight: HashMap::new(), + } + } + + #[must_use] + pub const fn session(&self) -> SessionId { + self.session + } + + #[must_use] + pub fn digest_due(&self, now: Instant) -> bool { + now >= self.next_digest + } + + #[must_use] + pub fn duration_until_digest(&self, now: Instant) -> Duration { + self.next_digest.saturating_duration_since(now) + } + + pub fn mark_digest_sent(&mut self, now: Instant) { + self.next_digest = next_jittered_deadline(now); + tracing::trace!(session = %self.session, next_digest_in = ?self.next_digest.saturating_duration_since(now), "collaboration anti-entropy digest sent"); + } + + pub fn schedule_immediate_digest(&mut self) { + self.next_digest = Instant::now(); + tracing::trace!(session = %self.session, "collaboration anti-entropy digest scheduled immediately"); + } + + pub fn on_neighbor_up(&mut self) { + tracing::debug!(session = %self.session, "collaboration anti-entropy neighbor-up recovery triggered"); + self.schedule_immediate_digest(); + } + + pub fn on_reconnect(&mut self) { + tracing::debug!(session = %self.session, "collaboration anti-entropy reconnect recovery triggered"); + self.schedule_immediate_digest(); + } + + #[must_use] + pub fn on_lagged(&mut self, fallback_peer: Option, our_vv: Vec, now: Instant) -> GapAction { + tracing::warn!(session = %self.session, fallback_peer = ?fallback_peer, our_vv_bytes = our_vv.len(), "collaboration anti-entropy gossip lagged"); + self.schedule_immediate_digest(); + let Some(from) = fallback_peer else { + tracing::warn!(session = %self.session, "collaboration anti-entropy lagged without fallback peer"); + return GapAction::None; + }; + self.begin_pull(from, our_vv, now) + } + + #[must_use] + pub fn consider_digest( + &mut self, + from: EndpointId, + digest_session: SessionId, + relation: VersionVectorRelation, + our_vv: Vec, + now: Instant, + ) -> GapAction { + tracing::trace!( + session = %self.session, + from = %from, + digest_session = %digest_session, + ?relation, + our_vv_bytes = our_vv.len(), + "collaboration anti-entropy digest considered", + ); + if digest_session != self.session { + tracing::warn!( + session = %self.session, + from = %from, + digest_session = %digest_session, + "collaboration anti-entropy digest lineage mismatch", + ); + return GapAction::LineageMismatch { + from, + expected: self.session, + got: digest_session, + }; + } + + match relation { + VersionVectorRelation::SenderHasMissingOps | VersionVectorRelation::Concurrent => self.begin_pull(from, our_vv, now), + VersionVectorRelation::Equal | VersionVectorRelation::WeHaveMissingOps => { + tracing::trace!(session = %self.session, from = %from, ?relation, "collaboration anti-entropy digest needs no pull"); + GapAction::None + }, + } + } + + /// Mark the outstanding pull to `from` as finished, clearing its dedup slot so + /// a fresh gap for that peer can start a new pull. + pub fn finish_pull(&mut self, from: EndpointId) { + let was_in_flight = self.pulls_in_flight.remove(&from).is_some(); + tracing::debug!(session = %self.session, from = %from, was_in_flight, in_flight = self.pulls_in_flight.len(), "collaboration anti-entropy pull finished"); + } + + /// Start a deduplicated pull to `from`: returns [`GapAction::Pull`] and registers + /// an in-flight slot (with a [`PULL_IN_FLIGHT_DEADLINE`] expiry), or + /// [`GapAction::None`] if a pull to that peer is already outstanding. Every pull — + /// digest-driven or triggered by a pending-dependency import — MUST go through + /// here so a single peer gets at most one in-flight pull, and so the matching + /// [`Self::finish_pull`] clears a slot this actually set. + pub fn begin_pull(&mut self, from: EndpointId, our_vv: Vec, now: Instant) -> GapAction { + self.expire_stale_pulls(now); + if self.pulls_in_flight.contains_key(&from) { + tracing::debug!(session = %self.session, from = %from, "collaboration anti-entropy skipped pull because one is already in flight for this peer"); + return GapAction::None; + } + self + .pulls_in_flight + .insert(from, now + PULL_IN_FLIGHT_DEADLINE); + tracing::debug!(session = %self.session, from = %from, our_vv_bytes = our_vv.len(), in_flight = self.pulls_in_flight.len(), "collaboration anti-entropy pull started"); + GapAction::Pull { from, our_vv } + } + + /// Drop in-flight pulls whose deadline has passed. A pull whose reply is never + /// delivered would otherwise pin its peer's dedup slot forever; expiring it + /// lets anti-entropy retry that peer without waiting on the wedged request. + fn expire_stale_pulls(&mut self, now: Instant) { + let before = self.pulls_in_flight.len(); + self.pulls_in_flight.retain(|_, deadline| *deadline > now); + let expired = before - self.pulls_in_flight.len(); + if expired > 0 { + tracing::warn!(session = %self.session, expired, in_flight = self.pulls_in_flight.len(), "collaboration anti-entropy expired wedged in-flight pulls past their deadline"); + } + } +} + +#[must_use] +pub fn next_jittered_deadline(now: Instant) -> Instant { + let base_millis = DIGEST_INTERVAL.as_millis() as u64; + let jitter_millis = base_millis * u64::from(DIGEST_JITTER_PERCENT) / 100; + let spread = jitter_millis.saturating_mul(2).saturating_add(1); + let offset = rand::random::() % spread; + let delay_millis = base_millis + .saturating_add(offset) + .saturating_sub(jitter_millis); + now + Duration::from_millis(delay_millis) +} diff --git a/crates/flowstate-collab/src/net/auth.rs b/crates/flowstate-collab/src/net/auth.rs new file mode 100644 index 00000000..cbeea205 --- /dev/null +++ b/crates/flowstate-collab/src/net/auth.rs @@ -0,0 +1,161 @@ +//! Symmetric bearer admission shared by every editor in a live session. + +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, OnceLock, RwLock}, +}; + +use anyhow::{Result, anyhow, ensure}; + +use crate::{ + admission::SessionAdmission, + ids::{PeerId, SessionId}, +}; + +#[derive(Debug)] +struct SessionAuthState { + admission: SessionAdmission, + peers: HashSet, +} + +#[derive(Clone, Debug, Default)] +pub struct SessionAuthRegistry { + inner: Arc>>, +} + +impl SessionAuthRegistry { + pub fn configure_session(&self, session: SessionId, admission: SessionAdmission) { + let mut sessions = self.write(); + match sessions.get_mut(&session) { + Some(state) if state.admission == admission => {}, + Some(state) => { + tracing::warn!(%session, "replacing collaboration admission secret; clearing authenticated peers"); + state.admission = admission; + state.peers.clear(); + }, + None => { + sessions.insert( + session, + SessionAuthState { + admission, + peers: HashSet::new(), + }, + ); + }, + } + } + + pub fn remove_session(&self, session: SessionId) { + self.write().remove(&session); + } + + pub fn authenticate_peer(&self, session: SessionId, peer: PeerId, admission: &SessionAdmission) -> Result<()> { + let mut sessions = self.write(); + let state = sessions + .get_mut(&session) + .ok_or_else(|| anyhow!("collaboration session is not configured for admission"))?; + ensure!(state.admission == *admission, "collaboration admission secret is invalid"); + state.peers.insert(peer); + drop(sessions); + Ok(()) + } + + #[must_use] + pub fn authorize_direct(&self, session: SessionId, peer: PeerId) -> bool { + self + .read() + .get(&session) + .is_some_and(|state| state.peers.contains(&peer)) + } + + #[must_use] + pub fn admission(&self, session: SessionId) -> Option { + self + .read() + .get(&session) + .map(|state| state.admission.clone()) + } + + fn read(&self) -> std::sync::RwLockReadGuard<'_, HashMap> { + self + .inner + .read() + .expect("collaboration auth registry lock poisoned") + } + + fn write(&self) -> std::sync::RwLockWriteGuard<'_, HashMap> { + self + .inner + .write() + .expect("collaboration auth registry lock poisoned") + } +} + +static LOCAL_ADMISSIONS: OnceLock>> = OnceLock::new(); + +fn local_admissions() -> &'static RwLock> { + LOCAL_ADMISSIONS.get_or_init(|| RwLock::new(HashMap::new())) +} + +pub fn install_local_admission(session: SessionId, admission: SessionAdmission) { + local_admissions() + .write() + .expect("local collaboration admission lock poisoned") + .insert(session, admission); +} + +pub fn clear_local_admission(session: SessionId) { + local_admissions() + .write() + .expect("local collaboration admission lock poisoned") + .remove(&session); +} + +#[must_use] +pub fn local_admission(session: SessionId) -> Option { + local_admissions() + .read() + .expect("local collaboration admission lock poisoned") + .get(&session) + .cloned() +} + +#[cfg(test)] +mod tests { + use iroh::SecretKey; + + use super::SessionAuthRegistry; + use crate::{SessionAdmission, SessionId}; + + #[test] + fn every_peer_authenticates_with_the_same_session_secret() { + let registry = SessionAuthRegistry::default(); + let session = SessionId::from_bytes([1; 32]); + let admission = SessionAdmission::from_bytes([2; 32]); + let first = SecretKey::from_bytes(&[3; 32]).public(); + let second = SecretKey::from_bytes(&[4; 32]).public(); + registry.configure_session(session, admission.clone()); + registry + .authenticate_peer(session, first, &admission) + .expect("first editor"); + registry + .authenticate_peer(session, second, &admission) + .expect("second editor"); + assert!(registry.authorize_direct(session, first)); + assert!(registry.authorize_direct(session, second)); + } + + #[test] + fn wrong_secret_is_rejected() { + let registry = SessionAuthRegistry::default(); + let session = SessionId::from_bytes([5; 32]); + registry.configure_session(session, SessionAdmission::from_bytes([6; 32])); + let peer = SecretKey::from_bytes(&[7; 32]).public(); + assert!( + registry + .authenticate_peer(session, peer, &SessionAdmission::from_bytes([8; 32])) + .is_err() + ); + assert!(!registry.authorize_direct(session, peer)); + } +} diff --git a/crates/flowstate-collab/src/net/blobs.rs b/crates/flowstate-collab/src/net/blobs.rs new file mode 100644 index 00000000..2206efef --- /dev/null +++ b/crates/flowstate-collab/src/net/blobs.rs @@ -0,0 +1,117 @@ +use std::{collections::VecDeque, num::NonZeroUsize}; + +use crate::ids::BlobId; + +pub const DEFAULT_MAX_BLOBS: usize = 64; +pub const DEFAULT_MAX_BYTES: usize = 16 * 1024 * 1024; + +#[derive(Debug)] +pub struct BlobOutbox { + max_blobs: NonZeroUsize, + max_bytes: usize, + total_bytes: usize, + entries: VecDeque<(BlobId, Vec)>, +} + +impl Default for BlobOutbox { + fn default() -> Self { + Self { + max_blobs: NonZeroUsize::new(DEFAULT_MAX_BLOBS).expect("constant is non-zero"), + max_bytes: DEFAULT_MAX_BYTES, + total_bytes: 0, + entries: VecDeque::new(), + } + } +} + +impl BlobOutbox { + #[must_use] + pub fn new(max_blobs: NonZeroUsize, max_bytes: usize) -> Self { + Self { + max_blobs, + max_bytes, + total_bytes: 0, + entries: VecDeque::new(), + } + } + + pub fn insert(&mut self, bytes: Vec) -> Option { + let id = BlobId::new(); + self.insert_with_id(id, bytes).then_some(id) + } + + pub fn insert_with_id(&mut self, id: BlobId, bytes: Vec) -> bool { + if bytes.len() > self.max_bytes { + return false; + } + + if let Some((_, existing)) = self + .entries + .iter_mut() + .find(|(candidate, _)| *candidate == id) + { + self.total_bytes = self + .total_bytes + .saturating_sub(existing.len()) + .saturating_add(bytes.len()); + *existing = bytes; + self.trim(); + return true; + } + + self.total_bytes = self.total_bytes.saturating_add(bytes.len()); + self.entries.push_back((id, bytes)); + self.trim(); + true + } + + #[must_use] + pub fn get(&self, id: BlobId) -> Option<&[u8]> { + self + .entries + .iter() + .find_map(|(candidate, bytes)| (*candidate == id).then_some(bytes.as_slice())) + } + + #[must_use] + pub fn len(&self) -> usize { + self.entries.len() + } + + #[must_use] + pub const fn total_bytes(&self) -> usize { + self.total_bytes + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn trim(&mut self) { + while self.entries.len() > self.max_blobs.get() || self.total_bytes > self.max_bytes { + let Some((_, bytes)) = self.entries.pop_front() else { + self.total_bytes = 0; + return; + }; + self.total_bytes = self.total_bytes.saturating_sub(bytes.len()); + } + } +} + +#[cfg(test)] +mod tests { + use std::num::NonZeroUsize; + + use crate::{BlobId, net::blobs::BlobOutbox}; + #[test] + fn insert_with_id_rejects_oversized_payloads_before_insertion() { + let mut outbox = BlobOutbox::new(NonZeroUsize::new(4).expect("non-zero max blobs"), 2); + let id = BlobId(7); + + assert!(!outbox.insert_with_id(id, vec![1, 2, 3])); + assert!(outbox.is_empty()); + assert_eq!(outbox.total_bytes(), 0); + assert_eq!(outbox.get(id), None); + } +} diff --git a/crates/flowstate-collab/src/net/direct.rs b/crates/flowstate-collab/src/net/direct.rs new file mode 100644 index 00000000..924ad585 --- /dev/null +++ b/crates/flowstate-collab/src/net/direct.rs @@ -0,0 +1,807 @@ +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, OnceLock, RwLock as StdRwLock}, + time::Duration, +}; + +use anyhow::{Context as _, Result, anyhow, bail, ensure}; +use async_channel::Sender; +use iroh::{ + Endpoint, EndpointId, + endpoint::{Connection, PathId, RecvStream, SendStream}, + protocol::{AcceptError, ProtocolHandler}, +}; +use tokio::{ + sync::{OwnedSemaphorePermit, RwLock, Semaphore}, + time::timeout, +}; + +use crate::{ + admission::SessionAdmission, + doc_io::DocIoHandle, + ids::{BlobId, SessionId}, + proto_direct::{ + AssetBytes, DIRECT_ALPN, DirectRequest, DirectResponseHeader, DiscoveryAdmissionGrant, MAX_FRAME_LEN, MAX_PAYLOAD_CHUNK_LEN, + MAX_PAYLOAD_LEN, WireCodec, decode_frame, encode_frame, + }, +}; + +use super::{PullProgress, auth::SessionAuthRegistry, blobs::BlobOutbox}; + +const DIRECT_SERVE_CONCURRENCY: usize = 4; +/// RTT below which the path is treated as "fast" (LAN/localhost) and wire +/// compression is skipped, because zstd decode (~700–900 MB/s) would become the +/// bottleneck rather than the link. See [`link_is_fast`]. +const FAST_LINK_RTT: Duration = Duration::from_millis(1); +const DIRECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); +static CLIENT_ENDPOINT: OnceLock>> = OnceLock::new(); + +#[derive(Clone)] +pub struct DirectSessionHandler { + requests: Sender, + /// Handle to the session's document I/O service. When present, snapshot / + /// update pulls are served through it (see [`serve_snapshot_via_io`]): the + /// I/O thread answers them under the write gate (Loro-first spec I-9a — a + /// raw ungated `LoroDoc` read is a commit barrier that can force-commit + /// mid-intent state, so the old shared read-handle path is outlawed), and + /// snapshot exports fork under the gate + export off it, so a peer's + /// recovery pull still cannot be starved behind local edits. `None` falls + /// back to the session-served request channel. + io: Option, +} + +impl DirectSessionHandler { + #[must_use] + pub fn new(requests: Sender, io: Option) -> Self { + Self { requests, io } + } +} + +impl std::fmt::Debug for DirectSessionHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DirectSessionHandler") + .field("has_io", &self.io.is_some()) + .finish_non_exhaustive() + } +} + +#[derive(Debug)] +pub enum DirectServeRequest { + Snapshot { reply: Sender>> }, + Updates { have_vv: Vec, reply: Sender>> }, + Asset { asset: u128, reply: Sender> }, +} + +#[derive(Clone, Debug, Default)] +pub struct DirectServeState { + inner: Arc>, + auth: SessionAuthRegistry, + standing_access: Arc>>, +} + +#[derive(Clone, Debug)] +struct StandingAccessGrant { + document_fingerprint: [u8; 32], + title: String, + identities: HashSet, +} + +#[derive(Debug, Default)] +struct DirectServeInner { + attached: HashSet, + blobs: HashMap, + handlers: HashMap, +} + +impl DirectServeState { + /// Session admission state shared by all direct requests. + #[must_use] + pub fn auth(&self) -> &SessionAuthRegistry { + &self.auth + } + + pub async fn attach_session(&self, session: SessionId) { + let mut inner = self.inner.write().await; + let inserted = inner.attached.insert(session); + tracing::debug!(%session, inserted, attached_sessions = inner.attached.len(), "attached collaboration direct session"); + } + + pub async fn detach_session(&self, session: SessionId) { + let mut inner = self.inner.write().await; + let removed = inner.attached.remove(&session); + let blob_count = inner + .blobs + .remove(&session) + .map_or(0, |outbox| outbox.len()); + let handler_removed = inner.handlers.remove(&session).is_some(); + self.standing_access.write().await.remove(&session); + tracing::debug!(%session, removed, blob_count, handler_removed, attached_sessions = inner.attached.len(), "detached collaboration direct session"); + } + + pub async fn register_handler(&self, session: SessionId, handler: DirectSessionHandler) { + let mut inner = self.inner.write().await; + inner.attached.insert(session); + let replaced = inner.handlers.insert(session, handler).is_some(); + tracing::debug!(%session, replaced, handler_count = inner.handlers.len(), "registered collaboration direct session handler"); + } + + pub async fn configure_standing_access( + &self, + session: SessionId, + document_fingerprint: [u8; 32], + title: String, + identities: HashSet, + ) { + self.standing_access.write().await.insert( + session, + StandingAccessGrant { + document_fingerprint, + title, + identities, + }, + ); + } + + #[allow( + clippy::significant_drop_tightening, + reason = "write guard scope is intentionally tight and no await occurs while held" + )] + pub async fn insert_blob(&self, session: SessionId, bytes: Vec) -> Result { + let byte_len = bytes.len(); + let (blob, outbox_len, outbox_bytes) = { + let mut inner = self.inner.write().await; + inner.attached.insert(session); + let outbox = inner.blobs.entry(session).or_default(); + let blob = BlobId::new(); + ensure!(outbox.insert_with_id(blob, bytes), "collaboration direct blob exceeds outbox capacity"); + let outbox_len = outbox.len(); + let outbox_bytes = outbox.total_bytes(); + (blob, outbox_len, outbox_bytes) + }; + tracing::debug!( + %session, + ?blob, + byte_len, + outbox_len, + outbox_bytes, + "stored collaboration direct blob for peer pull", + ); + Ok(blob) + } + + async fn serve(&self, request: DirectRequest, remote: EndpointId) -> ServeOutcome { + let session = request.session(); + let request_kind = direct_request_kind(&request); + let request_detail_bytes = direct_request_detail_bytes(&request); + tracing::trace!(%session, remote = %remote, request_kind, request_detail_bytes, "serving collaboration direct request"); + + if let DirectRequest::RequestAdmission { request } = &request { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if request.verify(now).is_err() || request.session != session { + return ServeOutcome::Header(DirectResponseHeader::Unauthorized); + } + let title = self + .standing_access + .read() + .await + .get(&session) + .and_then(|grant| { + (grant.document_fingerprint == request.document_fingerprint && grant.identities.contains(&request.identity)) + .then(|| grant.title.clone()) + }); + let Some(title) = title else { + return ServeOutcome::Header(DirectResponseHeader::Unauthorized); + }; + let Some(admission) = self.auth.admission(session) else { + return ServeOutcome::Header(DirectResponseHeader::NotAttached); + }; + let payload = match postcard::to_stdvec(&DiscoveryAdmissionGrant { admission, title }) { + Ok(payload) => payload, + Err(_) => return ServeOutcome::Header(DirectResponseHeader::NotFound), + }; + return ServeOutcome::Payload(payload); + } + + // The admission handshake and authorization gate live here, + // on the serving side, so enforcement never depends on the client. + if let DirectRequest::Authenticate { session, admission } = &request { + return match self.auth.authenticate_peer(*session, remote, admission) { + Ok(()) => { + tracing::debug!(%session, remote = %remote, "collaboration direct admission handshake accepted"); + ServeOutcome::Payload(Vec::new()) + }, + Err(error) => { + tracing::warn!(%session, remote = %remote, error = %format_args!("{error:#}"), "collaboration direct admission handshake rejected"); + ServeOutcome::Header(DirectResponseHeader::Unauthorized) + }, + }; + } + if !self.auth.authorize_direct(session, remote) { + tracing::warn!(%session, remote = %remote, request_kind, "collaboration direct request rejected without valid admission"); + return ServeOutcome::Header(DirectResponseHeader::Unauthorized); + } + + let handler = { + let inner = self.inner.read().await; + if !inner.attached.contains(&session) { + tracing::warn!(%session, request_kind, "collaboration direct request rejected because session is not attached"); + return ServeOutcome::Header(DirectResponseHeader::NotAttached); + } + if let DirectRequest::Blob { blob, .. } = request { + return inner + .blobs + .get(&session) + .and_then(|outbox| outbox.get(blob)) + .map_or_else( + || { + tracing::warn!(%session, ?blob, "collaboration direct blob request missed outbox"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + |bytes| { + tracing::debug!(%session, ?blob, bytes = bytes.len(), "collaboration direct blob request hit outbox"); + ServeOutcome::Payload(bytes.to_vec()) + }, + ); + } + inner.handlers.get(&session).cloned() + }; + + let Some(handler) = handler else { + tracing::warn!(%session, request_kind, "collaboration direct request rejected because handler is missing"); + return ServeOutcome::Header(DirectResponseHeader::NotFound); + }; + + match request { + DirectRequest::Snapshot { .. } => { + if let Some(io) = handler.io.clone() { + serve_snapshot_via_io(session, io).await + } else { + request_payload(handler.requests, session, request_kind, |reply| DirectServeRequest::Snapshot { reply }).await + } + }, + DirectRequest::Updates { have_vv, .. } => { + tracing::trace!(%session, have_vv_bytes = have_vv.len(), "serving collaboration direct updates request"); + if let Some(io) = handler.io.clone() { + serve_updates_via_io(session, io, have_vv).await + } else { + request_payload(handler.requests, session, request_kind, |reply| DirectServeRequest::Updates { + have_vv, + reply, + }) + .await + } + }, + DirectRequest::Asset { asset, .. } => request_asset(handler.requests, session, asset).await, + DirectRequest::Blob { .. } | DirectRequest::Authenticate { .. } | DirectRequest::RequestAdmission { .. } => { + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + } + } +} + +#[derive(Debug)] +enum ServeOutcome { + Header(DirectResponseHeader), + Payload(Vec), +} + +impl ServeOutcome { + fn kind(&self) -> &'static str { + match self { + Self::Header(header) => direct_response_header_kind(header), + Self::Payload(_) => "ok", + } + } + + fn payload_len(&self) -> usize { + match self { + Self::Header(_) => 0, + Self::Payload(payload) => payload.len(), + } + } +} + +#[derive(Clone, Debug)] +pub struct DirectProto { + state: DirectServeState, + permits: Arc, +} + +impl DirectProto { + #[must_use] + pub fn new(state: DirectServeState) -> Self { + Self { + state, + permits: Arc::new(Semaphore::new(DIRECT_SERVE_CONCURRENCY)), + } + } + + async fn handle_stream( + &self, + mut send: SendStream, + mut recv: RecvStream, + remote: EndpointId, + link_is_fast: bool, + _permit: OwnedSemaphorePermit, + ) -> Result<()> { + let request = read_frame::(&mut recv).await?; + let session = request.session(); + let request_kind = direct_request_kind(&request); + // Only the CRDT payloads (snapshots, update batches) benefit from zstd; assets + // are already compressed and blobs are opaque, so those stream verbatim. + let compressible = matches!(request, DirectRequest::Snapshot { .. } | DirectRequest::Updates { .. }); + tracing::debug!(%session, remote = %remote, request_kind, "received collaboration direct stream request"); + let outcome = self.state.serve(request, remote).await; + tracing::debug!(%session, request_kind, outcome = outcome.kind(), payload_bytes = outcome.payload_len(), "collaboration direct request served"); + write_response(&mut send, outcome, compressible, link_is_fast).await?; + Ok(()) + } +} + +impl ProtocolHandler for DirectProto { + async fn accept(&self, connection: Connection) -> Result<(), AcceptError> { + // The remote endpoint id comes from the connection's TLS handshake, so it + // is an authenticated sender identity, not self-reported data. + let remote = connection.remote_id(); + // Decide once per connection whether the path is fast enough that zstd decode + // would be the bottleneck rather than the link (then skip compression). + let link_is_fast = link_is_fast(&connection); + tracing::trace!(remote = %remote, link_is_fast, "accepted collaboration direct connection"); + while let Ok((send, recv)) = connection.accept_bi().await { + let Ok(permit) = self.permits.clone().try_acquire_owned() else { + let mut send = send; + if let Err(error) = write_response(&mut send, ServeOutcome::Header(DirectResponseHeader::Busy), false, link_is_fast).await { + tracing::warn!("flowstate collab direct busy response failed: {error:#}"); + } + continue; + }; + let proto = self.clone(); + tokio::spawn(async move { + if let Err(error) = proto + .handle_stream(send, recv, remote, link_is_fast, permit) + .await + { + tracing::error!(error = %format_args!("{error:#}"), "collaboration direct stream failed"); + } + }); + } + tracing::trace!("collaboration direct connection closed"); + Ok(()) + } +} + +pub(crate) fn install_endpoint(endpoint: Endpoint) { + let cache = CLIENT_ENDPOINT.get_or_init(|| StdRwLock::new(None)); + if let Ok(mut cached) = cache.write() { + *cached = Some(endpoint); + tracing::debug!(installed = true, "installed collaboration direct client endpoint"); + } else { + tracing::warn!("failed to install collaboration direct client endpoint"); + } +} + +pub async fn pull_with_fallback(req: DirectRequest, candidates: Vec, per_peer_timeout: Duration) -> Result> { + pull_with_fallback_progress(req, candidates, per_peer_timeout, None).await +} + +pub async fn pull_with_fallback_progress( + req: DirectRequest, + candidates: Vec, + per_peer_timeout: Duration, + progress: Option>, +) -> Result> { + let endpoint = CLIENT_ENDPOINT + .get() + .and_then(|cache| cache.read().ok().and_then(|cached| cached.clone())) + .ok_or_else(|| anyhow!("collaboration direct client endpoint is not running"))?; + ensure!(!endpoint.is_closed(), "collaboration direct client endpoint is not running"); + pull_with_endpoint_progress(&endpoint, req, candidates, per_peer_timeout, progress).await +} + +pub async fn pull_with_endpoint( + endpoint: &Endpoint, + req: DirectRequest, + candidates: Vec, + per_peer_timeout: Duration, +) -> Result> { + pull_with_endpoint_progress(endpoint, req, candidates, per_peer_timeout, None).await +} + +pub async fn pull_with_endpoint_progress( + endpoint: &Endpoint, + req: DirectRequest, + candidates: Vec, + per_peer_timeout: Duration, + progress: Option>, +) -> Result> { + let session = req.session(); + let request_kind = direct_request_kind(&req); + let candidate_count = candidates.len(); + if candidates.is_empty() { + tracing::warn!(%session, request_kind, "collaboration direct pull has no candidate peers"); + } + ensure!(!candidates.is_empty(), "direct pull has no candidate peers"); + tracing::debug!(%session, request_kind, candidate_count, ?per_peer_timeout, "starting collaboration direct pull"); + + let mut errors = Vec::new(); + for peer in candidates { + tracing::trace!(%session, request_kind, peer = %peer, "attempting collaboration direct pull peer"); + // §perf: pull_once uses req only by reference; avoid cloning DirectRequest + // (whose `have_vv` grows with document size) per candidate peer. + match timeout(per_peer_timeout, pull_once(endpoint, peer, &req, progress.as_ref())).await { + Ok(Ok(bytes)) => { + tracing::debug!(%session, request_kind, peer = %peer, bytes = bytes.len(), "collaboration direct pull peer succeeded"); + return Ok(bytes); + }, + Ok(Err(error)) => { + tracing::warn!(%session, request_kind, peer = %peer, error = %format_args!("{error:#}"), "collaboration direct pull peer failed"); + errors.push(format!("{peer}: {error:#}")); + }, + Err(_) => { + tracing::warn!(%session, request_kind, peer = %peer, ?per_peer_timeout, "collaboration direct pull peer timed out"); + errors.push(format!("{peer}: timed out after {per_peer_timeout:?}")); + }, + } + } + + tracing::error!(%session, request_kind, candidate_count, "collaboration direct pull failed for all candidates"); + Err(anyhow!("direct pull failed for all candidates: {}", errors.join("; "))) +} + +async fn pull_once(endpoint: &Endpoint, peer: EndpointId, req: &DirectRequest, progress: Option<&Sender>) -> Result> { + let session = req.session(); + let request_kind = direct_request_kind(req); + tracing::trace!(%session, request_kind, peer = %peer, "dialing collaboration direct peer"); + let connection = endpoint + .connect(peer, DIRECT_ALPN) + .await + .context("direct dial failed")?; + if let Some(admission) = super::auth::local_admission(session) { + authenticate_connection(&connection, session, admission) + .await + .context("collaboration admission handshake failed")?; + } + request_on_connection(&connection, req, progress).await +} + +async fn authenticate_connection(connection: &Connection, session: SessionId, admission: SessionAdmission) -> Result<()> { + tracing::trace!(%session, "sending collaboration direct admission handshake"); + let _ = request_on_connection(connection, &DirectRequest::Authenticate { session, admission }, None).await?; + Ok(()) +} + +async fn request_on_connection(connection: &Connection, req: &DirectRequest, progress: Option<&Sender>) -> Result> { + let session = req.session(); + let request_kind = direct_request_kind(req); + tracing::trace!(%session, request_kind, "opening collaboration direct stream"); + let (mut send, mut recv) = connection + .open_bi() + .await + .context("opening direct request stream failed")?; + let frame = encode_frame(req)?; + tracing::trace!(%session, request_kind, frame_bytes = frame.len(), "sending collaboration direct request frame"); + write_frame(&mut send, &frame).await?; + send.finish()?; + + let header = read_frame::(&mut recv).await?; + tracing::trace!(%session, request_kind, response = direct_response_header_kind(&header), "received collaboration direct response header"); + match header { + DirectResponseHeader::Ok { + codec, + wire_len, + uncompressed_len, + } => { + let wire = read_payload(&mut recv, wire_len, progress).await?; + let uncompressed_len = usize::try_from(uncompressed_len).context("direct payload is too large for this platform")?; + let payload = super::wire_compression::decompress_from_wire(codec, wire, uncompressed_len).context("decoding direct response payload")?; + tracing::trace!(%session, request_kind, wire_bytes = wire_len, bytes = payload.len(), codec = ?codec, "read collaboration direct response payload"); + Ok(payload) + }, + DirectResponseHeader::NotAttached => Err(anyhow!("peer is not attached to this session")), + DirectResponseHeader::NotFound => Err(anyhow!("peer does not have the requested collaboration data")), + DirectResponseHeader::Busy => Err(anyhow!("peer is busy serving collaboration data")), + DirectResponseHeader::Unauthorized => Err(anyhow!("peer rejected our collaboration session admission")), + } +} + +async fn request_payload( + requests: Sender, + session: SessionId, + request_kind: &'static str, + make_request: F, +) -> ServeOutcome +where + F: FnOnce(Sender>>) -> DirectServeRequest, +{ + let (reply_tx, reply_rx) = async_channel::bounded(1); + if requests.send(make_request(reply_tx)).await.is_err() { + tracing::warn!(%session, request_kind, "collaboration direct session request channel closed"); + return ServeOutcome::Header(DirectResponseHeader::NotAttached); + } + match timeout(DIRECT_RESPONSE_TIMEOUT, reply_rx.recv()).await { + Ok(Ok(Ok(bytes))) => { + tracing::trace!(%session, request_kind, bytes = bytes.len(), "collaboration direct session returned payload"); + ServeOutcome::Payload(bytes) + }, + Ok(Ok(Err(error))) => { + tracing::warn!(%session, request_kind, error = %format_args!("{error:#}"), "collaboration direct session failed to produce payload"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + Ok(Err(error)) => { + tracing::warn!(%session, request_kind, error = %error, "collaboration direct session payload reply channel closed"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + Err(_) => { + tracing::warn!(%session, request_kind, ?DIRECT_RESPONSE_TIMEOUT, "collaboration direct session payload timed out"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + } +} + +/// Serve a snapshot pull through the document I/O service. The gate-held part +/// is a brief `fork()`; the actual snapshot encode happens off-gate on the I/O +/// thread (spec I-9a long-export rule), so a large snapshot neither stalls +/// typing nor gets starved behind local edits. +async fn serve_snapshot_via_io(session: SessionId, io: DocIoHandle) -> ServeOutcome { + match io.snapshot_bytes().await { + Ok(bytes) => { + tracing::trace!(%session, bytes = bytes.len(), "served collaboration snapshot via the document I/O service"); + ServeOutcome::Payload(bytes) + }, + Err(error) => { + tracing::warn!(%session, error = %format_args!("{error:#}"), "exporting collaboration snapshot failed"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + } +} + +/// Serve an incremental-updates pull through the document I/O service (see +/// [`serve_snapshot_via_io`]); the version-vector decode and gate-held export +/// happen on the I/O thread. +async fn serve_updates_via_io(session: SessionId, io: DocIoHandle, have_vv: Vec) -> ServeOutcome { + match io.export_updates_for(have_vv).await { + Ok(bytes) => { + tracing::trace!(%session, bytes = bytes.len(), "served collaboration updates via the document I/O service"); + ServeOutcome::Payload(bytes) + }, + Err(error) => { + tracing::warn!(%session, error = %format_args!("{error:#}"), "exporting collaboration updates failed"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + } +} + +async fn request_asset(requests: Sender, session: SessionId, asset: u128) -> ServeOutcome { + let (reply_tx, reply_rx) = async_channel::bounded(1); + if requests + .send(DirectServeRequest::Asset { asset, reply: reply_tx }) + .await + .is_err() + { + tracing::warn!(%session, asset, "collaboration direct asset request channel closed"); + return ServeOutcome::Header(DirectResponseHeader::NotAttached); + } + match timeout(DIRECT_RESPONSE_TIMEOUT, reply_rx.recv()).await { + Ok(Ok(Ok(asset_bytes))) => { + tracing::trace!(%session, asset, bytes = asset_bytes.bytes.len(), "collaboration direct session returned asset"); + ServeOutcome::Payload(asset_bytes.bytes) + }, + Ok(Ok(Err(error))) => { + tracing::warn!(%session, asset, error = %format_args!("{error:#}"), "collaboration direct session failed to produce asset"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + Ok(Err(error)) => { + tracing::warn!(%session, asset, error = %error, "collaboration direct asset reply channel closed"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + Err(_) => { + tracing::warn!(%session, asset, ?DIRECT_RESPONSE_TIMEOUT, "collaboration direct asset timed out"); + ServeOutcome::Header(DirectResponseHeader::NotFound) + }, + } +} + +async fn read_frame(recv: &mut RecvStream) -> Result +where + T: for<'de> serde::Deserialize<'de>, +{ + let mut len_bytes = [0; 4]; + recv.read_exact(&mut len_bytes).await?; + let payload_len = u32::from_le_bytes(len_bytes) as usize; + ensure!(payload_len <= MAX_FRAME_LEN, "direct frame exceeds {MAX_FRAME_LEN} bytes"); + let mut frame = len_bytes.to_vec(); + frame.resize(4 + payload_len, 0); + recv.read_exact(&mut frame[4..]).await?; + decode_frame(&frame) +} + +/// Whether the connection's path is fast enough that zstd decode would be the +/// bottleneck rather than the link, in which case compression is skipped. A very +/// low RTT means LAN/localhost; higher-RTT WAN/relay paths gain ~2.5–3× effective +/// goodput from compression, so they keep it. +fn link_is_fast(connection: &Connection) -> bool { + // `PathId::ZERO` is the primary path (iroh's own net-report reads rtt the same + // way). An unknown rtt is treated as NOT fast, so we compress when in doubt. + connection + .rtt(PathId::ZERO) + .is_some_and(|rtt| rtt < FAST_LINK_RTT) +} + +async fn write_response(send: &mut SendStream, outcome: ServeOutcome, compressible: bool, link_is_fast: bool) -> Result<()> { + tracing::trace!( + outcome = outcome.kind(), + payload_bytes = outcome.payload_len(), + "writing collaboration direct response" + ); + match outcome { + ServeOutcome::Header(header) => write_frame(send, &encode_frame(&header)?).await?, + ServeOutcome::Payload(payload) => { + // Snapshots/updates are compressed by size (dictionary for small, long mode + // for big); non-CRDT payloads (assets) and fast links stream verbatim. + // §perf: WireBytes hands the caller the SAME Arc as the cache (or a borrow of + // `payload`), avoiding the 1-2 full compressed-payload memcpys the old Cow path took. + let (codec, wire): (WireCodec, super::wire_compression::WireBytes<'_>) = if compressible { + super::wire_compression::compress_for_wire(&payload, link_is_fast) + } else { + (WireCodec::None, super::wire_compression::WireBytes::Borrowed(payload.as_slice())) + }; + let header = DirectResponseHeader::Ok { + codec, + wire_len: wire.as_slice().len() as u64, + uncompressed_len: payload.len() as u64, + }; + tracing::trace!(codec = ?codec, wire_bytes = wire.as_slice().len(), payload_bytes = payload.len(), "wrote collaboration direct payload for the wire"); + write_frame(send, &encode_frame(&header)?).await?; + write_payload(send, wire.as_slice()).await?; + }, + } + send.finish()?; + Ok(()) +} + +async fn write_frame(send: &mut SendStream, frame: &[u8]) -> Result<()> { + tracing::trace!(frame_bytes = frame.len(), "writing collaboration direct frame"); + send.write_all(frame).await?; + Ok(()) +} + +async fn write_payload(send: &mut SendStream, payload: &[u8]) -> Result<()> { + tracing::trace!( + payload_bytes = payload.len(), + chunk_bytes = MAX_PAYLOAD_CHUNK_LEN, + "writing collaboration direct payload" + ); + for chunk in payload.chunks(MAX_PAYLOAD_CHUNK_LEN) { + send.write_all(chunk).await?; + } + Ok(()) +} + +async fn read_payload(recv: &mut RecvStream, total_len: u64, progress: Option<&Sender>) -> Result> { + let total_len_usize = usize::try_from(total_len).context("direct payload is too large for this platform")?; + ensure!(total_len_usize <= MAX_PAYLOAD_LEN, "direct payload exceeds {MAX_PAYLOAD_LEN} bytes"); + tracing::trace!( + payload_bytes = total_len_usize, + chunk_bytes = MAX_PAYLOAD_CHUNK_LEN, + "reading collaboration direct payload" + ); + // Reserve the exact buffer up front so the whole payload lands in one allocation with + // no reallocation on the happy path, but do NOT zero-fill it (as `vec![0; len]` would): + // the reserved pages are only committed as chunks are copied in, which keeps large + // snapshots — up to 1 GiB here — off the hot path's zeroing cost. `with_capacity` is a + // virtual reservation, so a peer that overstates `total_len` cannot force a physical + // commit beyond the bytes it actually sends. + let mut payload = Vec::with_capacity(total_len_usize); + if let Some(progress) = progress { + let _ = progress.try_send(PullProgress { got: 0, total: total_len }); + } + while payload.len() < total_len_usize { + let remaining = total_len_usize - payload.len(); + match recv.read_chunk(remaining).await? { + Some(bytes) => { + payload.extend_from_slice(&bytes); + if let Some(progress) = progress { + let _ = progress.try_send(PullProgress { + got: payload.len() as u64, + total: total_len, + }); + } + }, + None => bail!("direct payload ended early: expected {total_len_usize} bytes, received {}", payload.len()), + } + } + Ok(payload) +} + +fn direct_request_kind(request: &DirectRequest) -> &'static str { + match request { + DirectRequest::Authenticate { .. } => "authenticate", + DirectRequest::RequestAdmission { .. } => "request-admission", + DirectRequest::Snapshot { .. } => "snapshot", + DirectRequest::Updates { .. } => "updates", + DirectRequest::Blob { .. } => "blob", + DirectRequest::Asset { .. } => "asset", + } +} + +fn direct_request_detail_bytes(request: &DirectRequest) -> usize { + match request { + DirectRequest::Updates { have_vv, .. } => have_vv.len(), + DirectRequest::Authenticate { .. } + | DirectRequest::RequestAdmission { .. } + | DirectRequest::Snapshot { .. } + | DirectRequest::Blob { .. } + | DirectRequest::Asset { .. } => 0, + } +} + +fn direct_response_header_kind(header: &DirectResponseHeader) -> &'static str { + match header { + DirectResponseHeader::Ok { .. } => "ok", + DirectResponseHeader::NotAttached => "not_attached", + DirectResponseHeader::NotFound => "not_found", + DirectResponseHeader::Busy => "busy", + DirectResponseHeader::Unauthorized => "unauthorized", + } +} + +impl DirectRequest { + fn session(&self) -> SessionId { + match self { + Self::Authenticate { session, .. } + | Self::RequestAdmission { + request: crate::discovery::DiscoveryAdmissionRequest { session, .. }, + } + | Self::Snapshot { session } + | Self::Updates { session, .. } + | Self::Blob { session, .. } + | Self::Asset { session, .. } => *session, + } + } +} + +// Class 6 — asset (and any direct) pull must FAIL CLEANLY when candidate peers are +// unreachable: return the "failed for all candidates" error within the per-peer timeout +// budget, never hang the caller. The field logs showed `direct pull failed for all +// candidates request_kind="asset"`; these pin that the failure path is bounded and +// well-formed, using a local-only (`presets::Minimal`, no relays/DNS) endpoint so the test +// is deterministic and never touches the network. +#[cfg(test)] +mod tests { + use super::*; + use iroh::{SecretKey, endpoint::presets}; + + fn asset_request() -> DirectRequest { + DirectRequest::Asset { + session: SessionId::new(), + asset: 42, + } + } + + #[tokio::test] + async fn asset_pull_fails_cleanly_when_all_candidates_are_unreachable() -> Result<()> { + let endpoint = Endpoint::builder(presets::Minimal).bind().await?; + // A freshly-minted peer id that is not serving DIRECT_ALPN anywhere reachable. + let unreachable = SecretKey::generate().public(); + let result = pull_with_endpoint(&endpoint, asset_request(), vec![unreachable], Duration::from_millis(250)).await; + let error = result.expect_err("asset pull from an unreachable candidate must fail, not hang"); + assert!( + error + .to_string() + .contains("direct pull failed for all candidates"), + "expected the all-candidates-failed error, got: {error}" + ); + Ok(()) + } + + #[tokio::test] + async fn asset_pull_with_no_candidates_errors_without_reaching_the_network() -> Result<()> { + let endpoint = Endpoint::builder(presets::Minimal).bind().await?; + let result = pull_with_endpoint(&endpoint, asset_request(), Vec::new(), Duration::from_millis(250)).await; + assert!(result.is_err(), "asset pull with no candidate peers must error cleanly"); + Ok(()) + } +} diff --git a/crates/flowstate-collab/src/net/mod.rs b/crates/flowstate-collab/src/net/mod.rs new file mode 100644 index 00000000..93ed6562 --- /dev/null +++ b/crates/flowstate-collab/src/net/mod.rs @@ -0,0 +1,170 @@ +pub mod anti_entropy; +pub mod auth; +pub mod blobs; +pub mod direct; +pub mod runtime; +pub mod swarm; +pub(crate) mod wire_compression; + +use anyhow::Result; +use async_channel::Sender; +use iroh::{EndpointAddr, EndpointId, PublicKey}; + +use crate::{ + admission::SessionAdmission, + ids::{BlobId, SessionId}, + proto_direct::AssetBytes, + proto_gossip::GossipMsg, +}; + +use self::direct::DirectSessionHandler; + +pub type Reply = Sender; +pub type PeerAddr = EndpointAddr; + +#[derive(Clone, Debug)] +pub enum PublishPayload { + Update(Vec), + Presence(Vec), + Digest { vv: Vec }, +} + +impl PublishPayload { + #[must_use] + pub fn kind(&self) -> &'static str { + match self { + Self::Update(_) => "update", + Self::Presence(_) => "presence", + Self::Digest { .. } => "digest", + } + } + + #[must_use] + pub fn byte_len(&self) -> usize { + match self { + Self::Update(bytes) | Self::Presence(bytes) => bytes.len(), + Self::Digest { vv } => vv.len(), + } + } +} + +#[derive(Debug)] +pub enum NetCommand { + EnsureUp, + RegisterDirectHandler { + session: SessionId, + handler: DirectSessionHandler, + }, + ConfigureStandingAccess { + session: SessionId, + document_fingerprint: [u8; 32], + title: String, + identities: Vec, + }, + RequestDiscoveredTicket { + advertisement: Box, + request: Box, + reply: Reply>, + }, + CreateSession { + session: SessionId, + reply: Reply>, + }, + JoinSession { + session: SessionId, + bootstrap: Vec, + admission: SessionAdmission, + }, + LeaveSession { + session: SessionId, + }, + MintTicket { + session: SessionId, + reply: Reply>, + }, + Publish { + session: SessionId, + payload: PublishPayload, + }, + PullUpdates { + session: SessionId, + candidates: Vec, + our_vv: Vec, + reply: Reply>>, + }, + PullSnapshot { + session: SessionId, + candidates: Vec, + progress: Option>, + reply: Reply>>, + }, + PullBlob { + session: SessionId, + candidates: Vec, + blob: BlobId, + reply: Reply>>, + }, + PullAsset { + session: SessionId, + candidates: Vec, + asset: u128, + reply: Reply>, + }, + MintTicketAddr { + reply: Reply, + }, + Shutdown, +} + +impl NetCommand { + #[must_use] + pub fn kind(&self) -> &'static str { + match self { + Self::EnsureUp => "ensure_up", + Self::RegisterDirectHandler { .. } => "register_direct_handler", + Self::ConfigureStandingAccess { .. } => "configure_standing_access", + Self::RequestDiscoveredTicket { .. } => "request_discovered_ticket", + Self::CreateSession { .. } => "create_session", + Self::JoinSession { .. } => "join_session", + Self::LeaveSession { .. } => "leave_session", + Self::MintTicket { .. } => "mint_ticket", + Self::Publish { .. } => "publish", + Self::PullUpdates { .. } => "pull_updates", + Self::PullSnapshot { .. } => "pull_snapshot", + Self::PullBlob { .. } => "pull_blob", + Self::PullAsset { .. } => "pull_asset", + Self::MintTicketAddr { .. } => "mint_ticket_addr", + Self::Shutdown => "shutdown", + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct PullProgress { + pub got: u64, + pub total: u64, +} + +#[derive(Clone, Debug)] +pub struct TicketSeed { + pub inviter: EndpointAddr, + pub admission: SessionAdmission, +} + +/// Reply payload of [`NetCommand::MintTicket`]. +#[derive(Clone, Debug)] +pub struct MintedTicket { + pub inviter: EndpointAddr, + pub admission: SessionAdmission, +} + +#[derive(Clone, Debug)] +pub enum NetEvent { + Gossip { session: SessionId, from: EndpointId, msg: GossipMsg }, + IncompatibleVersion { session: SessionId, peer: EndpointId }, + NeighborUp { session: SessionId, peer: EndpointId }, + NeighborDown { session: SessionId, peer: EndpointId }, + GossipLagged { session: SessionId }, + SubscribeFailed { session: SessionId, error: String }, + EndpointOnline(bool), +} diff --git a/crates/flowstate-collab/src/net/runtime.rs b/crates/flowstate-collab/src/net/runtime.rs new file mode 100644 index 00000000..95bdd7c8 --- /dev/null +++ b/crates/flowstate-collab/src/net/runtime.rs @@ -0,0 +1,416 @@ +use std::{ + collections::HashMap, + sync::{Mutex, OnceLock}, + thread, + time::Duration, +}; + +use anyhow::{Context as _, Result}; +use iroh::{Endpoint, EndpointAddr, Watcher as _, endpoint::presets}; +use iroh_gossip::net::Gossip; +use tokio::time::timeout; + +use crate::{ + SessionId, + admission::SessionAdmission, + proto_direct::{AssetBytes, DirectRequest, DiscoveryAdmissionGrant}, + ticket::SessionTicket, +}; + +use super::{ + MintedTicket, NetCommand, NetEvent, TicketSeed, auth, + direct::{self, DirectProto, DirectServeState}, + swarm::SwarmHandle, +}; + +pub type CommandSender = async_channel::Sender; +pub type EventReceiver = async_channel::Receiver; + +const DIRECT_PULL_TIMEOUT: Duration = Duration::from_secs(10); +const ENDPOINT_ADDR_READY_TIMEOUT: Duration = Duration::from_secs(2); +/// FS-074: the command and event channels are bounded so a stalled network +/// thread or UI event pump applies backpressure instead of growing without +/// limit. Command producers use `try_send` and tolerate a full channel; event +/// producers coalesce or backpressure per `swarm::forward_event`. +const RUNTIME_COMMAND_CAPACITY: usize = 1024; +const RUNTIME_EVENT_CAPACITY: usize = 1024; +static RUNTIME: OnceLock>> = OnceLock::new(); + +fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_secs() +} + +#[derive(Clone)] +struct RuntimeBridge { + commands: CommandSender, + events: EventReceiver, +} + +impl RuntimeBridge { + fn is_alive(&self) -> bool { + !self.commands.is_closed() + } + + fn handles(&self) -> (CommandSender, EventReceiver) { + (self.commands.clone(), self.events.clone()) + } +} + +pub fn start() -> Result<(CommandSender, EventReceiver)> { + let runtime = RUNTIME.get_or_init(|| Mutex::new(None)); + let mut runtime = runtime.lock().expect("collaboration runtime lock poisoned"); + if let Some(runtime) = runtime.as_ref().filter(|runtime| runtime.is_alive()) { + tracing::debug!("reusing existing collaboration network runtime"); + return Ok(runtime.handles()); + } + + tracing::info!("starting collaboration network runtime"); + let next_runtime = spawn_runtime()?; + let handles = next_runtime.handles(); + *runtime = Some(next_runtime); + drop(runtime); + Ok(handles) +} + +fn spawn_runtime() -> Result { + let (cmd_tx, cmd_rx) = async_channel::bounded(RUNTIME_COMMAND_CAPACITY); + let (evt_tx, evt_rx) = async_channel::bounded(RUNTIME_EVENT_CAPACITY); + let thread_evt_tx = evt_tx.clone(); + + thread::Builder::new() + .name("flowstate-collab-net".into()) + .spawn(move || { + tracing::debug!("collaboration network thread started"); + let runtime = match tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = thread_evt_tx.try_send(NetEvent::EndpointOnline(false)); + tracing::error!(error = %format_args!("{error:#}"), "collaboration tokio runtime failed"); + return; + }, + }; + + if let Err(error) = runtime.block_on(net_main(cmd_rx, evt_tx)) { + let _ = thread_evt_tx.try_send(NetEvent::EndpointOnline(false)); + tracing::error!(error = %format_args!("{error:#}"), "collaboration networking stopped unexpectedly"); + } + tracing::debug!("collaboration network thread stopped"); + }) + .context("spawning collaboration networking thread failed")?; + + Ok(RuntimeBridge { + commands: cmd_tx, + events: evt_rx, + }) +} + +async fn net_main(cmd_rx: async_channel::Receiver, evt_tx: async_channel::Sender) -> Result<()> { + tracing::info!("binding collaboration endpoint"); + let endpoint = Endpoint::builder(presets::N0) + .bind() + .await + .context("binding collaboration endpoint failed")?; + let bound_addr = endpoint_addr(&endpoint); + tracing::info!(peer = %bound_addr.id, "collaboration endpoint bound"); + direct::install_endpoint(endpoint.clone()); + + let gossip = Gossip::builder().spawn(endpoint.clone()); + let direct_state = DirectServeState::default(); + let _router = iroh::protocol::Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .accept(crate::DIRECT_ALPN, DirectProto::new(direct_state.clone())) + .spawn(); + tracing::info!(peer = %bound_addr.id, "collaboration direct and gossip protocols installed"); + + let _ = evt_tx.send(NetEvent::EndpointOnline(true)).await; + let mut swarms = HashMap::new(); + + while let Ok(command) = cmd_rx.recv().await { + tracing::trace!(command = command.kind(), "collaboration network command received"); + match command { + NetCommand::EnsureUp => { + tracing::debug!("collaboration endpoint ensure-up requested"); + let _ = evt_tx.send(NetEvent::EndpointOnline(true)).await; + }, + NetCommand::RegisterDirectHandler { session, handler } => { + tracing::debug!(%session, "registering collaboration direct handler"); + direct_state.register_handler(session, handler).await; + }, + NetCommand::ConfigureStandingAccess { + session, + document_fingerprint, + title, + identities, + } => { + direct_state + .configure_standing_access(session, document_fingerprint, title, identities.into_iter().collect()) + .await; + }, + NetCommand::RequestDiscoveredTicket { + advertisement, + request, + reply, + } => { + let result = async { + advertisement.verify(unix_now())?; + anyhow::ensure!( + request.session == advertisement.session, + "discovery request session does not match advertisement" + ); + anyhow::ensure!( + request.document_fingerprint == advertisement.document_fingerprint, + "discovery request document does not match advertisement" + ); + let bytes = direct::pull_with_fallback( + DirectRequest::RequestAdmission { request: *request }, + vec![advertisement.endpoint.id], + DIRECT_PULL_TIMEOUT, + ) + .await?; + let grant: DiscoveryAdmissionGrant = postcard::from_bytes(&bytes).context("decoding discovery admission grant")?; + Ok(SessionTicket::new( + advertisement.session, + vec![advertisement.endpoint], + grant.title, + grant.admission, + )) + } + .await; + let _ = reply.send(result).await; + }, + NetCommand::CreateSession { session, reply } => { + let result = create_session(&mut swarms, &endpoint, &gossip, &direct_state, &evt_tx, session).await; + if let Err(error) = &result { + tracing::warn!(%session, error = %format_args!("{error:#}"), "collaboration create session failed"); + } + let _ = reply.send(result).await; + }, + NetCommand::JoinSession { + session, + bootstrap, + admission, + } => { + if let Err(error) = join_session(&mut swarms, &endpoint, &gossip, &direct_state, &evt_tx, session, bootstrap, admission).await { + tracing::warn!(%session, error = %format_args!("{error:#}"), "collaboration join session failed"); + let _ = evt_tx + .send(NetEvent::SubscribeFailed { + session, + error: format!("{error:#}"), + }) + .await; + } + }, + NetCommand::LeaveSession { session } => { + tracing::info!(%session, "leaving collaboration network session"); + if let Some(handle) = swarms.remove(&session) { + handle.stop().await; + } else { + tracing::debug!(%session, "leave requested for missing collaboration swarm"); + } + direct_state.detach_session(session).await; + // Forget the bearer admission so a later rejoin must install it again. + direct_state.auth().remove_session(session); + auth::clear_local_admission(session); + }, + NetCommand::Publish { session, payload } => { + let payload_kind = payload.kind(); + let payload_bytes = payload.byte_len(); + if let Some(handle) = swarms.get(&session) { + tracing::trace!(%session, payload_kind, payload_bytes, "publishing collaboration payload"); + if let Err(error) = handle.publish(payload).await { + tracing::warn!(%session, payload_kind, payload_bytes, error = %format_args!("{error:#}"), "collaboration publish failed"); + } + } else { + tracing::warn!(%session, payload_kind, payload_bytes, "dropped collaboration publish for missing swarm"); + } + }, + NetCommand::PullUpdates { + session, + candidates, + our_vv, + reply, + } => { + let candidate_count = candidates.len(); + let vv_bytes = our_vv.len(); + tracing::debug!(%session, candidate_count, vv_bytes, "pulling collaboration updates"); + let result = direct::pull_with_fallback(DirectRequest::Updates { session, have_vv: our_vv }, candidates, DIRECT_PULL_TIMEOUT).await; + log_direct_pull_result("updates", session, candidate_count, &result); + let _ = reply.send(result).await; + }, + NetCommand::PullSnapshot { + session, + candidates, + progress, + reply, + } => { + let candidate_count = candidates.len(); + tracing::debug!(%session, candidate_count, "pulling collaboration snapshot"); + let result = direct::pull_with_fallback_progress(DirectRequest::Snapshot { session }, candidates, DIRECT_PULL_TIMEOUT, progress).await; + log_direct_pull_result("snapshot", session, candidate_count, &result); + let _ = reply.send(result).await; + }, + NetCommand::PullBlob { + session, + candidates, + blob, + reply, + } => { + let candidate_count = candidates.len(); + tracing::debug!(%session, ?blob, candidate_count, "pulling collaboration blob"); + let result = direct::pull_with_fallback(DirectRequest::Blob { session, blob }, candidates, DIRECT_PULL_TIMEOUT).await; + log_direct_pull_result("blob", session, candidate_count, &result); + let _ = reply.send(result).await; + }, + NetCommand::PullAsset { + session, + candidates, + asset, + reply, + } => { + let candidate_count = candidates.len(); + tracing::debug!(%session, asset, candidate_count, "pulling collaboration asset"); + let result = direct::pull_with_fallback(DirectRequest::Asset { session, asset }, candidates, DIRECT_PULL_TIMEOUT) + .await + .map(|bytes| AssetBytes { bytes }); + match &result { + Ok(bytes) => tracing::debug!(%session, asset, candidate_count, bytes = bytes.bytes.len(), "collaboration asset pull succeeded"), + Err(error) => tracing::warn!(%session, asset, candidate_count, error = %format_args!("{error:#}"), "collaboration asset pull failed"), + } + let _ = reply.send(result).await; + }, + NetCommand::MintTicket { session, reply } => { + // Every authenticated editor carries the same ephemeral admission + // secret, so any live participant can mint an equivalent invite. + let inviter = reachable_endpoint_addr(&endpoint).await; + let result = direct_state + .auth() + .admission(session) + .map(|admission| MintedTicket { inviter, admission }) + .ok_or_else(|| anyhow::anyhow!("collaboration session admission is unavailable")); + let _ = reply.send(result).await; + }, + NetCommand::MintTicketAddr { reply } => { + let addr = reachable_endpoint_addr(&endpoint).await; + tracing::debug!(peer = %addr.id, "minting collaboration ticket address"); + let _ = reply.send(addr).await; + }, + NetCommand::Shutdown => { + tracing::info!(open_sessions = swarms.len(), "shutting down collaboration network runtime"); + break; + }, + } + } + + for (_, handle) in swarms { + tracing::debug!(session = %handle.session, "stopping collaboration swarm during shutdown"); + handle.stop().await; + } + endpoint.close().await; + tracing::info!(peer = %bound_addr.id, "collaboration endpoint closed"); + Ok(()) +} + +async fn create_session( + swarms: &mut HashMap, + endpoint: &Endpoint, + gossip: &Gossip, + direct_state: &DirectServeState, + evt_tx: &async_channel::Sender, + session: SessionId, +) -> Result { + tracing::info!(%session, "creating collaboration network session"); + direct_state.attach_session(session).await; + let admission = SessionAdmission::generate(); + direct_state + .auth() + .configure_session(session, admission.clone()); + auth::install_local_admission(session, admission.clone()); + replace_swarm( + swarms, + SwarmHandle::spawn( + endpoint.clone(), + gossip.clone(), + direct_state.clone(), + session, + Vec::new(), + evt_tx.clone(), + )?, + ) + .await; + let inviter = reachable_endpoint_addr(endpoint).await; + tracing::debug!(%session, peer = %inviter.id, "collaboration network session created"); + Ok(TicketSeed { inviter, admission }) +} + +async fn join_session( + swarms: &mut HashMap, + endpoint: &Endpoint, + gossip: &Gossip, + direct_state: &DirectServeState, + evt_tx: &async_channel::Sender, + session: SessionId, + bootstrap: Vec, + admission: SessionAdmission, +) -> Result<()> { + let bootstrap_count = bootstrap.len(); + tracing::info!(%session, bootstrap_count, "joining collaboration network session"); + direct_state.attach_session(session).await; + direct_state + .auth() + .configure_session(session, admission.clone()); + auth::install_local_admission(session, admission); + replace_swarm( + swarms, + SwarmHandle::spawn(endpoint.clone(), gossip.clone(), direct_state.clone(), session, bootstrap, evt_tx.clone())?, + ) + .await; + tracing::debug!(%session, bootstrap_count, "collaboration network join subscribed"); + Ok(()) +} + +async fn replace_swarm(swarms: &mut HashMap, handle: SwarmHandle) { + let session = handle.session; + if let Some(previous) = swarms.insert(session, handle) { + tracing::debug!(%session, "replacing existing collaboration swarm"); + previous.stop().await; + } +} + +fn endpoint_addr(endpoint: &Endpoint) -> EndpointAddr { + endpoint.addr() +} + +fn log_direct_pull_result(kind: &'static str, session: crate::SessionId, candidate_count: usize, result: &Result>) { + match result { + Ok(bytes) => tracing::debug!(%session, kind, candidate_count, bytes = bytes.len(), "collaboration direct pull succeeded"), + Err(error) => tracing::warn!(%session, kind, candidate_count, error = %format_args!("{error:#}"), "collaboration direct pull failed"), + } +} + +async fn reachable_endpoint_addr(endpoint: &Endpoint) -> EndpointAddr { + let mut addr_watcher = endpoint.watch_addr(); + let wait = async { + loop { + if !addr_watcher.get().is_empty() { + return; + } + tokio::select! { + () = endpoint.online() => return, + updated = addr_watcher.updated() => { + if updated.is_err() { + return; + } + }, + } + } + }; + let _ = timeout(ENDPOINT_ADDR_READY_TIMEOUT, wait).await; + endpoint_addr(endpoint) +} diff --git a/crates/flowstate-collab/src/net/swarm.rs b/crates/flowstate-collab/src/net/swarm.rs new file mode 100644 index 00000000..8ba70c3c --- /dev/null +++ b/crates/flowstate-collab/src/net/swarm.rs @@ -0,0 +1,517 @@ +use std::time::Duration; + +use anyhow::{Context as _, Result}; +use async_channel::{Receiver, Sender}; +use iroh::{Endpoint, EndpointAddr, EndpointId, address_lookup::memory::MemoryLookup}; +use iroh_gossip::{ + api::{Event, GossipSender}, + net::Gossip, + proto::TopicId, +}; +use n0_future::StreamExt as _; + +use crate::{ + ids::SessionId, + proto_gossip::{self, GOSSIP_INLINE_LIMIT, GossipMsg}, +}; + +use super::{NetEvent, PublishPayload, direct::DirectServeState}; + +/// Generous bound for queued publish commands; senders backpressure via +/// `send().await` when the swarm loop falls behind (FS-074). +const COMMAND_CHANNEL_CAPACITY: usize = 1024; +/// Presence refreshes recur on a keepalive timer, so a lost frame heals +/// itself; still, transient broadcast failures get a small bounded retry +/// before the frame is dropped (FS-072). +const PRESENCE_PUBLISH_ATTEMPTS: u32 = 3; +const PRESENCE_PUBLISH_BACKOFF: Duration = Duration::from_millis(100); + +#[derive(Debug)] +enum SwarmCommand { + Publish(PublishPayload), + Stop, +} + +#[derive(Clone, Debug)] +pub struct SwarmHandle { + pub session: SessionId, + tx: Sender, +} + +impl SwarmHandle { + pub fn spawn( + endpoint: Endpoint, + gossip: Gossip, + direct_state: DirectServeState, + session: SessionId, + bootstrap: Vec, + evt_tx: Sender, + ) -> Result { + tracing::info!(%session, bootstrap_count = bootstrap.len(), "spawning collaboration gossip swarm"); + register_bootstrap_addrs(&endpoint, &bootstrap)?; + let bootstrap_ids = bootstrap.iter().map(endpoint_id).collect::>(); + let local_peer = endpoint_id(&endpoint.addr()); + let (tx, rx) = async_channel::bounded(COMMAND_CHANNEL_CAPACITY); + tokio::spawn(async move { + if let Err(error) = run_session(gossip, direct_state, session, local_peer, bootstrap_ids, rx, evt_tx.clone()).await { + tracing::error!(%session, error = %format_args!("{error:#}"), "collaboration gossip swarm failed"); + let _ = evt_tx + .send(NetEvent::SubscribeFailed { + session, + error: format!("{error:#}"), + }) + .await; + } + }); + Ok(Self { session, tx }) + } + + pub async fn publish(&self, payload: PublishPayload) -> Result<()> { + tracing::trace!(session = %self.session, payload_kind = payload.kind(), payload_bytes = payload.byte_len(), "queueing collaboration gossip publish"); + self + .tx + .send(SwarmCommand::Publish(payload)) + .await + .context("collaboration swarm task stopped") + } + + pub async fn stop(&self) { + tracing::debug!(session = %self.session, "stopping collaboration gossip swarm"); + let _ = self.tx.send(SwarmCommand::Stop).await; + } +} + +async fn run_session( + gossip: Gossip, + direct_state: DirectServeState, + session: SessionId, + local_peer: EndpointId, + bootstrap: Vec, + rx: Receiver, + evt_tx: Sender, +) -> Result<()> { + tracing::info!(%session, bootstrap_count = bootstrap.len(), "subscribing to collaboration gossip topic"); + let topic = gossip + .subscribe(topic_id(session), bootstrap) + .await + .context("subscribing to collaboration gossip topic failed")?; + tracing::info!(%session, "subscribed to collaboration gossip topic"); + let (sender, mut receiver) = topic.split(); + let mut dropped_coalescable: u64 = 0; + + loop { + tokio::select! { + command = rx.recv() => { + match command { + // FS-071/FS-072: publish failures are classified inside + // `handle_publish` and never abort the swarm loop. + Ok(SwarmCommand::Publish(payload)) => handle_publish(&sender, &direct_state, session, payload).await, + Ok(SwarmCommand::Stop) => { + tracing::debug!(%session, "collaboration gossip swarm stop received"); + break; + }, + Err(error) => { + tracing::debug!(%session, error = %error, "collaboration gossip swarm command channel closed"); + break; + }, + } + }, + event = receiver.next() => { + // Only unrecoverable subscription loss exits the loop (FS-071): + // a `None`/`Err` here means the gossip topic itself is gone. + let event = event.context("collaboration gossip receiver ended")??; + handle_event(event, session, local_peer, &evt_tx, &mut dropped_coalescable).await; + }, + } + } + + tracing::info!(%session, dropped_coalescable, "collaboration gossip swarm stopped"); + Ok(()) +} + +/// Minimal broadcast abstraction so publish-failure handling is unit-testable +/// without a live gossip topic. +trait FrameSink { + async fn send_frame(&self, frame: Vec, neighbors_only: bool) -> Result<()>; +} + +impl FrameSink for GossipSender { + async fn send_frame(&self, frame: Vec, neighbors_only: bool) -> Result<()> { + if neighbors_only { + self.broadcast_neighbors(frame.into()).await?; + } else { + self.broadcast(frame.into()).await?; + } + Ok(()) + } +} + +/// Publish one payload, classifying failures (FS-071/FS-072/FS-074): +/// - frame build/encode/blob-insert failures: log and drop the frame; +/// - transient broadcast failures: presence gets a bounded retry with +/// backoff, everything else is logged and dropped (updates recover via +/// anti-entropy digests, digests recur on a timer). +/// +/// This function never returns an error, so a publish failure cannot kill +/// the swarm loop. +async fn handle_publish(sink: &S, direct_state: &DirectServeState, session: SessionId, payload: PublishPayload) { + let payload_kind = payload.kind(); + let payload_bytes = payload.byte_len(); + let retriable = matches!(payload, PublishPayload::Presence(_)); + let (message, neighbors_only) = match payload { + PublishPayload::Update(bytes) => match update_message(direct_state, session, bytes).await { + Ok(message) => (message, false), + Err(error) => { + tracing::warn!(%session, payload_kind, payload_bytes, error = %format_args!("{error:#}"), "building collaboration gossip update failed; dropping frame"); + return; + }, + }, + PublishPayload::Presence(bytes) => (GossipMsg::Presence(bytes), false), + PublishPayload::Digest { vv } => (GossipMsg::Digest { session, vv }, true), + }; + + let mut frame = match if neighbors_only { + proto_gossip::encode(&message) + } else { + proto_gossip::encode_inline(&message) + } { + Ok(frame) => frame, + Err(error) => { + tracing::warn!(%session, payload_kind, payload_bytes, error = %format_args!("{error:#}"), "encoding collaboration gossip frame failed; dropping frame"); + return; + }, + }; + let frame_bytes = frame.len(); + tracing::trace!( + %session, + payload_kind, + payload_bytes, + gossip_kind = message.kind(), + gossip_payload_bytes = message.payload_len(), + frame_bytes, + neighbors_only, + "broadcasting collaboration gossip frame", + ); + + let attempts = if retriable { PRESENCE_PUBLISH_ATTEMPTS } else { 1 }; + for attempt in 1..=attempts { + // §perf: clone only when a retry may still follow; the final/only attempt + // (the common updates/digests case, attempts == 1) moves the frame out. + let to_send = if attempt < attempts { frame.clone() } else { std::mem::take(&mut frame) }; + match sink.send_frame(to_send, neighbors_only).await { + Ok(()) => { + tracing::trace!(%session, payload_kind, frame_bytes, neighbors_only, attempt, "collaboration gossip frame broadcast complete"); + return; + }, + Err(error) if attempt < attempts => { + tracing::debug!(%session, payload_kind, attempt, error = %format_args!("{error:#}"), "collaboration gossip broadcast failed; retrying"); + tokio::time::sleep(PRESENCE_PUBLISH_BACKOFF * attempt).await; + }, + Err(error) => { + tracing::warn!(%session, payload_kind, payload_bytes, attempt, error = %format_args!("{error:#}"), "collaboration gossip broadcast failed; dropping frame"); + }, + } + } +} + +async fn update_message(direct_state: &DirectServeState, session: SessionId, bytes: Vec) -> Result { + let update_bytes = bytes.len(); + // §perf: move the payload into the message instead of cloning it; on the too-big + // path we destructure the bytes back out for the blob insert rather than pre-cloning. + let inline = GossipMsg::Update(bytes); + if proto_gossip::encoded_len(&inline)? <= GOSSIP_INLINE_LIMIT { + tracing::trace!(%session, update_bytes, "collaboration update will be sent inline over gossip"); + return Ok(inline); + } + + let GossipMsg::Update(bytes) = inline else { + unreachable!("inline was just constructed as GossipMsg::Update"); + }; + let blob = direct_state.insert_blob(session, bytes).await?; + tracing::debug!(%session, ?blob, update_bytes, "collaboration update exceeded gossip limit; publishing blob announcement"); + Ok(GossipMsg::UpdateAvailable { + blob, + len: update_bytes as u64, + }) +} + +async fn handle_event(event: Event, session: SessionId, local_peer: EndpointId, evt_tx: &Sender, dropped_coalescable: &mut u64) { + match event { + Event::Received(message) if message.delivered_from == local_peer => { + tracing::trace!( + %session, + from = %message.delivered_from, + frame_bytes = message.content.len(), + "ignored self-delivered collaboration gossip frame", + ); + }, + Event::Received(message) => match proto_gossip::decode(&message.content) { + Ok(msg) => { + tracing::trace!( + %session, + from = %message.delivered_from, + gossip_kind = msg.kind(), + gossip_payload_bytes = msg.payload_len(), + frame_bytes = message.content.len(), + "received collaboration gossip frame", + ); + forward_event( + evt_tx, + NetEvent::Gossip { + session, + from: message.delivered_from, + msg, + }, + session, + dropped_coalescable, + ) + .await; + }, + Err(error) if proto_gossip::is_protocol_version_mismatch(&error) => { + forward_event( + evt_tx, + NetEvent::IncompatibleVersion { + session, + peer: message.delivered_from, + }, + session, + dropped_coalescable, + ) + .await; + }, + Err(error) => tracing::warn!( + %session, + from = %message.delivered_from, + frame_bytes = message.content.len(), + error = %format_args!("{error:#}"), + "ignored malformed collaboration gossip frame", + ), + }, + Event::NeighborUp(peer) => { + tracing::info!(%session, peer = %peer, "collaboration gossip neighbor up"); + forward_event(evt_tx, NetEvent::NeighborUp { session, peer }, session, dropped_coalescable).await; + }, + Event::NeighborDown(peer) => { + tracing::info!(%session, peer = %peer, "collaboration gossip neighbor down"); + forward_event(evt_tx, NetEvent::NeighborDown { session, peer }, session, dropped_coalescable).await; + }, + Event::Lagged => { + tracing::warn!(%session, "collaboration gossip receiver lagged"); + forward_event(evt_tx, NetEvent::GossipLagged { session }, session, dropped_coalescable).await; + }, + } +} + +/// Forward an event to the UI-facing channel without blocking the network +/// task on coalescable traffic (FS-074): presence/digest frames recur, so +/// when the channel is full they are dropped and counted instead of stalling +/// gossip processing. Non-coalescable events (document updates, membership +/// changes) apply backpressure via `send().await`. +async fn forward_event(evt_tx: &Sender, event: NetEvent, session: SessionId, dropped_coalescable: &mut u64) { + if is_coalescable(&event) { + match evt_tx.try_send(event) { + Ok(()) => {}, + Err(async_channel::TrySendError::Full(_)) => { + *dropped_coalescable += 1; + tracing::warn!( + %session, + dropped_total = *dropped_coalescable, + "collaboration event channel is full; dropped a coalescable frame", + ); + }, + Err(async_channel::TrySendError::Closed(_)) => { + tracing::debug!(%session, "collaboration event channel closed; dropping event"); + }, + } + } else { + let _ = evt_tx.send(event).await; + } +} + +fn is_coalescable(event: &NetEvent) -> bool { + matches!( + event, + NetEvent::Gossip { + msg: GossipMsg::Presence(_) | GossipMsg::Digest { .. }, + .. + } | NetEvent::GossipLagged { .. } + ) +} + +fn register_bootstrap_addrs(endpoint: &Endpoint, bootstrap: &[EndpointAddr]) -> Result<()> { + if bootstrap.is_empty() { + return Ok(()); + } + tracing::debug!(bootstrap_count = bootstrap.len(), "registering collaboration bootstrap addresses"); + endpoint + .address_lookup()? + .add(MemoryLookup::from_endpoint_info(bootstrap.iter().cloned())); + Ok(()) +} + +fn endpoint_id(addr: &EndpointAddr) -> EndpointId { + addr.id +} + +fn topic_id(session: SessionId) -> TopicId { + TopicId::from_bytes(*session.as_bytes()) +} + +#[cfg(test)] +mod tests { + use std::sync::{ + Arc, + atomic::{AtomicU32, Ordering}, + }; + + use anyhow::{Result, anyhow}; + use iroh::SecretKey; + use iroh_gossip::{ + api::{Event, Message}, + proto::DeliveryScope, + }; + + use crate::{ + SessionId, + net::{NetEvent, PublishPayload, direct::DirectServeState}, + proto_gossip::{self, GossipMsg}, + }; + + use super::{FrameSink, PRESENCE_PUBLISH_ATTEMPTS, handle_event, handle_publish}; + + struct FailingSink { + attempts: Arc, + } + + impl FrameSink for FailingSink { + #[allow(clippy::unused_async_trait_impl, reason = "test mock must match the async FrameSink trait signature")] + async fn send_frame(&self, _frame: Vec, _neighbors_only: bool) -> Result<()> { + self.attempts.fetch_add(1, Ordering::SeqCst); + Err(anyhow!("simulated gossip publish failure")) + } + } + + fn received(frame: Vec, from: iroh::EndpointId) -> Event { + Event::Received(Message { + content: frame.into(), + scope: DeliveryScope::Neighbors, + delivered_from: from, + }) + } + + #[tokio::test] + async fn version_mismatch_emits_incompatible_version_once() { + let session = SessionId::new(); + let local_peer = SecretKey::generate().public(); + let peer = SecretKey::generate().public(); + let mut frame = proto_gossip::encode(&GossipMsg::Presence(Vec::new())).expect("valid gossip frame"); + frame[0] = frame[0].wrapping_add(1); + + let (evt_tx, evt_rx) = async_channel::unbounded(); + let mut dropped = 0; + handle_event(received(frame, peer), session, local_peer, &evt_tx, &mut dropped).await; + + match evt_rx.recv().await.expect("incompatible-version event") { + NetEvent::IncompatibleVersion { + session: event_session, + peer: event_peer, + } => { + assert_eq!(event_session, session); + assert_eq!(event_peer, peer); + }, + event => panic!("unexpected event: {event:?}"), + } + assert!(evt_rx.try_recv().is_err()); + } + + #[tokio::test] + async fn self_delivered_gossip_frame_is_ignored() { + let session = SessionId::new(); + let local_peer = SecretKey::generate().public(); + let frame = proto_gossip::encode(&GossipMsg::Update(vec![1, 2, 3])).expect("valid gossip frame"); + + let (evt_tx, evt_rx) = async_channel::unbounded(); + let mut dropped = 0; + handle_event(received(frame, local_peer), session, local_peer, &evt_tx, &mut dropped).await; + + assert!(evt_rx.try_recv().is_err()); + } + + /// FS-071/FS-072: a failing broadcast must not surface as an error (which + /// previously killed the swarm loop). Presence frames get a bounded retry; + /// other payloads are dropped after one attempt. + #[tokio::test] + async fn publish_failure_is_swallowed_with_bounded_presence_retry() { + let session = SessionId::new(); + let direct_state = DirectServeState::default(); + + let attempts = Arc::new(AtomicU32::new(0)); + let sink = FailingSink { attempts: attempts.clone() }; + handle_publish(&sink, &direct_state, session, PublishPayload::Presence(vec![1, 2, 3])).await; + assert_eq!(attempts.load(Ordering::SeqCst), PRESENCE_PUBLISH_ATTEMPTS); + + attempts.store(0, Ordering::SeqCst); + handle_publish(&sink, &direct_state, session, PublishPayload::Update(vec![4, 5, 6])).await; + assert_eq!(attempts.load(Ordering::SeqCst), 1); + + attempts.store(0, Ordering::SeqCst); + handle_publish(&sink, &direct_state, session, PublishPayload::Digest { vv: vec![7] }).await; + assert_eq!(attempts.load(Ordering::SeqCst), 1); + } + + /// Protocol v3 has symmetric editors: update and presence frames from a + /// subscribed peer both reach the session projection. + #[tokio::test] + async fn editor_updates_and_presence_are_forwarded() { + let session = SessionId::new(); + let local_peer = SecretKey::generate().public(); + let editor = SecretKey::generate().public(); + + let (evt_tx, evt_rx) = async_channel::unbounded(); + let mut dropped = 0; + let update = proto_gossip::encode(&GossipMsg::Update(vec![1])).expect("valid gossip frame"); + handle_event(received(update, editor), session, local_peer, &evt_tx, &mut dropped).await; + assert!(matches!( + evt_rx.try_recv(), + Ok(NetEvent::Gossip { + msg: GossipMsg::Update(_), + .. + }) + )); + + let presence = proto_gossip::encode(&GossipMsg::Presence(vec![2])).expect("valid gossip frame"); + handle_event(received(presence, editor), session, local_peer, &evt_tx, &mut dropped).await; + assert!( + matches!( + evt_rx.try_recv(), + Ok(NetEvent::Gossip { + msg: GossipMsg::Presence(_), + .. + }) + ), + "editor presence must flow" + ); + } + + /// FS-074: when the event channel is full, coalescable frames (presence, + /// digest) are dropped and counted instead of blocking the network task; + /// the queued event is preserved. + #[tokio::test] + async fn coalescable_frames_are_dropped_when_the_event_channel_is_full() { + let session = SessionId::new(); + let local_peer = SecretKey::generate().public(); + let peer = SecretKey::generate().public(); + + let (evt_tx, evt_rx) = async_channel::bounded(1); + evt_tx + .try_send(NetEvent::GossipLagged { session }) + .expect("prefill event channel"); + + let mut dropped = 0; + let presence = proto_gossip::encode(&GossipMsg::Presence(vec![1])).expect("valid gossip frame"); + handle_event(received(presence, peer), session, local_peer, &evt_tx, &mut dropped).await; + assert_eq!(dropped, 1); + assert!(matches!(evt_rx.try_recv(), Ok(NetEvent::GossipLagged { .. }))); + assert!(evt_rx.try_recv().is_err()); + } +} diff --git a/crates/flowstate-collab/src/net/wire_compression.rs b/crates/flowstate-collab/src/net/wire_compression.rs new file mode 100644 index 00000000..16cbba48 --- /dev/null +++ b/crates/flowstate-collab/src/net/wire_compression.rs @@ -0,0 +1,351 @@ +//! §perf: size-conditional zstd compression for the collab direct-transfer wire +//! payloads (full snapshots and incremental update batches). Two empirically +//! validated profiles, selected purely by payload size: +//! +//! - **Small payloads** (`< SMALL_PAYLOAD_MAX`: updates, small docs) → zstd with +//! the shipped 256 KiB dictionary. A small CRDT delta shares structure with the +//! dictionary corpus, giving ~2.52× (+27% over plain zstd). +//! - **Big payloads** (`>= SMALL_PAYLOAD_MAX`: full large-document snapshots) → +//! zstd with long-distance matching and NO dictionary, giving ~2.96× with faster +//! decode. The dictionary actively HURTS big files (~9% worse at L19), so it is +//! off there. +//! +//! The chosen [`WireCodec`] plus the wire/uncompressed lengths ride on +//! [`DirectResponseHeader::Ok`](crate::proto_direct::DirectResponseHeader) so the +//! receiver decodes correctly and preallocates the decode buffer up front. +//! +//! The prepared encoder/decoder dictionaries are digested ONCE (lazy statics) and +//! reused for every payload — never re-digested per call. + +use std::sync::{Arc, LazyLock, Mutex, MutexGuard}; + +use anyhow::{Context, Result, ensure}; +use twox_hash::XxHash3_128; +use zstd::bulk::{Compressor, Decompressor}; +use zstd::dict::{DecoderDictionary, EncoderDictionary}; +use zstd::stream::raw::{CParameter, DParameter}; + +use crate::proto_direct::{MAX_PAYLOAD_LEN, WireCodec}; + +/// The 256 KiB dictionary shipped in-binary (built from a Flowstate-document +/// corpus — see the `flowstate-dictbuild` harness). Both peers embed the identical +/// bytes, so a `ZstdDict` frame one peer produces always decodes on the other. +static FLOWSTATE_DICT: &[u8] = include_bytes!("../../assets/flowstate-cards-256k.zdict"); + +/// The zstd dictionary id baked into [`FLOWSTATE_DICT`]. The dict path is only safe +/// while both peers ship this id; a peer with a different embedded dictionary would +/// emit frames this side cannot decode. Recorded here so a swapped dictionary is a +/// visible constant change (see the `dict_id_matches_shipped_dictionary` test). +pub const FLOWSTATE_DICT_ID: u32 = 1_296_305_636; + +/// Payloads `>=` this size take the long-distance-matching profile (no dictionary); +/// smaller ones take the dictionary profile. ~4 MiB is the empirical crossover +/// where the dictionary stops helping and long-range matching starts to. +const SMALL_PAYLOAD_MAX: usize = 4 * 1024 * 1024; + +/// Below this, zstd framing overhead outweighs any gain; stream verbatim. +const MIN_COMPRESS_LEN: usize = 1024; + +/// Dictionary-profile level. Small payloads compress fast even at 19. +const DICT_LEVEL: i32 = 19; + +/// Long-profile level for LIVE (uncached) serving. Level 3 sustains ~200 MB/s and +/// still beats plain L19 on big files, whereas L19 long compresses at only +/// ~3–7 MB/s — far too slow to compress a multi-hundred-MB snapshot per request. +/// Raise to 19 only behind a compress-once cache. +const LONG_LEVEL: i32 = 3; + +/// Long-distance-matching window (2^27 = 128 MiB): large enough for big snapshots +/// while bounding decoder memory. The decoder must permit at least this window. +const LONG_WINDOW_LOG: u32 = 27; + +/// Prepared once, reused for every payload — the CDict/DDict digest is never +/// recomputed per call. +static ENCODER_DICT: LazyLock> = LazyLock::new(|| EncoderDictionary::copy(FLOWSTATE_DICT, DICT_LEVEL)); +static DECODER_DICT: LazyLock> = LazyLock::new(|| DecoderDictionary::copy(FLOWSTATE_DICT)); + +/// Whether the embedded dictionary's zstd id matches the recorded +/// [`FLOWSTATE_DICT_ID`]. Guards against a future edit swapping the `.zdict` file +/// without updating the constant: on mismatch the dictionary path is disabled +/// (small payloads fall back to the no-dictionary long profile) rather than +/// emitting `ZstdDict` frames that peers pinning the recorded id could not decode. +static DICT_ID_OK: LazyLock = LazyLock::new(|| { + let id = zstd::zstd_safe::get_dict_id_from_dict(FLOWSTATE_DICT).map(std::num::NonZeroU32::get); + let ok = id == Some(FLOWSTATE_DICT_ID); + if !ok { + tracing::error!( + ?id, + expected = FLOWSTATE_DICT_ID, + "shipped zstd dictionary id does not match FLOWSTATE_DICT_ID; disabling the dictionary compression path" + ); + } + ok +}); + +/// Compressed-payload cache. Compressing a snapshot at `DICT_LEVEL` (19) is the one +/// slow step (~6 MB/s), and the exact same snapshot is re-served to every peer that +/// joins the same document version — so we compress once and reuse. Keyed by a +/// 128-bit content hash of the payload, which makes it self-invalidating: any edit +/// changes the snapshot bytes → new key → miss → recompress. A stale entry is simply +/// never hit again and ages out via LRU. +struct CacheEntry { + key: u128, + codec: WireCodec, + bytes: Arc<[u8]>, +} + +/// Small bound: a serving peer hosts a handful of live sessions, and we only need the +/// current version of each. Also capped by total bytes so a few big snapshots can't +/// balloon memory. +const CACHE_MAX_ENTRIES: usize = 8; +const CACHE_MAX_BYTES: usize = 512 * 1024 * 1024; + +/// LRU by position: least-recently-used at the front, most-recent at the back. +static COMPRESSED_CACHE: LazyLock>> = LazyLock::new(|| Mutex::new(Vec::new())); + +fn lock_cache() -> MutexGuard<'static, Vec> { + // A poisoned cache must never take down serving — recover the inner value. + COMPRESSED_CACHE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn cache_get(key: u128) -> Option<(WireCodec, Arc<[u8]>)> { + let mut cache = lock_cache(); + let pos = cache.iter().position(|entry| entry.key == key)?; + let entry = cache.remove(pos); + let hit = (entry.codec, Arc::clone(&entry.bytes)); + cache.push(entry); // move to back = most-recently-used + drop(cache); // release the lock before returning; the Arc keeps the bytes alive + Some(hit) +} + +fn cache_put(key: u128, codec: WireCodec, bytes: Arc<[u8]>) { + let mut cache = lock_cache(); + if cache.iter().any(|entry| entry.key == key) { + return; // a concurrent serve already inserted this version + } + cache.push(CacheEntry { key, codec, bytes }); + let mut total: usize = cache.iter().map(|entry| entry.bytes.len()).sum(); + while cache.len() > CACHE_MAX_ENTRIES || (cache.len() > 1 && total > CACHE_MAX_BYTES) { + let evicted = cache.remove(0); // evict least-recently-used + total -= evicted.bytes.len(); + } + drop(cache); // release the lock promptly to minimize serve contention +} + +/// §perf: an owned handle to the bytes to stream on the wire, without copying the +/// compressed payload. Either a slice borrowed from the caller's `payload` (verbatim +/// path) or a shared reference to the cached compressed buffer (same `Arc` the cache +/// holds). Replaces the old `Cow<[u8]>` return, which forced a full `.to_vec()` copy +/// on both the cache-hit and fresh-compress paths. +pub enum WireBytes<'a> { + Borrowed(&'a [u8]), + Shared(Arc<[u8]>), +} + +impl WireBytes<'_> { + #[must_use] + pub fn as_slice(&self) -> &[u8] { + match self { + WireBytes::Borrowed(slice) => slice, + WireBytes::Shared(shared) => shared, + } + } +} + +/// Compress `payload` for the wire, choosing the codec by size. Returns the codec +/// and the bytes to stream: [`WireCodec::None`] borrowing `payload` verbatim when +/// compression is skipped (fast link, tiny payload) or fails to shrink; otherwise +/// the shared compressed bytes. Results are cached by content hash and the dictionary +/// is never re-digested. +#[must_use] +pub fn compress_for_wire(payload: &[u8], link_is_fast: bool) -> (WireCodec, WireBytes<'_>) { + // On a very fast (LAN/localhost) link the decode (~700–900 MB/s) becomes the + // ceiling, so compression buys nothing; and tiny payloads aren't worth framing. + if link_is_fast || payload.len() < MIN_COMPRESS_LEN { + return (WireCodec::None, WireBytes::Borrowed(payload)); + } + // Hashing (xxh3, ~10 GB/s) is a rounding error next to compression, and a cache hit + // skips the whole compress — the win that lets us keep level 19 without per-join cost. + let key = XxHash3_128::oneshot(payload); + if let Some((codec, bytes)) = cache_get(key) { + // §perf: hand back the cached Arc directly instead of copying it out. + return (codec, WireBytes::Shared(bytes)); + } + let compressed = if payload.len() < SMALL_PAYLOAD_MAX && *DICT_ID_OK { + compress_with_dict(payload).map(|bytes| (WireCodec::ZstdDict, bytes)) + } else { + compress_long(payload).map(|bytes| (WireCodec::ZstdLong, bytes)) + }; + match compressed { + // Only adopt compression when it actually shrank the payload. + Ok((codec, bytes)) if bytes.len() < payload.len() => { + // §perf: one Arc, shared by both the cache and the caller — no `.to_vec()` copy. + let shared: Arc<[u8]> = Arc::from(bytes); + cache_put(key, codec, Arc::clone(&shared)); + (codec, WireBytes::Shared(shared)) + }, + Ok(_) => (WireCodec::None, WireBytes::Borrowed(payload)), + Err(error) => { + tracing::warn!(error = %format_args!("{error:#}"), payload_bytes = payload.len(), "wire compression failed; sending payload uncompressed"); + (WireCodec::None, WireBytes::Borrowed(payload)) + }, + } +} + +fn compress_with_dict(payload: &[u8]) -> Result> { + let mut compressor = Compressor::with_prepared_dictionary(&ENCODER_DICT).context("preparing dictionary compressor")?; + compressor + .compress(payload) + .context("zstd dictionary compression") +} + +fn compress_long(payload: &[u8]) -> Result> { + let mut compressor = Compressor::new(LONG_LEVEL).context("creating long-mode compressor")?; + compressor + .set_parameter(CParameter::EnableLongDistanceMatching(true)) + .context("enabling long-distance matching")?; + compressor + .set_parameter(CParameter::WindowLog(LONG_WINDOW_LOG)) + .context("setting window log")?; + compressor + .compress(payload) + .context("zstd long-mode compression") +} + +/// Decompress a wire payload streamed under `codec` back to `uncompressed_len` +/// bytes (which the sender declared in the response header). [`WireCodec::None`] +/// payloads are returned verbatim. Reuses the prepared decoder dictionary and +/// raises the decoder window for long frames. +pub fn decompress_from_wire(codec: WireCodec, wire: Vec, uncompressed_len: usize) -> Result> { + ensure!( + uncompressed_len <= MAX_PAYLOAD_LEN, + "declared uncompressed length {uncompressed_len} exceeds {MAX_PAYLOAD_LEN} bytes" + ); + let decoded = match codec { + WireCodec::None => { + ensure!( + wire.len() == uncompressed_len, + "uncompressed payload length {} does not match declared {uncompressed_len}", + wire.len() + ); + return Ok(wire); + }, + WireCodec::ZstdDict => { + let mut decompressor = Decompressor::with_prepared_dictionary(&DECODER_DICT).context("preparing dictionary decompressor")?; + decompressor + .decompress(&wire, uncompressed_len) + .context("zstd dictionary decompression")? + }, + WireCodec::ZstdLong => { + let mut decompressor = Decompressor::new().context("creating long-mode decompressor")?; + decompressor + .set_parameter(DParameter::WindowLogMax(LONG_WINDOW_LOG)) + .context("raising decoder window for long frames")?; + decompressor + .decompress(&wire, uncompressed_len) + .context("zstd long-mode decompression")? + }, + }; + ensure!( + decoded.len() == uncompressed_len, + "decompressed length {} does not match declared {uncompressed_len}", + decoded.len() + ); + Ok(decoded) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn round_trip(payload: &[u8]) -> Vec { + let (codec, wire) = compress_for_wire(payload, false); + decompress_from_wire(codec, wire.as_slice().to_vec(), payload.len()).expect("decompress") + } + + #[test] + fn dict_id_matches_shipped_dictionary() { + // Guards against a swapped dictionary file whose id would make our frames + // undecodable on peers still shipping the recorded id. + let id = zstd::zstd_safe::get_dict_id_from_dict(FLOWSTATE_DICT).map(std::num::NonZeroU32::get); + assert_eq!(id, Some(FLOWSTATE_DICT_ID), "shipped dictionary id changed"); + assert!(*DICT_ID_OK, "dictionary compression path should be enabled"); + } + + #[test] + fn small_payload_round_trips_via_dictionary() { + let payload = b"{\"paragraph\":\"the quick brown fox\"}".repeat(64); + assert!(payload.len() < SMALL_PAYLOAD_MAX); + let (codec, _) = compress_for_wire(&payload, false); + assert_eq!(codec, WireCodec::ZstdDict); + assert_eq!(round_trip(&payload), payload); + } + + #[test] + fn big_payload_round_trips_via_long_no_dict() { + // Compressible payload above the 4 MiB crossover so the long profile is chosen + // and long-distance matching engages (~47 B block × 1024 × 100 ≈ 4.8 MiB). + let block = b"paragraph metadata cursor boundary run styles ".repeat(1024); + let payload = block.repeat(100); + assert!(payload.len() >= SMALL_PAYLOAD_MAX); + let (codec, wire) = compress_for_wire(&payload, false); + assert_eq!(codec, WireCodec::ZstdLong); + assert!(wire.as_slice().len() < payload.len()); + assert_eq!(round_trip(&payload), payload); + } + + #[test] + fn fast_link_and_tiny_payloads_skip_compression() { + let payload = b"hello world".repeat(1024); + assert_eq!(compress_for_wire(&payload, true).0, WireCodec::None); + assert_eq!(compress_for_wire(b"tiny", false).0, WireCodec::None); + } + + #[test] + fn repeat_compression_is_consistent() { + // The second call for the same content is served from the cache; it must return + // the exact same codec + bytes (a corrupt cache would diverge here) and round-trip. + let payload = b"{\"run\":\"cache probe content that is reasonably unique\"}".repeat(64); + assert!(payload.len() >= MIN_COMPRESS_LEN && payload.len() < SMALL_PAYLOAD_MAX); + let (codec1, wire1) = compress_for_wire(&payload, false); + let (codec2, wire2) = compress_for_wire(&payload, false); + assert_eq!(codec1, WireCodec::ZstdDict); + assert_eq!(codec2, codec1); + assert_eq!(wire1.as_slice(), wire2.as_slice(), "cached bytes must match freshly compressed bytes"); + assert_eq!(round_trip(&payload), payload); + } + + #[test] + fn cache_bounds_entries() { + // Deterministic invariant (unaffected by parallel tests sharing the global cache): + // overfilling must never leave more than the entry cap. Keys are placed high in the + // space so they don't collide with content hashes other tests insert. + let base: u128 = 0xC0DE << 112; + for i in 0..(CACHE_MAX_ENTRIES as u128 + 4) { + cache_put(base | i, WireCodec::ZstdLong, Arc::from(vec![0u8; 32])); + } + assert!( + lock_cache().len() <= CACHE_MAX_ENTRIES, + "eviction must bound the cache to CACHE_MAX_ENTRIES" + ); + } + + #[test] + fn incompressible_payload_falls_back_to_none() { + // High-entropy xorshift bytes don't shrink under zstd → keep verbatim. + let mut state = 0x2545_F491_4F6C_DD1Du64; + let payload: Vec = (0..16_384) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state & 0xff) as u8 + }) + .collect(); + let (codec, wire) = compress_for_wire(&payload, false); + assert_eq!(codec, WireCodec::None); + assert_eq!(wire.as_slice(), payload.as_slice()); + } +} diff --git a/crates/flowstate-collab/src/presence.rs b/crates/flowstate-collab/src/presence.rs new file mode 100644 index 00000000..eeea19de --- /dev/null +++ b/crates/flowstate-collab/src/presence.rs @@ -0,0 +1,471 @@ +//! Presence state and roster derivation. + +use std::sync::{Arc, Mutex}; + +use anyhow::{Context as _, Result}; +use loro::{ + LoroValue, Subscription, + awareness::{EphemeralStore, EphemeralStoreEvent}, +}; +use serde::{Deserialize, Serialize}; + +use crate::ids::{PALETTE, PeerId, SessionId}; + +pub const PRESENCE_TIMEOUT_MS: i64 = 30_000; +pub const PRESENCE_KEEPALIVE_SECS: u64 = 10; + +/// Part B (FS-080) presence hardening caps. Display names and selection cursor +/// encodings are attacker-controlled once they arrive over gossip, so they are +/// clamped on encode and rejected past these bounds on apply. +pub const MAX_PRESENCE_NAME_BYTES: usize = 64; +pub const MAX_PRESENCE_CURSOR_BYTES: usize = 256; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct PresenceState { + pub name: String, + pub color_rgb: u32, + pub selection: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct PresenceSelection { + pub anchor: SelectionEndpoint, + pub head: SelectionEndpoint, + pub direction: SelectionDirection, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct SelectionEndpoint { + pub cursor: Vec, + pub affinity: SelectionAffinity, + pub visual_gravity: VisualGravity, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum SelectionAffinity { + Before, + After, + Neutral, +} + +/// §16: the presence wire enum mirrors `gpui_flowtext::SelectionAffinity` +/// variant-for-variant, so the editor's stored affinity converts directly +/// without re-deriving a side from selection direction. Conversion is +/// centralized here rather than scattered across the runtime. +impl From for SelectionAffinity { + fn from(value: gpui_flowtext::SelectionAffinity) -> Self { + match value { + gpui_flowtext::SelectionAffinity::Before => Self::Before, + gpui_flowtext::SelectionAffinity::After => Self::After, + gpui_flowtext::SelectionAffinity::Neutral => Self::Neutral, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum VisualGravity { + Upstream, + Downstream, + Neutral, +} + +/// §16: the presence wire enum mirrors `gpui_flowtext::VisualGravity` +/// variant-for-variant, so the editor's stored gravity converts directly. +impl From for VisualGravity { + fn from(value: gpui_flowtext::VisualGravity) -> Self { + match value { + gpui_flowtext::VisualGravity::Upstream => Self::Upstream, + gpui_flowtext::VisualGravity::Downstream => Self::Downstream, + gpui_flowtext::VisualGravity::Neutral => Self::Neutral, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum SelectionDirection { + Forward, + Backward, + None, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct RosterEntry { + pub key: String, + pub name: String, + pub color_rgb: u32, + pub selection: Option, +} + +#[derive(Clone, Debug)] +pub struct PresenceStore { + store: EphemeralStore, + self_key: String, + self_color: u32, +} + +impl PresenceStore { + #[must_use] + pub fn new(self_peer: &PeerId) -> Self { + Self::new_with_color(self_peer, color_for_peer(self_peer)) + } + + #[must_use] + pub fn new_with_color(self_peer: &PeerId, color_rgb: u32) -> Self { + Self { + store: EphemeralStore::new(PRESENCE_TIMEOUT_MS), + self_key: peer_key(self_peer), + self_color: color_rgb & 0x00ff_ffff, + } + } + + #[must_use] + pub fn self_key(&self) -> &str { + &self.self_key + } + + #[must_use] + pub const fn self_color(&self) -> u32 { + self.self_color + } + + pub fn set_self(&self, state: &PresenceState) -> Result<()> { + let sanitized = sanitize_presence_state(state); + self + .store + .set(&self.self_key, LoroValue::Binary(encode_state(&sanitized)?.into())); + Ok(()) + } + + pub fn delete_self(&self) { + self.store.delete(&self.self_key); + } + + #[must_use] + pub fn encode_self(&self) -> Vec { + self.store.encode(&self.self_key) + } + + #[must_use] + pub fn encode_all(&self) -> Vec { + self.store.encode_all() + } + + /// Apply a presence frame delivered by `from`, enforcing Part B trust rules: + /// the frame may only add, overwrite, or remove `from`'s own entry (no + /// impersonation or griefing of other peers), and its display name / selection + /// cursors must be within [`MAX_PRESENCE_NAME_BYTES`] / [`MAX_PRESENCE_CURSOR_BYTES`]. + /// A violating frame is dropped (returns `Ok(())`); only a genuinely + /// undecodable store payload is an error. + pub fn apply_from(&self, from: &PeerId, bytes: &[u8]) -> Result<()> { + let expected_key = peer_key(from); + + // Replay into a scratch store seeded with our current state to learn which + // keys this frame would add, overwrite, or remove. Presence is a per-peer, + // self-authored signal: any frame touching a key other than the delivering + // peer's own is impersonation (or a relayed frame) and is dropped whole. + let scratch = EphemeralStore::new(PRESENCE_TIMEOUT_MS); + if let Err(error) = scratch.apply(&self.store.encode_all()) { + tracing::warn!(error = %error, "seeding collaboration presence guard store failed"); + } + let touched = Arc::new(Mutex::new(Vec::::new())); + let recorder = Arc::clone(&touched); + let subscription = scratch.subscribe(Box::new(move |event: &EphemeralStoreEvent| { + if let Ok(mut keys) = recorder.lock() { + keys.extend(event.added.iter().cloned()); + keys.extend(event.updated.iter().cloned()); + keys.extend(event.removed.iter().cloned()); + } + true + })); + scratch + .apply(bytes) + .map_err(|error| anyhow::anyhow!(error.to_string())) + .context("decoding collaboration presence update failed")?; + drop(subscription); + + let touched = touched + .lock() + .map_err(|_| anyhow::anyhow!("collaboration presence guard lock poisoned"))?; + for key in touched.iter() { + if key != &expected_key { + tracing::warn!(from = %from, key = %key, "dropped collaboration presence frame that mutates another peer's entry"); + return Ok(()); + } + } + drop(touched); + + // Reject an oversized/malformed value for the delivering peer's own key + // rather than importing it; sanitize-on-encode only binds honest peers. + if let Some(LoroValue::Binary(value)) = scratch.get_all_states().get(&expected_key).cloned() { + match decode_state(value.as_ref()) { + Ok(state) if presence_state_within_caps(&state) => {}, + Ok(_) => { + tracing::warn!(from = %from, "dropped collaboration presence frame exceeding display-name/selection caps"); + return Ok(()); + }, + Err(error) => { + tracing::warn!(from = %from, error = %format_args!("{error:#}"), "dropped undecodable collaboration presence frame"); + return Ok(()); + }, + } + } + + self + .store + .apply(bytes) + .map_err(|error| anyhow::anyhow!(error.to_string())) + .context("applying collaboration presence update failed") + } + + pub fn remove_outdated(&self) { + self.store.remove_outdated(); + } + + #[must_use] + pub fn roster(&self) -> Vec { + let mut entries = self + .store + .get_all_states() + .into_iter() + .filter_map(|(key, value)| roster_entry_from_value(key, value)) + .collect::>(); + entries.sort_by(|left, right| { + left + .name + .cmp(&right.name) + .then_with(|| left.key.cmp(&right.key)) + }); + entries + } + + pub fn subscribe_local_updates(&self, callback: impl Fn(&Vec) -> bool + Send + Sync + 'static) -> Subscription { + self.store.subscribe_local_updates(Box::new(callback)) + } + + pub fn subscribe(&self, callback: impl Fn(&EphemeralStoreEvent) -> bool + Send + Sync + 'static) -> Subscription { + self.store.subscribe(Box::new(callback)) + } +} + +pub fn encode_state(state: &PresenceState) -> Result> { + postcard::to_stdvec(state).context("encoding collaboration presence state failed") +} + +pub fn decode_state(bytes: &[u8]) -> Result { + postcard::from_bytes(bytes).context("decoding collaboration presence state failed") +} + +#[must_use] +pub fn peer_key(peer: &PeerId) -> String { + // §perf: manual nibble→hex lookup avoids invoking the fmt machinery twice per byte. + const HEX: &[u8; 16] = b"0123456789abcdef"; + let bytes = peer.as_bytes(); + let mut key = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + key.push(HEX[(byte >> 4) as usize] as char); + key.push(HEX[(byte & 0x0f) as usize] as char); + } + key +} + +#[must_use] +pub fn color_for_peer(peer: &PeerId) -> u32 { + PALETTE[SessionId::color_index_for_peer(peer)] +} + +fn roster_entry_from_value(key: String, value: LoroValue) -> Option { + let LoroValue::Binary(bytes) = value else { + return None; + }; + let state = decode_state(bytes.as_ref()).ok()?; + Some(RosterEntry { + color_rgb: state.color_rgb & 0x00ff_ffff, + key, + name: state.name, + selection: state.selection, + }) +} + +/// Clamp a presence state to the Part B caps before it is broadcast: strip +/// control characters, truncate the display name, and drop a selection whose +/// cursor encodings exceed the cap (a corrupt/oversized cursor is unusable). +fn sanitize_presence_state(state: &PresenceState) -> PresenceState { + let selection = match &state.selection { + Some(selection) if selection_within_caps(selection) => Some(selection.clone()), + _ => None, + }; + PresenceState { + name: sanitize_display_name(&state.name), + color_rgb: state.color_rgb & 0x00ff_ffff, + selection, + } +} + +fn sanitize_display_name(name: &str) -> String { + let cleaned: String = name.chars().filter(|c| !c.is_control()).collect(); + truncate_to_bytes(&cleaned, MAX_PRESENCE_NAME_BYTES) +} + +fn truncate_to_bytes(text: &str, max_bytes: usize) -> String { + if text.len() <= max_bytes { + return text.to_string(); + } + let mut end = max_bytes; + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + text[..end].to_string() +} + +/// Whether a decoded remote presence state satisfies the Part B caps. Applied +/// as a rejection gate on `apply_from` (we cannot rewrite raw store bytes in +/// place without re-broadcasting, so an out-of-cap frame is dropped instead). +fn presence_state_within_caps(state: &PresenceState) -> bool { + state.name.len() <= MAX_PRESENCE_NAME_BYTES + && !state.name.chars().any(|c| c.is_control()) + && state.selection.as_ref().is_none_or(selection_within_caps) +} + +fn selection_within_caps(selection: &PresenceSelection) -> bool { + selection.anchor.cursor.len() <= MAX_PRESENCE_CURSOR_BYTES && selection.head.cursor.len() <= MAX_PRESENCE_CURSOR_BYTES +} + +#[cfg(test)] +mod tests { + use iroh::SecretKey; + + use super::*; + + fn test_peer(seed: u8) -> PeerId { + SecretKey::from_bytes(&[seed; 32]).public() + } + + fn named_state(name: &str) -> PresenceState { + PresenceState { + name: name.to_string(), + color_rgb: 0x3b82f6, + selection: None, + } + } + + #[test] + fn presence_frame_is_bound_to_the_delivering_peer() { + let peer_a = test_peer(1); + let peer_b = test_peer(2); + let peer_c = test_peer(3); + + let store_a = PresenceStore::new(&peer_a); + store_a + .set_self(&named_state("Ada")) + .expect("set self presence"); + let frame = store_a.encode_self(); + + // Delivered by its author, A's frame is accepted into B's roster. + let honest = PresenceStore::new(&peer_b); + honest + .apply_from(&peer_a, &frame) + .expect("honest presence applies"); + assert!( + honest + .roster() + .iter() + .any(|entry| entry.key == peer_key(&peer_a)), + "author-delivered presence must populate the roster", + ); + + // The identical frame delivered by an impersonating peer C is rejected: it + // would mutate A's entry, which C is not authorized to touch. + let guarded = PresenceStore::new(&peer_b); + guarded + .apply_from(&peer_c, &frame) + .expect("impersonated presence is dropped, not fatal"); + assert!( + guarded + .roster() + .iter() + .all(|entry| entry.key != peer_key(&peer_a)), + "presence delivered by a peer other than its author must be rejected", + ); + } + + #[test] + fn impersonating_peer_cannot_delete_another_peers_entry() { + let peer_a = test_peer(4); + let peer_b = test_peer(5); + let peer_c = test_peer(6); + + // B has learned A's presence. + let store_a = PresenceStore::new(&peer_a); + store_a + .set_self(&named_state("Ada")) + .expect("set self presence"); + let victim = PresenceStore::new(&peer_b); + victim + .apply_from(&peer_a, &store_a.encode_self()) + .expect("learn A"); + assert!( + victim + .roster() + .iter() + .any(|entry| entry.key == peer_key(&peer_a)) + ); + + // C crafts a frame that deletes A's entry and delivers it first-person. + store_a.delete_self(); + let delete_frame = store_a.encode_self(); + victim + .apply_from(&peer_c, &delete_frame) + .expect("griefing delete is dropped, not fatal"); + assert!( + victim + .roster() + .iter() + .any(|entry| entry.key == peer_key(&peer_a)), + "a peer must not be able to delete another peer's presence entry", + ); + } + + #[test] + fn display_name_is_sanitized_on_encode() { + let long_tail = "z".repeat(200); + let dirty = format!("Ada\u{7}\nLovelace{long_tail}"); + let sanitized = sanitize_display_name(&dirty); + assert!( + sanitized.len() <= MAX_PRESENCE_NAME_BYTES, + "name must be capped to {MAX_PRESENCE_NAME_BYTES} bytes" + ); + assert!(!sanitized.chars().any(|c| c.is_control()), "control characters must be stripped"); + assert!(sanitized.starts_with("AdaLovelace"), "leading printable characters are preserved"); + } + + #[test] + fn multibyte_display_name_truncates_on_a_char_boundary() { + // 'é' is two bytes in UTF-8; truncation must never split a code point. + let name = "é".repeat(MAX_PRESENCE_NAME_BYTES); + let sanitized = sanitize_display_name(&name); + assert!(sanitized.len() <= MAX_PRESENCE_NAME_BYTES); + assert!(std::str::from_utf8(sanitized.as_bytes()).is_ok()); + } + + #[test] + fn oversized_presence_value_is_rejected_on_apply() { + let peer_a = test_peer(7); + let peer_b = test_peer(8); + + // Craft a frame that bypasses sanitize-on-encode by writing the raw state + // straight into an ephemeral store under A's key. + let raw = EphemeralStore::new(PRESENCE_TIMEOUT_MS); + let oversized = named_state(&"q".repeat(MAX_PRESENCE_NAME_BYTES * 4)); + raw.set(&peer_key(&peer_a), LoroValue::Binary(encode_state(&oversized).expect("encode").into())); + let frame = raw.encode(&peer_key(&peer_a)); + + let guarded = PresenceStore::new(&peer_b); + guarded + .apply_from(&peer_a, &frame) + .expect("oversized presence is dropped, not fatal"); + assert!( + guarded.roster().is_empty(), + "a presence value exceeding the display-name cap must be rejected", + ); + } +} diff --git a/crates/flowstate-collab/src/proto_direct.rs b/crates/flowstate-collab/src/proto_direct.rs new file mode 100644 index 00000000..7e2ee81a --- /dev/null +++ b/crates/flowstate-collab/src/proto_direct.rs @@ -0,0 +1,129 @@ +use anyhow::{Result, anyhow, ensure}; +use serde::{Deserialize, Serialize}; + +use crate::admission::SessionAdmission; +use crate::discovery::DiscoveryAdmissionRequest; +use crate::ids::{BlobId, SessionId}; +use crate::proto_gossip::PROTOCOL_VERSION; + +pub const DIRECT_ALPN: &[u8] = b"flowstate/collab-direct/0"; +/// Ceiling for the small serialized control frames (`DirectRequest` / +/// `DirectResponseHeader`). These are metadata, so 2 MiB is generous. +pub const MAX_FRAME_LEN: usize = 2 * 1024 * 1024; +/// Ceiling for a streamed bulk payload (snapshots, update batches, assets). +/// Unlike a control frame, a payload is delivered in chunks and can be large: +/// a collaboration snapshot is the full CRDT document state and grows with the +/// document. 1 GiB is a generous upper bound that still caps a misbehaving peer's +/// self-declared length. This is intentionally separate from `MAX_FRAME_LEN` — +/// reusing the frame ceiling here would defeat the chunked streaming entirely. +pub const MAX_PAYLOAD_LEN: usize = 1024 * 1024 * 1024; +pub const MAX_PAYLOAD_CHUNK_LEN: usize = 256 * 1024; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum DirectRequest { + /// Proves possession of the symmetric editor admission secret before any + /// document data is served. + Authenticate { + session: SessionId, + admission: SessionAdmission, + }, + RequestAdmission { + request: DiscoveryAdmissionRequest, + }, + Snapshot { + session: SessionId, + }, + Updates { + session: SessionId, + have_vv: Vec, + }, + Blob { + session: SessionId, + blob: BlobId, + }, + Asset { + session: SessionId, + asset: u128, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct DiscoveryAdmissionGrant { + pub admission: SessionAdmission, + pub title: String, +} + +/// How a streamed direct payload is encoded on the wire. Rides on the response +/// header so the receiver knows how to decode and how much to preallocate. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum WireCodec { + /// Streamed verbatim; `wire_len == uncompressed_len`, no decode step. + None, + /// zstd framed with the shipped 256 KiB dictionary — the small-payload profile + /// (updates / small docs), where the dictionary's shared structure wins most. + ZstdDict, + /// zstd with long-distance matching and NO dictionary — the big-payload profile + /// (full large-document snapshots), where the dictionary actively hurts. + ZstdLong, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum DirectResponseHeader { + /// A payload follows. `codec` is how the streamed `wire_len` bytes are encoded; + /// `uncompressed_len` is the decoded size the receiver preallocates (equal to + /// `wire_len` when `codec` is `None`). + Ok { + codec: WireCodec, + wire_len: u64, + uncompressed_len: u64, + }, + NotAttached, + NotFound, + Busy, + /// The request was refused because the sender presented no valid admission + /// (missing handshake or wrong session secret). + Unauthorized, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AssetBytes { + pub bytes: Vec, +} + +pub fn encode_frame(value: &T) -> Result> { + let payload = postcard::to_stdvec(&(PROTOCOL_VERSION, value))?; + ensure!(payload.len() <= MAX_FRAME_LEN, "direct frame exceeds {MAX_FRAME_LEN} bytes"); + let len = payload.len() as u32; + let mut frame = len.to_le_bytes().to_vec(); + frame.extend(payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result +where + T: for<'de> Deserialize<'de>, +{ + ensure!(frame.len() >= 4, "direct frame is missing its length prefix"); + let payload_len = u32::from_le_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize; + ensure!(payload_len <= MAX_FRAME_LEN, "direct frame exceeds {MAX_FRAME_LEN} bytes"); + let payload = frame + .get(4..) + .ok_or_else(|| anyhow!("direct frame payload is missing"))?; + ensure!(payload.len() == payload_len, "direct frame length prefix does not match payload length"); + let (version, value): (u16, T) = postcard::from_bytes(payload)?; + if version != PROTOCOL_VERSION { + return Err(anyhow!("unsupported direct protocol version {version}")); + } + Ok(value) +} + +pub fn split_frame(frame: &[u8]) -> Result<(usize, &[u8])> { + ensure!(frame.len() >= 4, "direct frame is missing its length prefix"); + let payload_len = u32::from_le_bytes([frame[0], frame[1], frame[2], frame[3]]) as usize; + ensure!(payload_len <= MAX_FRAME_LEN, "direct frame exceeds {MAX_FRAME_LEN} bytes"); + let payload = frame + .get(4..) + .ok_or_else(|| anyhow!("direct frame payload is missing"))?; + ensure!(payload.len() == payload_len, "direct frame length prefix does not match payload length"); + Ok((payload_len, payload)) +} diff --git a/crates/flowstate-collab/src/proto_gossip.rs b/crates/flowstate-collab/src/proto_gossip.rs new file mode 100644 index 00000000..98ebfee8 --- /dev/null +++ b/crates/flowstate-collab/src/proto_gossip.rs @@ -0,0 +1,85 @@ +use std::fmt; + +use anyhow::{Result, ensure}; +use serde::{Deserialize, Serialize}; + +use crate::ids::{BlobId, SessionId}; + +/// Version 3 uses symmetric, session-lifetime editor admission. Pre-release +/// protocol versions are intentionally not accepted. +pub const PROTOCOL_VERSION: u16 = 3; +pub const GOSSIP_INLINE_LIMIT: usize = 2 * 1024; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProtocolVersionMismatch { + pub version: u16, +} + +impl fmt::Display for ProtocolVersionMismatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "unsupported gossip protocol version {}", self.version) + } +} + +impl std::error::Error for ProtocolVersionMismatch {} + +#[must_use] +pub fn is_protocol_version_mismatch(error: &anyhow::Error) -> bool { + error.is::() +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum GossipMsg { + Update(Vec), + UpdateAvailable { blob: BlobId, len: u64 }, + Presence(Vec), + Digest { session: SessionId, vv: Vec }, +} + +impl GossipMsg { + #[must_use] + pub fn kind(&self) -> &'static str { + match self { + Self::Update(_) => "update", + Self::UpdateAvailable { .. } => "update_available", + Self::Presence(_) => "presence", + Self::Digest { .. } => "digest", + } + } + + #[must_use] + pub fn payload_len(&self) -> u64 { + match self { + Self::Update(bytes) | Self::Presence(bytes) => bytes.len() as u64, + Self::UpdateAvailable { len, .. } => *len, + Self::Digest { vv, .. } => vv.len() as u64, + } + } +} + +pub fn encode(msg: &GossipMsg) -> Result> { + let mut bytes = PROTOCOL_VERSION.to_le_bytes().to_vec(); + bytes.extend(postcard::to_stdvec(msg)?); + Ok(bytes) +} + +pub fn encode_inline(msg: &GossipMsg) -> Result> { + let bytes = encode(msg)?; + ensure!(bytes.len() <= GOSSIP_INLINE_LIMIT, "gossip frame exceeds {GOSSIP_INLINE_LIMIT} bytes"); + Ok(bytes) +} + +pub fn encoded_len(msg: &GossipMsg) -> Result { + // This intentionally uses the encoder: postcard has no stable counting + // serializer here, and callers only need a small inline/blob decision. + Ok(encode(msg)?.len()) +} + +pub fn decode(bytes: &[u8]) -> Result { + ensure!(bytes.len() >= 2, "gossip frame is missing protocol version"); + let version = u16::from_le_bytes([bytes[0], bytes[1]]); + if version != PROTOCOL_VERSION { + return Err(ProtocolVersionMismatch { version }.into()); + } + Ok(postcard::from_bytes(&bytes[2..])?) +} diff --git a/crates/flowstate-collab/src/self_check.rs b/crates/flowstate-collab/src/self_check.rs new file mode 100644 index 00000000..9db3fa2e --- /dev/null +++ b/crates/flowstate-collab/src/self_check.rs @@ -0,0 +1,33 @@ +//! Local projection drift detection. + +use std::hash::{Hash, Hasher}; + +use gpui_flowtext::{Block, DocumentProjection, paragraph_text}; +use twox_hash::XxHash64; + +#[must_use] +pub fn projection_hash(document: &DocumentProjection) -> u64 { + let mut hasher = XxHash64::default(); + let mut paragraph_ix = 0; + for block in document.blocks.iter() { + match block { + Block::Paragraph(paragraph) => { + paragraph.style.hash(&mut hasher); + for run in ¶graph.runs { + run.len.hash(&mut hasher); + run.styles.hash(&mut hasher); + } + paragraph_text(document, paragraph_ix) + .as_bytes() + .hash(&mut hasher); + "p".hash(&mut hasher); + paragraph_ix += 1; + }, + Block::Image(_) | Block::Equation(_) | Block::Table(_) => { + std::mem::discriminant(block).hash(&mut hasher); + format!("{block:?}").hash(&mut hasher); + }, + } + } + hasher.finish() +} diff --git a/crates/flowstate-collab/src/ticket.rs b/crates/flowstate-collab/src/ticket.rs new file mode 100644 index 00000000..ebd874a4 --- /dev/null +++ b/crates/flowstate-collab/src/ticket.rs @@ -0,0 +1,149 @@ +use anyhow::{Context as _, Result, ensure}; +use iroh::EndpointAddr; +use iroh_tickets::{ParseError, Ticket}; +use serde::{Deserialize, Serialize}; + +use crate::{SessionId, admission::SessionAdmission, proto_gossip::PROTOCOL_VERSION}; + +pub const TICKET_KIND: &str = "fscollab"; +pub const INVITE_URL_PREFIX: &str = "flowstate://join#"; + +/// Symmetric editor invite for one ephemeral live session (wire version 3). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SessionTicket { + pub version: u16, + pub session: SessionId, + pub bootstrap: Vec, + pub title: String, + pub admission: SessionAdmission, + tag: [u8; 32], +} + +impl SessionTicket { + #[must_use] + pub fn new(session: SessionId, bootstrap: Vec, title: String, admission: SessionAdmission) -> Self { + let mut ticket = Self { + version: PROTOCOL_VERSION, + session, + bootstrap, + title, + admission, + tag: [0; 32], + }; + ticket.tag = ticket + .expected_tag() + .expect("ticket metadata serialization should not fail"); + ticket + } + + #[must_use] + pub const fn is_supported_version(&self) -> bool { + self.version == PROTOCOL_VERSION + } + + /// Authenticate all user-visible and routing metadata before any dial. + pub fn verify_for_join(&self) -> Result<()> { + ensure!(self.is_supported_version(), "unsupported collaboration protocol version"); + ensure!(!self.bootstrap.is_empty(), "collaboration invite has no reachable participants"); + ensure!(self.tag == self.expected_tag()?, "collaboration invite metadata is invalid"); + Ok(()) + } + + fn expected_tag(&self) -> Result<[u8; 32]> { + let payload = + postcard::to_stdvec(&(self.version, self.session, &self.bootstrap, &self.title)).context("encoding collaboration invite metadata")?; + Ok(self.admission.tag(&payload)) + } + + #[must_use] + pub fn encode_text(&self) -> String { + self.encode_string() + } + + /// User-facing deep link. The bearer ticket lives in the URL fragment so an + /// eventual HTTPS bridge never sends it to a web server in the request path. + #[must_use] + pub fn encode_invite_link(&self) -> String { + format!("{INVITE_URL_PREFIX}{}", self.encode_text()) + } + + pub fn decode_text(text: &str) -> Result { + let text = text.trim(); + let ticket = text.strip_prefix(INVITE_URL_PREFIX).unwrap_or(text); + Self::decode_string(ticket) + } +} + +impl Ticket for SessionTicket { + const KIND: &'static str = TICKET_KIND; + + fn encode_bytes(&self) -> Vec { + postcard::to_stdvec(self).expect("SessionTicket serialization should not fail") + } + + fn decode_bytes(bytes: &[u8]) -> Result { + let ticket: Self = postcard::from_bytes(bytes).map_err(|_| ParseError::verification_failed("invalid collaboration ticket payload"))?; + ticket + .verify_for_join() + .map_err(|_| ParseError::verification_failed("invalid collaboration ticket authentication"))?; + Ok(ticket) + } +} + +#[cfg(test)] +mod tests { + use iroh::{EndpointAddr, SecretKey}; + + use super::SessionTicket; + use crate::{SessionAdmission, SessionId}; + + fn ticket() -> SessionTicket { + SessionTicket::new( + SessionId::from_bytes([1; 32]), + vec![EndpointAddr::new(SecretKey::from_bytes(&[2; 32]).public())], + "Shared brief".to_string(), + SessionAdmission::from_bytes([3; 32]), + ) + } + + #[test] + fn ticket_round_trips_and_authenticates_metadata() { + let encoded = ticket().encode_text(); + let decoded = SessionTicket::decode_text(&encoded).expect("ticket should decode"); + decoded + .verify_for_join() + .expect("ticket should authenticate"); + } + + #[test] + fn deep_link_and_raw_ticket_are_both_accepted() { + let ticket = ticket(); + assert_eq!( + SessionTicket::decode_text(&ticket.encode_text()) + .unwrap() + .session, + ticket.session + ); + assert_eq!( + SessionTicket::decode_text(&ticket.encode_invite_link()) + .unwrap() + .session, + ticket.session + ); + } + + #[test] + fn tampered_metadata_is_rejected() { + let mut ticket = ticket(); + ticket.title.push_str(" (forged)"); + assert!(ticket.verify_for_join().is_err()); + } + + #[test] + fn any_participant_endpoint_can_be_the_bootstrap() { + let mut ticket = ticket(); + ticket.bootstrap = vec![EndpointAddr::new(SecretKey::from_bytes(&[9; 32]).public())]; + ticket.tag = ticket.expected_tag().expect("tag"); + assert!(ticket.verify_for_join().is_ok()); + } +} diff --git a/crates/flowstate-collab/tests/anti_entropy.rs b/crates/flowstate-collab/tests/anti_entropy.rs new file mode 100644 index 00000000..405d0568 --- /dev/null +++ b/crates/flowstate-collab/tests/anti_entropy.rs @@ -0,0 +1,191 @@ +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use flowstate_collab::{ + SessionId, + net::anti_entropy::{AntiEntropyState, GapAction, VersionVectorRelation}, + }; + use iroh::{EndpointId, SecretKey}; + + #[test] + fn digest_gap_starts_one_pull_until_the_pull_finishes() { + let session = SessionId::from_bytes([1; 32]); + let peer = peer(9); + let now = Instant::now(); + let mut state = AntiEntropyState::new(session, now); + + assert_eq!( + state.consider_digest(peer, session, VersionVectorRelation::SenderHasMissingOps, vec![1, 2, 3], now), + GapAction::Pull { + from: peer, + our_vv: vec![1, 2, 3], + } + ); + assert_eq!( + state.consider_digest(peer, session, VersionVectorRelation::Concurrent, vec![4, 5], now), + GapAction::None + ); + + state.finish_pull(peer); + assert_eq!( + state.consider_digest(peer, session, VersionVectorRelation::Concurrent, vec![6], now), + GapAction::Pull { from: peer, our_vv: vec![6] } + ); + } + + #[test] + fn equal_or_sender_behind_digest_does_not_pull() { + let session = SessionId::from_bytes([2; 32]); + let peer = peer(10); + let now = Instant::now(); + let mut state = AntiEntropyState::new(session, now); + + assert_eq!( + state.consider_digest(peer, session, VersionVectorRelation::Equal, vec![1], now), + GapAction::None + ); + assert_eq!( + state.consider_digest(peer, session, VersionVectorRelation::WeHaveMissingOps, vec![2], now), + GapAction::None + ); + } + + #[test] + fn lineage_mismatch_is_reported_without_starting_a_pull() { + let session = SessionId::from_bytes([3; 32]); + let other = SessionId::from_bytes([4; 32]); + let peer = peer(11); + let now = Instant::now(); + let mut state = AntiEntropyState::new(session, now); + + assert_eq!( + state.consider_digest(peer, other, VersionVectorRelation::SenderHasMissingOps, vec![1], now), + GapAction::LineageMismatch { + from: peer, + expected: session, + got: other, + } + ); + assert_eq!( + state.consider_digest(peer, session, VersionVectorRelation::SenderHasMissingOps, vec![2], now), + GapAction::Pull { from: peer, our_vv: vec![2] } + ); + } + + #[test] + fn lagged_uses_the_fallback_peer_and_deduplicates_in_flight_pulls() { + let session = SessionId::from_bytes([5; 32]); + let peer = peer(12); + let now = Instant::now(); + let mut state = AntiEntropyState::new(session, now); + + assert_eq!(state.on_lagged(None, vec![1], now), GapAction::None); + assert_eq!(state.on_lagged(Some(peer), vec![2], now), GapAction::Pull { from: peer, our_vv: vec![2] }); + assert_eq!(state.on_lagged(Some(peer), vec![3], now), GapAction::None); + } + + #[test] + fn a_pull_in_flight_for_one_peer_does_not_block_another_peer() { + let session = SessionId::from_bytes([7; 32]); + let peer_a = peer(20); + let peer_b = peer(21); + let now = Instant::now(); + let mut state = AntiEntropyState::new(session, now); + + assert_eq!( + state.consider_digest(peer_a, session, VersionVectorRelation::SenderHasMissingOps, vec![1], now), + GapAction::Pull { + from: peer_a, + our_vv: vec![1] + } + ); + // A second gap for A while its pull is outstanding is deduplicated... + assert_eq!( + state.consider_digest(peer_a, session, VersionVectorRelation::SenderHasMissingOps, vec![2], now), + GapAction::None + ); + // ...but a gap with a different peer B is served concurrently. + assert_eq!( + state.consider_digest(peer_b, session, VersionVectorRelation::SenderHasMissingOps, vec![3], now), + GapAction::Pull { + from: peer_b, + our_vv: vec![3] + } + ); + // Finishing A's pull only frees A's dedup slot. + state.finish_pull(peer_a); + assert_eq!( + state.consider_digest(peer_b, session, VersionVectorRelation::SenderHasMissingOps, vec![4], now), + GapAction::None + ); + assert_eq!( + state.consider_digest(peer_a, session, VersionVectorRelation::SenderHasMissingOps, vec![5], now), + GapAction::Pull { + from: peer_a, + our_vv: vec![5] + } + ); + } + + #[test] + fn a_hung_pull_expires_after_its_deadline_so_the_peer_is_retried() { + let session = SessionId::from_bytes([8; 32]); + let peer_a = peer(22); + let peer_b = peer(23); + let start = Instant::now(); + let mut state = AntiEntropyState::new(session, start); + + // A pull to A starts and then wedges: `finish_pull` is never called. + assert_eq!( + state.consider_digest(peer_a, session, VersionVectorRelation::SenderHasMissingOps, vec![1], start), + GapAction::Pull { + from: peer_a, + our_vv: vec![1] + } + ); + // Before the deadline, A stays deduplicated while B is unaffected. + let soon = start + Duration::from_secs(1); + assert_eq!( + state.consider_digest(peer_a, session, VersionVectorRelation::SenderHasMissingOps, vec![2], soon), + GapAction::None + ); + assert_eq!( + state.consider_digest(peer_b, session, VersionVectorRelation::SenderHasMissingOps, vec![3], soon), + GapAction::Pull { + from: peer_b, + our_vv: vec![3] + } + ); + // Past the deadline, A's wedged pull is force-expired and A is retried even + // though its original pull was never finished. + let past_deadline = start + Duration::from_secs(120); + assert_eq!( + state.consider_digest(peer_a, session, VersionVectorRelation::SenderHasMissingOps, vec![4], past_deadline), + GapAction::Pull { + from: peer_a, + our_vv: vec![4] + } + ); + } + + #[test] + fn neighbor_and_reconnect_events_schedule_an_immediate_digest() { + let session = SessionId::from_bytes([6; 32]); + let now = Instant::now(); + let mut state = AntiEntropyState::new(session, now); + + assert!(!state.digest_due(now + Duration::from_secs(1))); + state.on_neighbor_up(); + assert!(state.digest_due(Instant::now() + Duration::from_millis(1))); + + state.mark_digest_sent(Instant::now()); + assert!(!state.digest_due(Instant::now() + Duration::from_secs(1))); + state.on_reconnect(); + assert!(state.digest_due(Instant::now() + Duration::from_millis(1))); + } + + fn peer(seed: u8) -> EndpointId { + SecretKey::from_bytes(&[seed; 32]).public() + } +} diff --git a/crates/flowstate-collab/tests/bare_heal_cost.rs b/crates/flowstate-collab/tests/bare_heal_cost.rs new file mode 100644 index 00000000..a2e3dd77 --- /dev/null +++ b/crates/flowstate-collab/tests/bare_heal_cost.rs @@ -0,0 +1,151 @@ +//! §oom-leads #9 / task #40: the REMOTE-STRUCT-BARE heal cost. A raw peer +//! inserts naked `\n`s straight into the body text (no paragraph record, no +//! style mark — the docx `w:br` / adversarial shape); every import must heal +//! the record-less boundary. The heal used to cost ~2 whole-doc +//! materializations per round (a regional bail + the repair pass's flat +//! re-projection); this pins the work budget so it can only go down. + +use std::sync::Arc; + +use flowstate_collab::crdt_runtime::CrdtRuntime; +use flowstate_collab::local_write::{ + GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, SplitParagraphIntent, TextAnchor, WriteGate, +}; +use flowstate_document::{ParagraphStyle, instrument}; + +fn new_handle(title: &str) -> (LocalDocHandle, Arc>) { + let core = CrdtRuntime::new_empty(title).expect("runtime"); + LocalDocHandle::new(core, LocalWriteConfig::default()) +} + +/// Seed a multi-paragraph doc through the real write path. +fn seed(handle: &LocalDocHandle, paragraphs: usize) { + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..paragraphs { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("paragraph {i} body text with some length to it"), + style_override: None, + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("last paragraph"); + } +} + +/// The bench's REMOTE-STRUCT-BARE shape, N rounds; returns the work delta. +fn run_bare_rounds(gate: &Arc>, rounds: usize) -> instrument::WorkCounts { + // Raw peer: forks the canonical doc and edits the body text directly — + // no flowstate runtime, so no records/marks accompany its boundaries. + let snapshot = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard + .doc() + .export(loro::ExportMode::Snapshot) + .expect("snapshot") + }; + let peer_doc = loro::LoroDoc::new(); + peer_doc + .import_with(&snapshot, "remote") + .expect("peer join"); + let body_b = flowstate_document::loro_schema::body_text(&peer_doc); + let mut peer_vv = peer_doc.state_vv(); + + let before = instrument::snapshot(); + for _ in 0..rounds { + let len = body_b.len_unicode(); + body_b + .insert(len / 2, "\n") + .expect("bare structural insert"); + peer_doc.commit(); + let update = peer_doc + .export(loro::ExportMode::updates(&peer_vv)) + .expect("peer export"); + peer_vv = peer_doc.state_vv(); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&update) + .expect("import bare structural"); + } + instrument::snapshot().since(before) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Diagnostic escape hatch: `cargo test -p flowstate-collab --test + /// bare_heal_cost -- --ignored --nocapture` with `RUST_LOG=warn` + + /// `FLOWSTATE_DERIVE_DEBUG=1` prints the ladder decision and any regional + /// bail per round. + #[test] + #[ignore = "diagnostic: run with RUST_LOG=warn FLOWSTATE_DERIVE_DEBUG=1 --nocapture"] + fn bare_heal_diagnostic() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .try_init(); + let (handle, gate) = new_handle("bare-heal-diag"); + seed(&handle, 40); + let work = run_bare_rounds(&gate, 8); + eprintln!( + "bare-heal work over 8 rounds: full_projections={} body_to_delta={} cursor_resolves={}", + work.full_projections, work.body_to_delta_builds, work.cursor_pos_resolves + ); + } + + /// §task #40 gate: the per-round heal must not pay whole-doc materializations. + /// Baseline before the fix: ~1.85 full projections/round — a `loro_id_u128` + /// LAW MISMATCH (blake3 in `flowstate-document` vs uuid-v5 in `crdt_runtime`) made + /// every repaired anchor-keyed record unresolvable to the runtime (regional + /// bail + defect re-report + repair-pass flat re-projection, EVERY round). + /// After the unified law + the regional anchor-key probe + the neutral-batch + /// repair skip, the steady state is ZERO full projections. + #[test] + fn bare_boundary_heal_work_is_bounded() { + let (handle, gate) = new_handle("bare-heal-gate"); + seed(&handle, 40); + let rounds = 8; + let work = run_bare_rounds(&gate, rounds); + assert!( + work.full_projections <= 2, + "bare-boundary heal paid {} full projections over {rounds} rounds (steady state must be ~0; ~2/round means the heal regressed)", + work.full_projections + ); + // Convergence: the healed live projection must equal an independent full + // materialization, text and styles. + let live = handle.projection().expect("projection"); + let canonical = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::document_from_loro(guard.doc()).expect("materializes") + }; + assert_eq!( + live.paragraphs.len(), + canonical.paragraphs.len(), + "live == canonical paragraph count after heals" + ); + let live_text: Vec = (0..live.paragraphs.len()) + .map(|ix| flowstate_document::paragraph_text(&live, ix)) + .collect(); + let canonical_text: Vec = (0..canonical.paragraphs.len()) + .map(|ix| flowstate_document::paragraph_text(&canonical, ix)) + .collect(); + assert_eq!(live_text, canonical_text, "live == canonical text after heals"); + let live_styles: Vec<_> = live.paragraphs.iter().map(|p| p.style).collect(); + let canonical_styles: Vec<_> = canonical.paragraphs.iter().map(|p| p.style).collect(); + assert_eq!(live_styles, canonical_styles, "live == canonical styles after heals"); + } +} diff --git a/crates/flowstate-collab/tests/caret_soak.rs b/crates/flowstate-collab/tests/caret_soak.rs new file mode 100644 index 00000000..b727fd70 --- /dev/null +++ b/crates/flowstate-collab/tests/caret_soak.rs @@ -0,0 +1,408 @@ +//! Presence-caret soak against the Loro-first intent API (spec §13.8 flavor). +//! +//! The old optimistic-reconcile caret soak raced pending editor batches behind +//! an in-flight raw-command runtime transaction; that surface no longer +//! exists — with `LocalDocHandle` the "optimistic" apply IS the committed +//! apply, so there is nothing to reconcile. What survives is the caret +//! DURABILITY question: a presence selection captured as encoded Loro cursors +//! must keep resolving to an in-bounds document position on every replica while +//! concurrent intents and imports churn the document underneath it. +//! +//! Shape: N peers edit concurrently through their `LocalDocHandle`s; each +//! round every peer captures a presence selection at a random caret position +//! (gate-held, `CrdtRuntime::presence_selection`), the mesh syncs, and then +//! EVERY peer resolves EVERY captured selection through +//! `CrdtRuntime::resolve_presence_carets`. Asserted per round: no panics, every +//! resolved caret lands inside the resolving peer's document bounds, and the +//! replicas converge (body text + projection). + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use flowstate_collab::crdt_runtime::{CrdtRuntime, RuntimePresenceCaretRequest}; + use flowstate_collab::local_write::{ + DeleteRangeIntent, GateHolder, InsertTextIntent, JoinParagraphsIntent, LocalDocHandle, LocalWriteConfig, SetMarksIntent, + SetParagraphStyleIntent, SplitParagraphIntent, TextAnchor, WriteGate, WriteRejected, + }; + use flowstate_collab::presence::PresenceSelection; + use flowstate_document::{ + DocumentOffset, DocumentProjection, EditorSelection, ParagraphStyle, RunSemanticStyle, RunStyles, paragraph_text, paragraph_text_len, + }; + + /// Deterministic xorshift PRNG — reproducible soak, no wall-clock dependence. + struct Rng(u64); + + impl Rng { + fn new(seed: u64) -> Self { + Self(seed.max(1)) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, bound: usize) -> usize { + if bound == 0 { 0 } else { (self.next() % bound as u64) as usize } + } + } + + struct Peer { + handle: LocalDocHandle, + gate: Arc>, + synced_vv: Vec, + } + + impl Peer { + fn new(title: &str, peer_count: usize) -> Self { + let core = CrdtRuntime::new_empty(title).expect("runtime"); + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + Self { + handle, + gate, + synced_vv: vec![loro::VersionVector::default(); peer_count], + } + } + + fn projection(&self) -> DocumentProjection { + self.handle.projection().expect("projection") + } + + fn export_updates_since(&self, vv: &loro::VersionVector) -> Vec { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard + .doc() + .export(loro::ExportMode::updates(vv)) + .expect("export") + } + + fn state_vv(&self) -> loro::VersionVector { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard.doc().state_vv() + } + + fn import(&self, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + let mut guard = self.gate.lock(GateHolder::ImportChunk).expect("gate"); + guard.import_remote_update(bytes).expect("import"); + } + + fn body_text(&self) -> String { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + } + + /// Capture a presence selection for a collapsed caret at `offset`, holding + /// the gate as the Presence holder (cursor encode is a doc read). + fn capture_presence(&self, offset: DocumentOffset) -> Option { + let guard = self.gate.lock(GateHolder::Presence).expect("gate"); + guard.presence_selection(&EditorSelection::collapsed(offset)) + } + + /// Resolve remote presence selections against this peer's canonical state. + fn resolve_presence(&self, requests: Vec) -> Vec { + let guard = self.gate.lock(GateHolder::Presence).expect("gate"); + guard.resolve_presence_carets(requests).carets + } + } + + /// Full-mesh sync: every peer pulls every other peer's new history. + fn sync_all(peers: &mut [Peer]) { + for _ in 0..2 { + for from in 0..peers.len() { + let from_vv_now = peers[from].state_vv(); + for to in 0..peers.len() { + if from == to { + continue; + } + let since = peers[to].synced_vv[from].clone(); + let bytes = peers[from].export_updates_since(&since); + peers[to].import(&bytes); + peers[to].synced_vv[from] = from_vv_now.clone(); + } + } + } + } + + fn random_styles(rng: &mut Rng) -> RunStyles { + let mut styles = RunStyles::default(); + if rng.below(2) == 0 { + styles.semantic = RunSemanticStyle::Custom((rng.below(4) + 1) as u8); + } + if rng.below(3) == 0 { + styles.direct_underline = true; + } + styles + } + + /// One random text intent against `peer` (ASCII-only content so every raw byte + /// hint is char-boundary-clean). Rejections from stale identities are legal + /// outcomes (I-15); anything that wedges the doc is not. + fn random_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let projection = peer.projection(); + if projection.paragraphs.is_empty() { + return Ok(()); + } + let paragraph_ix = rng.below(projection.paragraphs.len()); + let paragraph = projection.ids.paragraph_ids[paragraph_ix]; + let text_len = paragraph_text_len(&projection.paragraphs[paragraph_ix]); + let byte = rng.below(text_len + 1); + match rng.below(10) { + 0..=4 => peer + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, byte), + text: format!("s{step}"), + style_override: None, + }) + .map(|_| ()), + 5 => { + if text_len == 0 { + return Ok(()); + } + let start = rng.below(text_len); + let end = (start + 1 + rng.below(3)).min(text_len); + peer + .handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(paragraph, start), + end: TextAnchor::new(paragraph, end), + }) + .map(|_| ()) + }, + 6 if projection.paragraphs.len() < 24 => peer + .handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, byte), + inherited_style: ParagraphStyle::Normal, + }) + .map(|_| ()), + 7 => { + if paragraph_ix + 1 >= projection.paragraphs.len() { + return Ok(()); + } + let second = projection.ids.paragraph_ids[paragraph_ix + 1]; + peer + .handle + .join_paragraphs(JoinParagraphsIntent { first: paragraph, second }) + .map(|_| ()) + }, + 8 => { + if text_len == 0 { + return Ok(()); + } + let start = rng.below(text_len); + let end = (start + 1 + rng.below(4)).min(text_len); + peer + .handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(paragraph, start), + end: TextAnchor::new(paragraph, end), + styles: random_styles(rng), + }) + .map(|_| ()) + }, + _ => peer + .handle + .set_paragraph_style(SetParagraphStyleIntent { + paragraph, + style: if rng.below(2) == 0 { + ParagraphStyle::Normal + } else { + ParagraphStyle::Custom(1) + }, + }) + .map(|_| ()), + } + } + + fn tolerate(result: Result<(), WriteRejected>, seed: u64, round: usize, op: usize) { + match result { + Ok(()) => {}, + Err(fatal @ (WriteRejected::GatePoisoned | WriteRejected::CompensationFailed { .. })) => { + panic!("seed {seed} round {round} op {op}: doc wedged: {fatal}") + }, + // Stale identity / empty / structural / compensated rejections are legal + // fuzz outcomes; only convergence is asserted. + Err(_) => {}, + } + } + + fn assert_replicas_converged(peers: &[Peer], seed: u64, round: usize) { + let reference_text = peers[0].body_text(); + let reference = peers[0].projection(); + for (ix, peer) in peers.iter().enumerate().skip(1) { + assert_eq!( + peer.body_text(), + reference_text, + "seed {seed} round {round}: peer {ix} body text diverged" + ); + let projection = peer.projection(); + assert_eq!( + projection.ids.paragraph_ids, reference.ids.paragraph_ids, + "seed {seed} round {round}: peer {ix} paragraph ids diverged" + ); + for paragraph_ix in 0..reference.paragraphs.len() { + assert_eq!( + paragraph_text(&projection, paragraph_ix), + paragraph_text(&reference, paragraph_ix), + "seed {seed} round {round}: peer {ix} paragraph {paragraph_ix} text diverged" + ); + } + } + } + + fn run_presence_soak(seed: u64, peers_n: usize, rounds: usize, ops_per_round: usize) { + let mut rng = Rng::new(seed); + let mut peers: Vec = (0..peers_n) + .map(|_| Peer::new("caret soak", peers_n)) + .collect(); + sync_all(&mut peers); + + for round in 0..rounds { + // Concurrent intents (pre-sync, so cursor capture races the round's edits). + for op in 0..ops_per_round { + let peer_ix = rng.below(peers_n); + let step = round * ops_per_round + op; + tolerate(random_intent(&mut rng, &peers[peer_ix], step), seed, round, op); + } + + // Every peer captures a presence caret at a random position in its own + // (diverged) replica. `None` is legal only for degenerate offsets; a caret + // in a live paragraph must encode. + let mut selections: Vec = Vec::new(); + for peer in &peers { + let projection = peer.projection(); + if projection.paragraphs.is_empty() { + continue; + } + let paragraph_ix = rng.below(projection.paragraphs.len()); + let byte = rng.below(paragraph_text_len(&projection.paragraphs[paragraph_ix]) + 1); + let offset = DocumentOffset { + paragraph: paragraph_ix, + byte, + }; + if let Some(selection) = peer.capture_presence(offset) { + selections.push(selection); + } + } + + sync_all(&mut peers); + + // Post-sync, every peer resolves every captured selection. Unresolvable + // cursors (target deleted concurrently) drop out — legal. Every caret that + // DOES resolve must land inside the resolving peer's document bounds. + for (peer_ix, peer) in peers.iter().enumerate() { + let projection = peer.projection(); + let requests: Vec = selections + .iter() + .map(|selection| RuntimePresenceCaretRequest { + selection: selection.clone(), + color_rgb: 0x00FF_00FF, + }) + .collect(); + let carets = peer.resolve_presence(requests); + for caret in &carets { + assert!( + caret.offset.paragraph < projection.paragraphs.len(), + "seed {seed} round {round}: peer {peer_ix} resolved caret paragraph {} out of bounds ({} paragraphs)", + caret.offset.paragraph, + projection.paragraphs.len(), + ); + let len = paragraph_text_len(&projection.paragraphs[caret.offset.paragraph]); + assert!( + caret.offset.byte <= len, + "seed {seed} round {round}: peer {peer_ix} resolved caret byte {} beyond paragraph end {len}", + caret.offset.byte, + ); + } + } + + assert_replicas_converged(&peers, seed, round); + } + } + + #[test] + fn presence_caret_soak_two_peers() { + for seed in [0xC0FF_EE12, 7] { + run_presence_soak(seed, 2, 8, 10); + } + } + + #[test] + fn presence_caret_soak_three_peers() { + for seed in [42, 20260707] { + run_presence_soak(seed, 3, 6, 8); + } + } + + /// Directed caret-durability regression (the spirit of the old enter/char + /// spam): a caret parked at the end of a paragraph on peer 0 must keep + /// resolving to that paragraph's end while peer 1 concurrently prepends text + /// and splits paragraphs upstream of it — resolution goes through the encoded + /// cursor, never through raw offsets. + #[test] + fn caret_at_paragraph_end_survives_upstream_churn() { + let mut peers: Vec = (0..2).map(|_| Peer::new("caret churn", 2)).collect(); + sync_all(&mut peers); + + let paragraph = peers[0].projection().ids.paragraph_ids[0]; + peers[0] + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "anchor".into(), + style_override: None, + }) + .expect("seed insert"); + sync_all(&mut peers); + + // Park the caret at the end of "anchor" on peer 0. + let selection = peers[0] + .capture_presence(DocumentOffset { paragraph: 0, byte: 6 }) + .expect("caret in live text must encode"); + + for i in 0..10 { + // Peer 1 churns upstream: prepend + split at the front of the paragraph. + peers[1] + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: format!("x{i}"), + style_override: None, + }) + .expect("upstream insert"); + if i % 3 == 0 { + let _ = peers[1].handle.split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, 2), + inherited_style: ParagraphStyle::Normal, + }); + } + sync_all(&mut peers); + + for (peer_ix, peer) in peers.iter().enumerate() { + let projection = peer.projection(); + let carets = peer.resolve_presence(vec![RuntimePresenceCaretRequest { + selection: selection.clone(), + color_rgb: 0x00FF_FFFF, + }]); + assert_eq!(carets.len(), 1, "iteration {i}: peer {peer_ix} must still resolve the parked caret"); + let caret = &carets[0]; + let text = paragraph_text(&projection, caret.offset.paragraph); + assert!( + text.as_bytes()[..caret.offset.byte].ends_with(b"anchor"), + "iteration {i}: peer {peer_ix} caret must stay glued to the end of 'anchor', got byte {} in {text:?}", + caret.offset.byte, + ); + } + } + assert_replicas_converged(&peers, 0, 0); + } +} diff --git a/crates/flowstate-collab/tests/comment_threads.rs b/crates/flowstate-collab/tests/comment_threads.rs new file mode 100644 index 00000000..1b6a3e70 --- /dev/null +++ b/crates/flowstate-collab/tests/comment_threads.rs @@ -0,0 +1,96 @@ +//! Comment thread lifecycle against the runtime API: the thread author is +//! recorded at creation and gates deletion, message edits are author-gated, +//! and bodies are sanitized before they reach the CRDT. + +#[cfg(test)] +mod tests { + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, TextAnchor}; + use gpui_flowtext::{DocumentOffset, EditorSelection}; + + fn commented_doc() -> (LocalDocHandle, std::sync::Arc>) { + let core = CrdtRuntime::new_empty("comment-threads").expect("runtime"); + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let projection = handle.projection().expect("projection"); + let paragraph = projection.ids.paragraph_ids[0]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "hello comment anchors".into(), + style_override: None, + }) + .expect("insert text"); + (handle, gate) + } + + fn selection(start: usize, end: usize) -> EditorSelection { + let mut selection = EditorSelection::collapsed(DocumentOffset { paragraph: 0, byte: start }); + selection.head = DocumentOffset { paragraph: 0, byte: end }; + selection + } + + #[test] + fn thread_author_is_stored_and_gates_deletion() { + let (_handle, gate) = commented_doc(); + let mut guard = gate.lock(GateHolder::DocumentService).expect("gate"); + let comment_id = guard + .create_comment(&selection(0, 5), "first!", 7, "Alex") + .expect("create comment"); + + let threads = guard.comments(); + assert_eq!(threads.len(), 1); + assert_eq!(threads[0].comment_id, comment_id); + assert_eq!(threads[0].messages[0].author_user_id, 7); + assert_eq!(threads[0].quoted_text, "hello"); + + // A replying non-author must not be able to delete the thread — even + // though their message now exists (and could win a timestamp race). + guard + .reply_to_comment(comment_id, "me too", 9, "Blair") + .expect("reply"); + let error = guard + .delete_comment(comment_id, 9) + .expect_err("non-author delete"); + assert!(error.to_string().contains("thread author"), "unexpected error: {error}"); + assert_eq!(guard.comments().len(), 1); + + guard.delete_comment(comment_id, 7).expect("author delete"); + assert!(guard.comments().is_empty()); + drop(guard); + } + + #[test] + fn message_edits_are_author_gated_and_bodies_sanitized() { + let (_handle, gate) = commented_doc(); + let mut guard = gate.lock(GateHolder::DocumentService).expect("gate"); + let comment_id = guard + .create_comment(&selection(0, 5), "line one\r\nline\u{7} two", 7, "Alex") + .expect("create comment"); + + let threads = guard.comments(); + assert_eq!(threads[0].messages[0].body, "line one\nline two"); + + let message_id = threads[0].messages[0].message_id; + let error = guard + .edit_comment_message(comment_id, message_id, "hijacked", 9) + .expect_err("non-author edit"); + assert!(error.to_string().contains("author"), "unexpected error: {error}"); + + guard + .edit_comment_message(comment_id, message_id, "revised", 7) + .expect("author edit"); + assert_eq!(guard.comments()[0].messages[0].body, "revised"); + + assert!( + guard + .create_comment(&selection(2, 2), "empty selection", 7, "Alex") + .is_err() + ); + assert!( + guard + .create_comment(&selection(0, 5), " \r\u{0} ", 7, "Alex") + .is_err() + ); + drop(guard); + } +} diff --git a/crates/flowstate-collab/tests/doc_io_pump.rs b/crates/flowstate-collab/tests/doc_io_pump.rs new file mode 100644 index 00000000..f78684fc --- /dev/null +++ b/crates/flowstate-collab/tests/doc_io_pump.rs @@ -0,0 +1,271 @@ +//! §act-eleven C8: the doc I/O pump harness. `DocIoHandle` is the field's +//! delivery surface — the §6.4 import coalescing, the P1.A second drain behind +//! a held gate, deferred non-import ordering, and the publish pump — and it +//! previously had ONE test spawning it (a package-export latency probe). This +//! suite drives it with scripted drips: bursts behind a held gate, duplicated +//! and out-of-order blobs, and interleaved non-import requests. + +use anyhow::Result; +use flowstate_collab::crdt_runtime::CrdtRuntime; +use flowstate_collab::doc_io::DocIoHandle; +use flowstate_collab::local_write::{GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, TextAnchor}; +use std::sync::Arc; + +/// A fully-based (source, receiver) pair — sentinel init ops EXCHANGED both +/// ways (the known one-way trap) — plus `edits` sequential source inserts +/// exported as one delta blob PER EDIT (the network drip shape) and the +/// source's final body text. +type ReceiverGate = Arc>; + +fn drip_pair(edits: usize) -> Result<(String, Vec>, ReceiverGate)> { + let source = CrdtRuntime::new_empty("drip-source")?; + let (source_handle, source_gate) = LocalDocHandle::new(source, LocalWriteConfig::default()); + let receiver = CrdtRuntime::new_empty("drip-receiver")?; + let (_receiver_handle, receiver_gate) = LocalDocHandle::new(receiver, LocalWriteConfig::default()); + let full_export = |gate: &ReceiverGate| { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("init export") + }; + { + let mut guard = receiver_gate + .lock(GateHolder::ImportChunk) + .expect("gate healthy"); + guard.import_remote_update(&full_export(&source_gate))? + }; + { + let mut guard = source_gate + .lock(GateHolder::ImportChunk) + .expect("gate healthy"); + guard.import_remote_update(&full_export(&receiver_gate))? + }; + + let paragraph = *source_handle + .projection()? + .ids + .paragraph_ids + .last() + .expect("source paragraph"); + let mut blobs = Vec::with_capacity(edits); + for edit_ix in 0..edits { + let vv_before = { + let guard = source_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + guard.doc().state_vv() + }; + source_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("{}", edit_ix % 10), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("drip edit {edit_ix} rejected: {error:?}"))?; + let blob = { + let guard = source_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&vv_before)) + .expect("delta export") + }; + blobs.push(blob); + } + let final_text = { + let guard = source_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + }; + Ok((final_text, blobs, receiver_gate)) +} + +fn body_text(gate: &ReceiverGate) -> String { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate healthy"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() +} + +fn import_batches(gate: &ReceiverGate) -> u64 { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate healthy"); + guard.import_batches_served() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A 40-blob drip fired while the gate is HELD must coalesce into a few + /// batched imports (≤16 blobs per §6.4 chunk, folded further by the P1.A + /// second drain), every reply must arrive, and the receiver must converge. + #[test] + fn held_gate_drip_coalesces_into_bounded_batches() -> Result<()> { + const EDITS: usize = 40; + let (source_text, blobs, gate) = drip_pair(EDITS)?; + let io = DocIoHandle::spawn(Arc::clone(&gate)).expect("io service"); + + let batches_before = import_batches(&gate); + // Hold the gate so the io thread blocks on its FIRST import; every blob + // sent meanwhile queues behind it and must fold into few batches. + let hold = gate + .lock(GateHolder::DocumentService) + .expect("gate healthy"); + // One OS thread per blob: async fns are LAZY, so sequentially awaiting + // them would send each request only after the previous replied (zero queue + // depth, zero coalescing). Independent blocked threads enqueue all 40 + // requests concurrently behind the held gate — the real network shape. + // NOTE: the intermediate Vec is REQUIRED (spawn all senders BEFORE joining + // any — a lazy iterator would serialize spawn/join pairs and give zero + // queue depth, exactly the lazy-future bug this replaced). + #[allow(clippy::needless_collect, reason = "spawn-all-then-join-all; lazy iteration would serialize the burst")] + let senders: Vec<_> = blobs + .iter() + .map(|blob| { + let io_import = io.clone(); + let blob = blob.clone(); + std::thread::spawn(move || pollster::block_on(io_import.import_remote_update(blob))) + }) + .collect(); + // Give every sender time to enqueue behind the held gate, then release. + std::thread::sleep(std::time::Duration::from_millis(200)); + drop(hold); + let replies: Vec<_> = senders + .into_iter() + .map(|sender| sender.join().expect("sender join")) + .collect(); + + assert_eq!(replies.len(), EDITS); + for (blob_ix, reply) in replies.iter().enumerate() { + assert!(reply.is_ok(), "blob {blob_ix} reply failed: {reply:?}"); + } + assert_eq!(body_text(&gate), source_text, "receiver did not converge to the source text"); + let batches = import_batches(&gate) - batches_before; + // 40 blobs at ≤16/chunk = ≥3 batches; the bound allows the io thread to + // have consumed a couple of blobs before the queue filled. The property + // under guard: NOT one-batch-per-blob (the pre-§6.4 field shape = 40). + assert!( + (3..=8).contains(&batches), + "expected 3..=8 coalesced batches for {EDITS} blobs behind a held gate, saw {batches}" + ); + Ok(()) + } + + /// Duplicated + reversed blobs: Loro buffers out-of-order updates and + /// re-applies duplicates as no-ops — every reply is Ok and the receiver + /// converges regardless of delivery order. (The recorded-inverse no-op-drip + /// preservation is unit-tested in `local_write`; here the property is the + /// PUMP's: no reply is lost or errored under pathological delivery.) + #[test] + fn duplicated_and_reversed_drip_converges() -> Result<()> { + const EDITS: usize = 12; + let (source_text, blobs, gate) = drip_pair(EDITS)?; + let io = DocIoHandle::spawn(Arc::clone(&gate)).expect("io service"); + + let mut schedule: Vec> = Vec::new(); + for blob in blobs.iter().rev() { + schedule.push(blob.clone()); + schedule.push(blob.clone()); // duplicate every blob + } + let replies = pollster::block_on(async { + let mut replies = Vec::new(); + for blob in schedule { + replies.push(io.import_remote_update(blob).await); + } + replies + }); + for (delivery_ix, reply) in replies.iter().enumerate() { + assert!(reply.is_ok(), "delivery {delivery_ix} reply failed: {reply:?}"); + } + assert_eq!( + body_text(&gate), + source_text, + "receiver did not converge under duplicated reversed delivery" + ); + Ok(()) + } + + /// A non-import request interleaved mid-burst is DEFERRED, not dropped: it + /// must still reply correctly after the import chunk completes. + #[test] + fn non_import_request_interleaved_with_burst_still_replies() -> Result<()> { + const EDITS: usize = 10; + let (source_text, blobs, gate) = drip_pair(EDITS)?; + let io = DocIoHandle::spawn(Arc::clone(&gate)).expect("io service"); + + let hold = gate + .lock(GateHolder::DocumentService) + .expect("gate healthy"); + let io_imports = io.clone(); + let import_thread = std::thread::spawn(move || { + pollster::block_on(async move { + let mut replies = Vec::new(); + for blob in blobs { + replies.push(io_imports.import_remote_update(blob).await); + } + replies + }) + }); + // Interleave a snapshot request into the queued burst. + let io_snapshot = io.clone(); + let snapshot_thread = std::thread::spawn(move || pollster::block_on(io_snapshot.projection_snapshot())); + std::thread::sleep(std::time::Duration::from_millis(100)); + drop(hold); + + let replies = import_thread.join().expect("imports join"); + for reply in &replies { + assert!(reply.is_ok(), "import reply failed: {reply:?}"); + } + let snapshot = snapshot_thread.join().expect("snapshot join"); + assert!(snapshot.is_ok(), "deferred non-import request lost/errored: {snapshot:?}"); + assert_eq!(body_text(&gate), source_text); + Ok(()) + } + + /// The publish pump drains committed local events exactly once: first pump + /// returns the `LocalUpdate` batch, an immediate second pump returns nothing. + #[test] + fn pump_publish_drains_exactly_once() -> Result<()> { + let runtime = CrdtRuntime::new_empty("pump-once")?; + let (handle, gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let paragraph = handle.projection()?.ids.paragraph_ids[0]; + // Drain the seed/init traffic first so the assertion isolates OUR edit. + let io = DocIoHandle::spawn(Arc::clone(&gate)).expect("io service"); + let _ = pollster::block_on(io.pump_publish())?; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: "published".into(), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("edit rejected: {error:?}"))?; + let first = pollster::block_on(io.pump_publish())?; + assert!(!first.is_empty(), "first pump must return the committed LocalUpdate events"); + let second = pollster::block_on(io.pump_publish())?; + assert!( + second.is_empty(), + "second pump must be empty — events were already drained (double-publish hazard)" + ); + Ok(()) + } + + #[test] + fn drip_pair_source_holds_no_unsynced_ops() -> Result<()> { + // §task #40 net: after `drip_pair`'s bidirectional seed, the SOURCE must not + // have committed ops the receiver lacks (repair commits during the seed + // import) — the dripped blobs' deps would leave every import PENDING + // forever (canonical "\n\n" while all replies are Ok). + let (_, blobs, gate) = drip_pair(2)?; + { + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + for blob in &blobs { + let events = guard.import_remote_update(blob)?; + let _ = events; + } + } + let text = body_text(&gate); + assert!(text.contains('0'), "receiver applied blob 0 (deps satisfied); got {text:?}"); + Ok(()) + } +} diff --git a/crates/flowstate-collab/tests/field_regressions.rs b/crates/flowstate-collab/tests/field_regressions.rs new file mode 100644 index 00000000..5b107464 --- /dev/null +++ b/crates/flowstate-collab/tests/field_regressions.rs @@ -0,0 +1,624 @@ +//! Regression proofs for the 2026-07-07 04:29–04:36 field-run trio: +//! (A) the projection-stream ordering race ("canonical patch batch base +//! frontier mismatch" cascades), and (C) package exports holding the write +//! gate / I/O loop for up to 18.3s (frozen typing, starved imports, peer pull +//! timeouts). (B — the `session_timers` drift-replace leftover — is deleted +//! code; its regression is (A)'s ordered stream.) + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::doc_io::DocIoHandle; + use flowstate_collab::local_write::{ + GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteAuthority, LocalWriteConfig, SplitParagraphIntent, TextAnchor, WriteGate, + }; + use gpui_flowtext::ProjectionStreamItem; + use gpui_flowtext::{DocumentProjection, ParagraphStyle}; + + fn new_handle(title: &str) -> (LocalDocHandle, Arc>) { + let core = CrdtRuntime::new_empty(title).expect("runtime"); + LocalDocHandle::new(core, LocalWriteConfig::default()) + } + + fn body_string(gate: &Arc>) -> String { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + } + + /// Simulated editor: applies stream items exactly the way + /// `sync_projection_from_authority` does, and PANICS on any base-frontier + /// mismatch — the field symptom that must now be unreachable. + fn apply_stream(editor_projection: &mut DocumentProjection, items: Vec) { + for item in items { + match item { + ProjectionStreamItem::Patches(batch) => { + if batch.new_frontier == editor_projection.frontier { + continue; + } + assert_eq!( + batch.base_frontier, editor_projection.frontier, + "ORDERING RACE: patch base does not match the editor frontier (the field bug)" + ); + gpui_flowtext::apply_projection_patch_batch(editor_projection, &batch).expect("ordered batch applies"); + }, + ProjectionStreamItem::Replace(document) => *editor_projection = *document, + } + } + } + + /// Field bug A: a remote import lands between two local intents. The old + /// split delivery (synchronous local batch + async remote batch) applied the + /// second local batch onto a stale editor frontier and dropped it. The + /// ordered stream must interleave the import batch at its committed position + /// so every item applies cleanly. + #[test] + fn ordered_stream_interleaves_imports_between_local_intents() { + let (handle, gate) = new_handle("ordering"); + let paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + + // A converged peer to produce genuine remote traffic. + let mut peer = CrdtRuntime::new_empty("ordering").expect("peer"); + let seed = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export") + }; + peer.import_remote_update(&seed).expect("peer seed"); + let peer_seed = peer + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer export"); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate"); + guard + .import_remote_update(&peer_seed) + .expect("local imports peer seed"); + drop(guard); + let mut peer_vv = peer.doc().state_vv(); + + // Editor attaches: canonical projection + drain whatever the seed produced. + let mut editor = LocalWriteAuthority::canonical_projection(&handle).expect("attach projection"); + let _ = handle.drain_projection_stream().expect("attach drain"); + + for round in 0..12 { + // Local intent 1. + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("l{round}a"), + style_override: None, + }) + .expect("local intent"); + + // Remote import lands BETWEEN the editor's syncs (peer edits + we import + // while the editor hasn't drained yet). + let peer_body = flowstate_document::loro_schema::body_text(peer.doc()); + peer_body.insert(1, "R").expect("peer insert"); + peer.doc().commit(); + let update = peer + .doc() + .export(loro::ExportMode::updates(&peer_vv)) + .expect("export"); + peer_vv = peer.doc().state_vv(); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate"); + guard.import_remote_update(&update).expect("import"); + drop(guard); + + // Local intent 2 — under the OLD split delivery this batch's base was + // ahead of the editor's frontier and got dropped. + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("l{round}b"), + style_override: None, + }) + .expect("local intent"); + + // Editor drains ONCE per round: local batch, import batch(es), local + // batch — in commit order, every base matching. + apply_stream(&mut editor, handle.drain_projection_stream().expect("drain")); + } + + // The editor's replica equals the canonical projection exactly. + let canonical = LocalWriteAuthority::canonical_projection(&handle).expect("canonical"); + assert_eq!(editor.frontier, canonical.frontier, "editor tracked every committed change"); + assert_eq!(editor.paragraphs.len(), canonical.paragraphs.len()); + for ix in 0..editor.paragraphs.len() { + assert_eq!( + flowstate_document::paragraph_text(&editor, ix), + flowstate_document::paragraph_text(&canonical, ix), + "paragraph {ix} text" + ); + } + } + + /// 2026-07-07 ctrl-A + undo field freeze: undo/redo of a mass styled delete + /// must complete, converge, and match a fresh full rematerialization — the + /// path crosses the linear event compose (vendored), the dead-anchor-skip + /// materializer, and the O(1)-per-defect repair pass, each of which was a + /// multi-minute freeze in the field. + #[test] + fn mass_delete_undo_redo_round_trips() { + let (handle, _gate) = new_handle("mass-undo"); + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..60 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("styled paragraph {i} with enough text to matter"), + style_override: Some(flowstate_document::RunStyles { + semantic: flowstate_document::RunSemanticStyle::Custom(2), + ..Default::default() + }), + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("last paragraph"); + } + let mut editor = LocalWriteAuthority::canonical_projection(&handle).expect("attach"); + let _ = handle.drain_projection_stream().expect("attach drain"); + let full_text = |projection: &DocumentProjection| { + (0..projection.paragraphs.len()) + .map(|ix| flowstate_document::paragraph_text(projection, ix)) + .collect::>() + }; + let before = full_text(&editor); + + // ctrl-A delete. + let projection = handle.projection().expect("projection"); + let first = projection.ids.paragraph_ids[0]; + let last = *projection.ids.paragraph_ids.last().expect("paragraphs"); + handle + .delete_range(flowstate_collab::local_write::DeleteRangeIntent { + start: TextAnchor::new(first, 0), + end: TextAnchor::new(last, usize::MAX), + }) + .expect("select-all delete"); + apply_stream(&mut editor, handle.drain_projection_stream().expect("drain")); + assert_eq!(editor.paragraphs.len(), 1, "select-all delete leaves the sentinel paragraph"); + + // Undo restores every paragraph's text; redo empties again; both keep the + // editor byte-identical to canonical. + handle.apply_undo().expect("undo mass delete"); + apply_stream(&mut editor, handle.drain_projection_stream().expect("drain")); + let canonical = LocalWriteAuthority::canonical_projection(&handle).expect("canonical"); + assert_eq!(editor.frontier, canonical.frontier); + assert_eq!(full_text(&editor), before, "undo restored the full document text"); + assert_eq!(full_text(&editor), full_text(&canonical)); + + handle.apply_redo().expect("redo mass delete"); + apply_stream(&mut editor, handle.drain_projection_stream().expect("drain")); + let canonical = LocalWriteAuthority::canonical_projection(&handle).expect("canonical"); + assert_eq!(editor.frontier, canonical.frontier); + assert_eq!(editor.paragraphs.len(), canonical.paragraphs.len()); + assert_eq!(canonical.paragraphs.len(), 1, "redo re-empties the document"); + } + + /// §act-three B.1 timing A/B (run on demand): seed a large multi-paragraph + /// document, mass-delete it, and time undo+redo with the recorded-inverse + /// fast path vs the checkout-based slow path + /// (`FLOWSTATE_DISABLE_FAST_UNDO=1`). Prints the deltas; not asserted (timing + /// is machine-dependent). Run with: + /// ```text + /// cargo test -p flowstate-collab --test field_regressions --release \ + /// recorded_inverse_undo_timing -- --ignored --nocapture + /// ``` + #[test] + #[ignore = "timing harness: run on demand with --nocapture"] + fn recorded_inverse_undo_timing() { + use std::time::Instant; + let paragraphs = std::env::var("B1_PARAGRAPHS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(4000usize); + let (handle, _gate) = new_handle("b1-timing"); + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..paragraphs { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("paragraph {i} with a realistic amount of styled debate-card body text to weigh the range"), + style_override: Some(flowstate_document::RunStyles { + semantic: flowstate_document::RunSemanticStyle::Custom(2), + ..Default::default() + }), + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("last"); + } + let projection = handle.projection().expect("projection"); + let first = projection.ids.paragraph_ids[0]; + let last = *projection.ids.paragraph_ids.last().expect("paragraphs"); + let delete = |handle: &LocalDocHandle| { + handle + .delete_range(flowstate_collab::local_write::DeleteRangeIntent { + start: TextAnchor::new(first, 0), + end: TextAnchor::new(last, usize::MAX), + }) + .expect("select-all delete"); + }; + delete(&handle); + // Warm one undo/redo cycle. + let t = Instant::now(); + handle.apply_undo().expect("undo"); + let undo_ms = t.elapsed().as_secs_f64() * 1e3; + let _ = handle.drain_projection_stream(); + let t = Instant::now(); + handle.apply_redo().expect("redo"); + let redo_ms = t.elapsed().as_secs_f64() * 1e3; + let _ = handle.drain_projection_stream(); + eprintln!( + "B.1 timing: paragraphs={paragraphs} fast_undo_disabled={} undo={undo_ms:.1}ms redo={redo_ms:.1}ms", + std::env::var("FLOWSTATE_DISABLE_FAST_UNDO").is_ok() + ); + } + + /// Spec §6-R: a remote peer's structural edits (Enter/Backspace storms) must + /// take the REGIONAL rematerializer — exact patches (`ProjectionPatched`), + /// never a full-document rebuild (`ProjectionUpdated`) — and the ordered + /// stream must keep the editor byte-identical to canonical throughout. + #[test] + fn remote_structural_chunks_take_the_regional_path() { + let (handle, gate) = new_handle("regional"); + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + + // A meaty multi-paragraph doc so regions are genuine sub-spans. + for i in 0..80 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("paragraph {i} body text with some length to it"), + style_override: None, + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("last paragraph"); + } + + // Peer forks from the doc's own snapshot; export baseline predates runtime + // startup so its chunks are causally complete. + let snapshot = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard + .doc() + .export(loro::ExportMode::Snapshot) + .expect("snapshot") + }; + let peer_doc = loro::LoroDoc::new(); + peer_doc + .import_with(&snapshot, "remote") + .expect("peer join"); + let mut peer_vv = peer_doc.state_vv(); + let peer_core = flowstate_collab::crdt_runtime::CrdtRuntime::from_doc(peer_doc, None, None).expect("peer runtime"); + let (peer_handle, peer_gate) = + LocalDocHandle::new(peer_core, flowstate_collab::local_write::LocalWriteConfig { release_audit_sample: None }); + let peer_target = peer_handle + .projection() + .expect("peer projection") + .ids + .paragraph_ids[40]; + + // Editor attaches to the main handle. + let mut editor = LocalWriteAuthority::canonical_projection(&handle).expect("attach projection"); + let _ = handle.drain_projection_stream().expect("attach drain"); + + // Storm: 10 chunks, each a typing burst + an Enter (structural), then a + // Backspace-join chunk and a cross-paragraph delete chunk. + let mut structural_chunks = 0_usize; + let mut rebuilds = 0_usize; + for round in 0..12 { + match round { + 10 => { + // Backspace at paragraph start: join the two peers' split rows. + let projection = peer_handle.projection().expect("peer projection"); + let ix = projection + .ids + .paragraph_ids + .iter() + .position(|id| *id == peer_target) + .expect("target present"); + let second = projection.ids.paragraph_ids[ix + 1]; + peer_handle + .join_paragraphs(flowstate_collab::local_write::JoinParagraphsIntent { first: peer_target, second }) + .expect("peer join"); + }, + 11 => { + // Cross-paragraph delete spanning a boundary. + let projection = peer_handle.projection().expect("peer projection"); + let ix = projection + .ids + .paragraph_ids + .iter() + .position(|id| *id == peer_target) + .expect("target present"); + let next = projection.ids.paragraph_ids[ix + 1]; + peer_handle + .delete_range(flowstate_collab::local_write::DeleteRangeIntent { + start: TextAnchor::new(peer_target, 3), + end: TextAnchor::new(next, 2), + }) + .expect("peer cross-paragraph delete"); + }, + _ => { + for _ in 0..3 { + peer_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_target, usize::MAX), + text: "r".to_string(), + style_override: None, + }) + .expect("peer typing"); + } + peer_handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(peer_target, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("peer split"); + }, + } + let update = { + let guard = peer_gate.lock(GateHolder::ExportUpdates).expect("gate"); + let update = guard + .doc() + .export(loro::ExportMode::updates(&peer_vv)) + .expect("export"); + peer_vv = guard.doc().state_vv(); + update + }; + structural_chunks += 1; + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate"); + let events = guard.import_remote_update(&update).expect("import"); + drop(guard); + for event in &events { + if matches!(event, flowstate_collab::crdt_runtime::RuntimeEvent::ProjectionUpdated { .. }) { + rebuilds += 1; + } + } + apply_stream(&mut editor, handle.drain_projection_stream().expect("drain")); + } + assert_eq!( + rebuilds, 0, + "all {structural_chunks} structural chunks must take the regional patched path, never a full rebuild" + ); + + let canonical = LocalWriteAuthority::canonical_projection(&handle).expect("canonical"); + assert_eq!(editor.frontier, canonical.frontier, "editor tracked every structural import"); + assert_eq!(editor.paragraphs.len(), canonical.paragraphs.len()); + for ix in 0..editor.paragraphs.len() { + assert_eq!( + flowstate_document::paragraph_text(&editor, ix), + flowstate_document::paragraph_text(&canonical, ix), + "paragraph {ix} text" + ); + } + } + + /// Field bug C: package exports held the `DocumentService` gate (and the + /// serial I/O loop) for up to 18.3s — typing froze behind the gate and + /// imports starved. Post-fix the export forks under a SHORT hold and + /// assembles off-thread: typing during an in-flight package export must stay + /// responsive, and no non-import gate hold may approach the old freeze. + #[test] + fn package_export_does_not_stall_typing() { + let (handle, gate) = new_handle("package-stall"); + let projection = handle.projection().expect("projection"); + let mut paragraph = projection.ids.paragraph_ids[0]; + + // A meaty document so assembly is measurably slower than a fork. + for i in 0..300 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("paragraph {i} with a reasonable amount of body text for sizing"), + style_override: None, + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("last paragraph"); + } + + let io = DocIoHandle::spawn(Arc::clone(&gate)).expect("io service"); + // Fire the package export (the recovery-snapshot shape from the field + // logs) and type while it assembles. + let export = std::thread::spawn(move || pollster::block_on(io.package_bytes("Recovery snapshot".to_string()))); + + let mut worst_keystroke = std::time::Duration::ZERO; + for i in 0..40 { + let started = std::time::Instant::now(); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("{i}"), + style_override: None, + }) + .expect("typing during export"); + worst_keystroke = worst_keystroke.max(started.elapsed()); + } + let bytes = export + .join() + .expect("export thread") + .expect("package bytes"); + assert!(!bytes.is_empty(), "export still produces the package"); + + // Generous debug-build ceiling: the OLD shape held the gate for the whole + // assembly (seconds); the fork-only hold is milliseconds. + assert!( + worst_keystroke < std::time::Duration::from_millis(750), + "typing stalled behind the package export: worst keystroke {worst_keystroke:?}" + ); + } + + /// Longevity: a DEEP undo/redo chain (every op undone to the seed, then + /// fully redone) round-trips exactly and keeps the maintained projection + /// equal to a fresh rematerialization at every checkpoint — undo depth must + /// never accumulate drift. + #[test] + fn deep_undo_redo_chain_round_trips() { + let (handle, gate) = new_handle("deep-undo"); + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + let mut checkpoints: Vec = vec![body_string(&gate)]; + for step in 0..60 { + if step % 7 == 6 { + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("paragraph"); + } else { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("w{step} "), + style_override: None, + }) + .expect("insert"); + } + checkpoints.push(body_string(&gate)); + } + + let self_consistent = |label: &str| { + let maintained = handle.projection().expect("projection"); + let fresh = LocalWriteAuthority::canonical_projection(&handle).expect("canonical"); + assert_eq!(maintained.paragraphs.len(), fresh.paragraphs.len(), "{label}: paragraph count drifted"); + for ix in 0..maintained.paragraphs.len() { + assert_eq!( + flowstate_document::paragraph_text(&maintained, ix), + flowstate_document::paragraph_text(&fresh, ix), + "{label}: paragraph {ix} drifted" + ); + } + }; + + // Unwind the WHOLE chain, checking text at every step and consistency + // every tenth. + for step in (0..60).rev() { + handle.apply_undo().expect("undo"); + assert_eq!(body_string(&gate), checkpoints[step], "undo depth {} text mismatch", 60 - step); + if step % 10 == 0 { + self_consistent("undo checkpoint"); + } + } + // And forward again. + for step in 0..60 { + handle.apply_redo().expect("redo"); + assert_eq!(body_string(&gate), checkpoints[step + 1], "redo step {step} text mismatch"); + if step % 10 == 0 { + self_consistent("redo checkpoint"); + } + } + } + + /// Revisions during live edits: opening and forking a stored revision is a + /// READ — it must not disturb the maintained projection, the ordered + /// stream, or subsequent undo. + #[test] + fn revision_open_during_live_edits_is_inert() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("live-revisions.db8"); + let doc = flowstate_document::new_loro_document("Live").expect("doc"); + let mut package = flowstate_document::DocumentPackage::from_loro_snapshot(&doc, "Live").expect("package"); + let revision_id = package + .create_named_revision(&doc, "Blank", "Blank document", None, None) + .expect("revision"); + package.write(&path).expect("write"); + + let runtime = CrdtRuntime::open_package(&path).expect("open"); + let (handle, gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: "typed before open".into(), + style_override: None, + }) + .expect("insert"); + + // Open + fork the revision mid-session. + let mut guard = gate.lock(GateHolder::UndoRedo).expect("gate"); + let events = guard + .command(flowstate_collab::crdt_runtime::SemanticCommand::OpenRevision { revision_id }) + .expect("open revision"); + assert_eq!(events.len(), 1, "revision open emits exactly the read event"); + let events = guard + .command(flowstate_collab::crdt_runtime::SemanticCommand::ForkRevision { revision_id }) + .expect("fork revision"); + assert_eq!(events.len(), 1, "revision fork emits exactly the fork event"); + drop(guard); + + // The live document is untouched: same text, editing + undo keep working. + assert!(body_string(&gate).contains("typed before open")); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: " and after".into(), + style_override: None, + }) + .expect("insert after open"); + handle.apply_undo().expect("undo"); + let text = body_string(&gate); + assert!( + text.contains("typed before open") && !text.contains("and after"), + "undo reverts only the live edit: {text:?}" + ); + let maintained = handle.projection().expect("projection"); + let fresh = LocalWriteAuthority::canonical_projection(&handle).expect("canonical"); + assert_eq!(maintained.paragraphs.len(), fresh.paragraphs.len()); + } +} diff --git a/crates/flowstate-collab/tests/intent_complexity.rs b/crates/flowstate-collab/tests/intent_complexity.rs new file mode 100644 index 00000000..b1122faf --- /dev/null +++ b/crates/flowstate-collab/tests/intent_complexity.rs @@ -0,0 +1,306 @@ +//! Complexity-shape counter assertions (Loro-first spec §11/§13.10, I-8). +//! +//! Counts, not clocks: a single-character insert into a LARGE document must +//! touch O(1) containers and emit O(inserted-bytes) work — never per-run, +//! per-paragraph, or per-block work. These assertions cannot be satisfied by +//! faster hardware or left-on-the-table optimizations; they pin the complexity +//! class itself. Wall-clock ratchet ceilings live in the release perf harness +//! (env-gated), not here. + +#[cfg(test)] +mod tests { + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{ + DeleteRangeIntent, InsertObjectIntent, InsertTextIntent, JoinParagraphsIntent, LocalDocHandle, LocalWriteConfig, SetMarksIntent, + SetParagraphStylesIntent, SplitParagraphIntent, TextAnchor, + }; + use flowstate_document::{ + AssetId, InputBlock, InputBlockAlignment, InputImageBlock, InputImageSizing, ParagraphStyle, RunStyles, block_ix_for_paragraph, + block_ix_scan_count, reset_block_ix_scan_count, + }; + + /// Paragraph count for the "large" fixture. Kept CI-friendly in debug builds + /// (the always-on debug audit full-rebuilds per commit by design); the shape + /// assertions are size-independent, which is exactly the point. + const PARAGRAPHS: usize = 400; + + fn large_fixture() -> LocalDocHandle { + let core = CrdtRuntime::new_empty("complexity").expect("runtime"); + let (handle, _gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..PARAGRAPHS - 1 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("paragraph {i} body text with several words"), + style_override: None, + }) + .expect("seed insert"); + let outcome = handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + let projection = match outcome { + flowstate_collab::local_write::LocalWriteOutcome::Committed(_) => handle.projection().expect("projection"), + flowstate_collab::local_write::LocalWriteOutcome::CommittedWithRebuild { .. } => handle.projection().expect("projection"), + }; + paragraph = *projection.ids.paragraph_ids.last().expect("last paragraph"); + } + handle + } + + /// Like [`large_fixture`] but with image objects interspersed every + /// `OBJECT_EVERY` paragraphs, so `blocks.len() != paragraphs.len()` and the + /// O(1) aligned fast path in `block_ix_for_paragraph` MISSES — the exact + /// condition under which a per-paragraph caller becomes the §perf-heaven T2 + /// quadratic. Returns the handle plus the paragraph count. + fn object_bearing_fixture() -> (LocalDocHandle, usize) { + const OBJECT_EVERY: usize = 8; + let core = CrdtRuntime::new_empty("t2-complexity").expect("runtime"); + let (handle, _gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + // 1. Build the paragraphs (text + split), no objects — the stable seed. + let mut paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..PARAGRAPHS - 1 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("paragraph {i} body text with several words"), + style_override: None, + }) + .expect("seed insert"); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .expect("seed split"); + paragraph = *handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .last() + .expect("last paragraph"); + } + // 2. Insert image objects at byte 0 of every OBJECT_EVERY-th paragraph — the + // identity-anchored position the convergence tests use (robust, unlike the + // end-of-paragraph interleave). Tolerate the rare reject; we only need SOME + // objects so `blocks.len() != paragraphs.len()` and the aligned fast path + // misses. Snapshot the ids first so the shifting projection doesn't matter. + let ids = handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .clone(); + for (n, paragraph_id) in ids.iter().enumerate() { + if n % OBJECT_EVERY == OBJECT_EVERY / 2 { + let _ = handle.insert_object(InsertObjectIntent { + at: TextAnchor::new(*paragraph_id, 0), + block: InputBlock::Image(InputImageBlock { + asset_id: AssetId(1), + alt_text: format!("img{n}"), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Left, + external_url: None, + }), + }); + } + } + let projection = handle.projection().expect("projection"); + (handle, projection.paragraphs.len()) + } + + /// §perf-heaven T2 tripwire. On an object-bearing doc (fast path misses), the + /// mass-op patch-synthesis loops must NOT scan `block_ix_for_paragraph` once + /// per paragraph — that was the O(paragraphs²) cost behind mass-restyle, + /// select-all mark, replace-all, and cross-paragraph delete. This test is + /// self-validating: it first proves the counter TRIPS under a naive + /// per-paragraph lookup (so the guard is not vacuous), then asserts the real + /// mass ops keep it bounded by a small constant. + #[test] + fn mass_ops_do_not_scan_block_index_per_paragraph() { + let (handle, paragraph_count) = object_bearing_fixture(); + let projection = handle.projection().expect("projection"); + assert!( + projection.blocks.len() > projection.paragraphs.len(), + "fixture must carry objects so the aligned fast path misses (blocks={}, paragraphs={})", + projection.blocks.len(), + projection.paragraphs.len(), + ); + + // (a) Prove the tripwire can TRIP: a naive per-paragraph lookup on THIS doc + // drives one O(blocks) scan per paragraph. If this does not climb, the + // fixture is not exercising the quadratic-prone path and the guard below + // would be meaningless. + reset_block_ix_scan_count(); + for ix in 0..projection.paragraphs.len() { + let _ = block_ix_for_paragraph(&projection, ix); + } + let naive_scans = block_ix_scan_count(); + assert!( + naive_scans >= paragraph_count as u64, + "object fixture should force a scan per paragraph under naive lookup, got {naive_scans} for {paragraph_count} paragraphs", + ); + + // (b) The real select-all mark op must stay bounded — O(a few), not + // O(paragraphs). It routes through the hoisted `paragraph_block_rows`. + let all = projection.ids.paragraph_ids.to_vec(); + let first = *all.first().expect("first paragraph"); + let last = *all.last().expect("last paragraph"); + reset_block_ix_scan_count(); + handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(first, 0), + end: TextAnchor::new(last, usize::MAX), + styles: RunStyles::default(), + }) + .expect("mass set-marks commits"); + let mark_scans = block_ix_scan_count(); + assert!( + mark_scans <= 16, + "select-all mark scanned block index per paragraph (T2 quadratic regression): {mark_scans} scans over {paragraph_count} paragraphs", + ); + + // (c) The real mass paragraph-style op must stay bounded too. + reset_block_ix_scan_count(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: all, + style: ParagraphStyle::Custom(2), + }) + .expect("mass set-paragraph-styles commits"); + let style_scans = block_ix_scan_count(); + assert!( + style_scans <= 16, + "mass restyle scanned block index per paragraph (T2 quadratic regression): {style_scans} scans over {paragraph_count} paragraphs", + ); + } + + #[test] + fn single_char_insert_is_o1_in_document_size() { + let handle = large_fixture(); + let projection = handle.projection().expect("projection"); + assert!(projection.paragraphs.len() >= PARAGRAPHS - 1); + let middle = projection.ids.paragraph_ids[projection.paragraphs.len() / 2]; + + let outcome = handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(middle, 3), + text: "x".into(), + style_override: None, + }) + .expect("insert commits"); + let counters = outcome.commit().counters; + assert!(!counters.full_rebuild, "single-char insert must never full-rebuild (I-14)"); + assert!( + counters.containers_touched <= 2, + "insert touches O(1) containers, got {}", + counters.containers_touched + ); + assert!(counters.loro_ops <= 4, "insert emits O(1) Loro ops, got {}", counters.loro_ops); + assert_eq!( + counters.marks_emitted, 0, + "plain typing emits ZERO style ops (spec §9 — inheritance is Loro's job)" + ); + assert!(counters.patch_count <= 2, "insert patches O(1) paragraphs, got {}", counters.patch_count); + } + + #[test] + fn insert_with_style_override_marks_only_inserted_range() { + let handle = large_fixture(); + let projection = handle.projection().expect("projection"); + let middle = projection.ids.paragraph_ids[projection.paragraphs.len() / 2]; + let outcome = handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(middle, 0), + text: "bold".into(), + style_override: Some(RunStyles::default()), + }) + .expect("insert commits"); + let counters = outcome.commit().counters; + assert!(!counters.full_rebuild); + // Marks bounded by the style-key count, never by run/paragraph counts. + assert!( + counters.marks_emitted <= 8, + "override marks are O(style keys), got {}", + counters.marks_emitted + ); + assert!(counters.loro_ops <= 12, "override insert stays O(1), got {}", counters.loro_ops); + } + + #[test] + fn split_and_join_are_o1_in_document_size() { + let handle = large_fixture(); + let projection = handle.projection().expect("projection"); + let mid_ix = projection.paragraphs.len() / 2; + let middle = projection.ids.paragraph_ids[mid_ix]; + + let outcome = handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(middle, 5), + inherited_style: ParagraphStyle::Normal, + }) + .expect("split commits"); + let counters = outcome.commit().counters; + assert!(!counters.full_rebuild, "split must patch, not rebuild"); + assert!(counters.patch_count <= 3, "split patches O(1) blocks, got {}", counters.patch_count); + + let projection = handle.projection().expect("projection"); + let first = projection.ids.paragraph_ids[mid_ix]; + let second = projection.ids.paragraph_ids[mid_ix + 1]; + let outcome = handle + .join_paragraphs(JoinParagraphsIntent { first, second }) + .expect("join commits"); + let counters = outcome.commit().counters; + assert!(!counters.full_rebuild, "join must patch, not rebuild"); + assert!(counters.patch_count <= 3, "join patches O(1) blocks, got {}", counters.patch_count); + } + + #[test] + fn delete_range_cost_bounded_by_range_not_document() { + let handle = large_fixture(); + let projection = handle.projection().expect("projection"); + let middle = projection.ids.paragraph_ids[projection.paragraphs.len() / 2]; + let outcome = handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(middle, 0), + end: TextAnchor::new(middle, 5), + }) + .expect("delete commits"); + let counters = outcome.commit().counters; + assert!(!counters.full_rebuild); + assert!(counters.containers_touched <= 2); + assert!(counters.patch_count <= 2); + } + + #[test] + fn style_range_cost_bounded_by_touched_paragraphs() { + let handle = large_fixture(); + let projection = handle.projection().expect("projection"); + let ix = projection.paragraphs.len() / 2; + let paragraph = projection.ids.paragraph_ids[ix]; + let outcome = handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(paragraph, 0), + end: TextAnchor::new(paragraph, usize::MAX), + styles: RunStyles::default(), + }) + .expect("marks commit"); + let counters = outcome.commit().counters; + assert!(!counters.full_rebuild); + assert!( + counters.patch_count <= 2, + "one-paragraph restyle patches one paragraph, got {}", + counters.patch_count + ); + assert!( + counters.marks_emitted <= 8, + "restyle emits O(style keys) marks, got {}", + counters.marks_emitted + ); + } +} diff --git a/crates/flowstate-collab/tests/intent_fuzz.rs b/crates/flowstate-collab/tests/intent_fuzz.rs new file mode 100644 index 00000000..0add65d2 --- /dev/null +++ b/crates/flowstate-collab/tests/intent_fuzz.rs @@ -0,0 +1,627 @@ +//! N-peer randomized intent convergence fuzz (Loro-first spec §13.7). +//! +//! Every peer edits exclusively through its `LocalDocHandle` (the one write +//! path), with randomized intents — inserts, deletes, splits, joins, marks, +//! paragraph styles, expand-mark boundary typing, split-at-styled-run-end, +//! rich-fragment paste, undo, redo — and randomized update exchange. After every sync round, all peers must +//! agree on BOTH canonical Loro state and the materialized projection. +//! Undo/redo run through the production `apply_undo`/`apply_redo` path (the +//! Loro `UndoManager` with remote-origin exclusion), so concurrent undos +//! against interleaved remote edits are part of the convergence bar. +//! Includes the §13.13 zero-patch-frontier-churn regression (the 19:28 field +//! wound): a frontier-only remote advance must never disturb local typing. + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{ + DeleteRangeIntent, FragmentBlock, GateHolder, InsertRichFragmentIntent, InsertTextIntent, JoinParagraphsIntent, LocalDocHandle, + LocalWriteConfig, ReplaceMatch, ReplaceMatchesIntent, SetMarksIntent, SetParagraphStyleIntent, SplitParagraphIntent, TextAnchor, WriteGate, + WriteRejected, + }; + use flowstate_document::{Block, DocumentProjection, InputParagraph, InputRun, ParagraphStyle, RunSemanticStyle, RunStyles, paragraph_text}; + + /// Deterministic xorshift PRNG — reproducible fuzz, no wall-clock dependence. + struct Rng(u64); + + impl Rng { + fn new(seed: u64) -> Self { + Self(seed.max(1)) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, bound: usize) -> usize { + if bound == 0 { 0 } else { (self.next() % bound as u64) as usize } + } + } + + struct Peer { + handle: LocalDocHandle, + gate: Arc>, + /// Version vector snapshot per remote peer we last pulled from — plain + /// full-history exchange keeps the harness simple and deterministic. + synced_vv: Vec, + } + + impl Peer { + fn new(title: &str, peer_count: usize) -> Self { + let core = CrdtRuntime::new_empty(title).expect("runtime"); + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + Self { + handle, + gate, + synced_vv: vec![loro::VersionVector::default(); peer_count], + } + } + + fn projection(&self) -> DocumentProjection { + self.handle.projection().expect("projection") + } + + fn export_updates_since(&self, vv: &loro::VersionVector) -> Vec { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard + .doc() + .export(loro::ExportMode::updates(vv)) + .expect("export") + } + + fn state_vv(&self) -> loro::VersionVector { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard.doc().state_vv() + } + + fn import(&self, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + let mut guard = self.gate.lock(GateHolder::ImportChunk).expect("gate"); + guard.import_remote_update(bytes).expect("import"); + } + + fn body_text(&self) -> String { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + } + + /// A fresh full rematerialization of THIS peer's canonical Loro state — + /// deliberately not `canonical_projection()` (which returns the maintained + /// projection and would make self-consistency checks vacuous). + fn fresh_canonical(&self) -> DocumentProjection { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::document_from_loro(guard.doc()).expect("fresh rematerialization") + } + } + + fn random_styles(rng: &mut Rng) -> RunStyles { + let mut styles = RunStyles::default(); + if rng.below(2) == 0 { + styles.semantic = RunSemanticStyle::Custom((rng.below(4) + 1) as u8); + } + if rng.below(3) == 0 { + styles.direct_underline = true; + } + if rng.below(4) == 0 { + styles.strikethrough = true; + } + styles + } + + /// One random intent against `peer`. Rejections for stale identities are legal + /// outcomes (I-15) — the fuzz only demands convergence, not acceptance. + fn random_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let projection = peer.projection(); + if projection.paragraphs.is_empty() { + return Ok(()); + } + let paragraph_ix = rng.below(projection.paragraphs.len()); + let paragraph = projection.ids.paragraph_ids[paragraph_ix]; + let text_len = flowstate_document::paragraph_text_len(&projection.paragraphs[paragraph_ix]); + let byte = rng.below(text_len + 1); + let arm = rng.below(18); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() { + eprintln!("step {step}: arm {arm} paragraph_ix {paragraph_ix} byte {byte}"); + } + match arm { + // §act-three B.1 equivalence leg (spec §9.2): a qualifying MASS delete + // followed by undo (the recorded-inverse fast path when nothing + // interleaves; the checkout slow path when remote history or the arm's + // own coin flips got in the way — both must satisfy the same round + // oracle: projection == fresh canonical materialization, N-peer + // convergence). Seeds its own bulk so the capture threshold is met + // regardless of accumulated document size. + 16 => { + let bulk: Vec = (0..24) + .map(|ix| { + FragmentBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text: format!( + "m{step}r{ix:02} bulk row content padding padding padding padding padding padding padding padding padding padding padding" + ), + styles: if ix % 3 == 0 { random_styles(rng) } else { RunStyles::default() }, + }], + }) + }) + .collect(); + peer + .handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, byte), + blocks: bulk, + }) + .map(|_| ())?; + // Re-read the projection: the fragment changed the paragraph map. + let projection = peer.projection(); + let start_ix = paragraph_ix.min(projection.paragraphs.len().saturating_sub(1)); + let end_ix = (start_ix + 18 + rng.below(6)).min(projection.paragraphs.len() - 1); + if end_ix <= start_ix { + return Ok(()); + } + let start_id = projection.ids.paragraph_ids[start_ix]; + let end_id = projection.ids.paragraph_ids[end_ix]; + let end_len = flowstate_document::paragraph_text_len(&projection.paragraphs[end_ix]); + peer + .handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(start_id, rng.below(4)), + end: TextAnchor::new(end_id, rng.below(end_len + 1)), + }) + .map(|_| ())?; + if rng.below(4) != 0 { + peer.handle.apply_undo().map(|_| ())?; + if rng.below(2) == 0 { + peer.handle.apply_redo().map(|_| ())?; + } + } + Ok(()) + }, + // Weighted toward typing: inserts dominate real sessions. + 0..=3 => { + let text = format!("s{step}p{paragraph_ix}"); + peer + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, byte), + text, + style_override: (rng.below(4) == 0).then(|| random_styles(rng)), + }) + .map(|_| ()) + }, + 4 => { + // Boundary typing at the very end of the paragraph — the expand-`After` + // inheritance case (spec §9). + peer + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("e{step}"), + style_override: None, + }) + .map(|_| ()) + }, + 5 => { + if text_len == 0 { + return Ok(()); + } + let start = rng.below(text_len); + let end = (start + 1 + rng.below(3)).min(text_len); + peer + .handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(paragraph, start), + end: TextAnchor::new(paragraph, end), + }) + .map(|_| ()) + }, + 6 => { + // Split — including split-at-styled-run-end (mark the tail first on a + // coin flip so the sentinel-hygiene rule gets fuzzed, spec §13.7). + if rng.below(2) == 0 && text_len > 0 { + let mark_start = text_len.saturating_sub(1 + rng.below(2)).min(text_len - 1); + let _ = peer.handle.set_marks(SetMarksIntent { + start: TextAnchor::new(paragraph, mark_start), + end: TextAnchor::new(paragraph, text_len), + styles: random_styles(rng), + }); + } + peer + .handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, byte), + inherited_style: if rng.below(2) == 0 { + ParagraphStyle::Normal + } else { + ParagraphStyle::Custom(1) + }, + }) + .map(|_| ()) + }, + 7 => { + if paragraph_ix + 1 >= projection.paragraphs.len() { + return Ok(()); + } + let second = projection.ids.paragraph_ids[paragraph_ix + 1]; + peer + .handle + .join_paragraphs(JoinParagraphsIntent { first: paragraph, second }) + .map(|_| ()) + }, + 8 => { + if text_len == 0 { + return Ok(()); + } + let start = rng.below(text_len); + let end = (start + 1 + rng.below(4)).min(text_len); + peer + .handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(paragraph, start), + end: TextAnchor::new(paragraph, end), + styles: random_styles(rng), + }) + .map(|_| ()) + }, + 9 => { + // Rich-fragment paste (the compound intent): two short paragraphs in one + // gate hold / one commit. + let fragment_paragraph = |text: String| { + FragmentBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text, + styles: RunStyles::default(), + }], + }) + }; + peer + .handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, byte), + blocks: vec![fragment_paragraph(format!("f{step}a")), fragment_paragraph(format!("f{step}b"))], + }) + .map(|_| ()) + }, + 10 => peer + .handle + .set_paragraph_style(SetParagraphStyleIntent { + paragraph, + style: if rng.below(2) == 0 { + ParagraphStyle::Normal + } else { + ParagraphStyle::Custom((rng.below(3) + 1) as u8) + }, + }) + .map(|_| ()), + // Undo/redo (spec §10): production `UndoManager` path. "Nothing to + // undo" is a legal no-op; the convergence bar below still applies to + // whatever ops the transform emits against interleaved remote history. + 11 => peer.handle.apply_undo().map(|_| ()), + 12 => peer.handle.apply_redo().map(|_| ()), + // Undo GROUP (spec §10): a word-burst of grouped inserts. Undoing later + // must revert the whole group as one member; a non-disjoint remote + // import may close the group early (F3) — both fates are legal, and + // convergence must hold either way. + 13 => { + let _ = peer.handle.begin_undo_group(); + for burst in 0..(2 + rng.below(3)) { + let _ = peer.handle.insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("g{step}b{burst}"), + style_override: None, + }); + } + peer.handle.finish_undo_group().map(|_| ()) + }, + // Batched selection-wide restyle (§11): one SetParagraphStyles intent + // over a random contiguous span — the editor's select-all restyle + // shape. Exercises the batched boundary resolution, the skip-not-reject + // law for stale/interstitial rows, and single-member undo of a mass + // restyle (arms 11/12 undo it against interleaved remote history). + 14 => { + let span = 1 + rng.below(4.min(projection.paragraphs.len() - paragraph_ix).max(1)); + let paragraphs = (paragraph_ix..(paragraph_ix + span).min(projection.paragraphs.len())) + .map(|ix| projection.ids.paragraph_ids[ix]) + .collect(); + peer + .handle + .set_paragraph_styles(flowstate_collab::local_write::SetParagraphStylesIntent { + paragraphs, + style: if rng.below(2) == 0 { + ParagraphStyle::Normal + } else { + ParagraphStyle::Custom((rng.below(3) + 1) as u8) + }, + }) + .map(|_| ()) + }, + // §act-eleven C1: AUTOSAVE CHECKPOINT interleaved with everything else. + // The in-place checkpoint path commits undo-inert "meta" metadata + + // revision ops (vendor patch #14), creates/compacts the package, and + // re-arms the recorded-inverse stacks — the exact autosave x undo x + // import interleaving class that killed field undo (act-ten A10.2), + // previously covered only by two targeted unit tests. A checkpoint must + // never fail; convergence of the meta ops rides the normal sync rounds. + 15 => { + let dir = std::env::temp_dir().join(format!("flowstate-fuzz-ckpt-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fuzz checkpoint dir"); + let path = dir.join(format!("peer-{:p}.db8", std::sync::Arc::as_ptr(&peer.gate))); + let mut guard = peer + .gate + .lock(GateHolder::DocumentService) + .expect("gate healthy"); + guard + .checkpoint_package("Fuzz autosave", Some(path)) + .expect("fuzz checkpoint must succeed"); + drop(guard); + Ok(()) + }, + // Replace-all (find & replace): every occurrence of a random pattern in + // up to three random paragraphs, one compound intent. Exercises the + // multi-range back-to-front application, the per-paragraph readback + // shift, and the concurrent skip/prune rules. + _ => { + let needle = b"aes0"[rng.below(4)] as char; + let mut matches = Vec::new(); + for _ in 0..(1 + rng.below(3)) { + let target_ix = rng.below(projection.paragraphs.len()); + let target = projection.ids.paragraph_ids[target_ix]; + let text = flowstate_document::paragraph_text(&projection, target_ix); + for (byte, ch) in text.char_indices() { + if ch == needle { + matches.push(ReplaceMatch { + start: TextAnchor::new(target, byte), + end: TextAnchor::new(target, byte + ch.len_utf8()), + styles: (rng.below(3) == 0).then(|| random_styles(rng)), + }); + } + } + } + let replacement = match rng.below(3) { + 0 => String::new(), + 1 => format!("r{step}"), + _ => "×".to_string(), + }; + peer + .handle + .replace_matches(ReplaceMatchesIntent { matches, replacement }) + .map(|_| ()) + }, + } + } + + /// Full-mesh sync, iterated to quiescence: every peer pulls every other + /// peer's new history until no version vector moves. Iteration (rather than + /// a fixed two passes) matters because importing can COMMIT: an undo + /// restores registry records whose cursor anchors died with the deleted + /// text, so each importer deterministically re-derives and repairs them + /// (convergent fresh ops) — those repair updates need delivery too, exactly + /// like the field's anti-entropy. A bounded pass count keeps a repair loop + /// from masquerading as convergence. + fn sync_all(peers: &mut [Peer]) { + for _pass in 0..8 { + let before: Vec<_> = peers.iter().map(Peer::state_vv).collect(); + for from in 0..peers.len() { + let from_vv_now = peers[from].state_vv(); + for to in 0..peers.len() { + if from == to { + continue; + } + let since = peers[to].synced_vv[from].clone(); + let bytes = peers[from].export_updates_since(&since); + peers[to].import(&bytes); + peers[to].synced_vv[from] = from_vv_now.clone(); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() + && let Err(reason) = projections_agree(&peers[to].projection(), &peers[to].fresh_canonical()) + { + panic!("sync pass {_pass}: peer {to} deviates from own canonical after importing from {from}: {reason}"); + } + } + } + if peers.iter().map(Peer::state_vv).collect::>() == before { + return; + } + } + panic!("sync_all did not quiesce within 8 passes (non-converging repair loop?)"); + } + + fn projections_agree(left: &DocumentProjection, right: &DocumentProjection) -> Result<(), String> { + if left.paragraphs.len() != right.paragraphs.len() { + return Err(format!("paragraph count {} != {}", left.paragraphs.len(), right.paragraphs.len())); + } + for ix in 0..left.paragraphs.len() { + if paragraph_text(left, ix) != paragraph_text(right, ix) { + return Err(format!( + "paragraph[{ix}] text {:?} != {:?}", + paragraph_text(left, ix), + paragraph_text(right, ix) + )); + } + if left.paragraphs[ix].style != right.paragraphs[ix].style { + return Err(format!("paragraph[{ix}] style differs")); + } + if left.paragraphs[ix].runs != right.paragraphs[ix].runs { + return Err(format!( + "paragraph[{ix}] runs differ: {:?} != {:?}", + left.paragraphs[ix].runs, right.paragraphs[ix].runs + )); + } + } + if left.ids.paragraph_ids != right.ids.paragraph_ids { + return Err("paragraph ids differ".to_string()); + } + if left.ids.block_ids != right.ids.block_ids { + return Err("block ids differ".to_string()); + } + // Structural block agreement (kinds; the object/table suites in + // object_table_convergence.rs additionally compare table topology). + let kind = |block: &Block| match block { + Block::Paragraph(_) => "paragraph", + Block::Image(_) => "image", + Block::Equation(_) => "equation", + Block::Table(_) => "table", + }; + if left.blocks.len() != right.blocks.len() { + return Err(format!("block count {} != {}", left.blocks.len(), right.blocks.len())); + } + for ix in 0..left.blocks.len() { + if kind(&left.blocks[ix]) != kind(&right.blocks[ix]) { + return Err(format!("block[{ix}] kind {} != {}", kind(&left.blocks[ix]), kind(&right.blocks[ix]))); + } + } + Ok(()) + } + + fn run_fuzz(seed: u64, peers_n: usize, rounds: usize, ops_per_round: usize) { + let mut rng = Rng::new(seed); + let mut peers: Vec = (0..peers_n).map(|_| Peer::new("fuzz", peers_n)).collect(); + // Initial full exchange so the mergeable seeds converge before editing. + sync_all(&mut peers); + + let per_op_check = std::env::var("FUZZ_PER_OP_CHECK").is_ok(); + for round in 0..rounds { + for op in 0..ops_per_round { + let peer_ix = rng.below(peers_n); + let step = round * ops_per_round + op; + match random_intent(&mut rng, &peers[peer_ix], step) { + Ok(()) => {}, + Err(WriteRejected::EmptyIntent | WriteRejected::StructureViolation(_) | WriteRejected::UnresolvedParagraph(_)) => {}, + Err(other) => panic!("seed {seed} round {round} op {op}: unexpected rejection {other}"), + } + if per_op_check && let Err(reason) = projections_agree(&peers[peer_ix].projection(), &peers[peer_ix].fresh_canonical()) { + panic!("seed {seed} round {round} op {op} (step {step}, peer {peer_ix}): projection deviates from own canonical: {reason}"); + } + } + sync_all(&mut peers); + + // Self-consistency first (the one derivation law): every peer's + // maintained projection must equal a fresh full rematerialization of + // its own canonical state — classifies a failure as projection-derivation + // (self-inconsistent) vs CRDT-level (self-consistent but cross-diverged). + for (ix, peer) in peers.iter().enumerate() { + if let Err(reason) = projections_agree(&peer.projection(), &peer.fresh_canonical()) { + panic!("seed {seed} round {round}: peer {ix} projection deviates from own canonical: {reason}"); + } + } + // Convergence bar (spec §13.7): canonical Loro text AND projection agree. + let reference_text = peers[0].body_text(); + let reference_projection = peers[0].projection(); + for (ix, peer) in peers.iter().enumerate().skip(1) { + // Equal op logs are a precondition for the state comparison: a version + // vector mismatch here is a harness delivery hole, not divergence. + assert_eq!( + peer.state_vv(), + peers[0].state_vv(), + "seed {seed} round {round}: peer {ix} version vector differs after quiescent sync" + ); + assert_eq!( + peer.body_text(), + reference_text, + "seed {seed} round {round}: peer {ix} body text diverged" + ); + if let Err(reason) = projections_agree(&peer.projection(), &reference_projection) { + panic!("seed {seed} round {round}: peer {ix} projection diverged: {reason}"); + } + } + } + } + + #[test] + fn two_peer_intent_fuzz_converges() { + for seed in [1, 7, 42, 505, 1337, 8738] { + run_fuzz(seed, 2, 6, 12); + } + } + + #[test] + fn three_peer_intent_fuzz_converges() { + for seed in [3, 99, 4242, 20260707] { + run_fuzz(seed, 3, 5, 10); + } + } + + /// §act-eleven C10: the long-horizon randomized soak — a wide seed sweep at + /// larger op counts, ignored by default (fixed CI seeds stay deterministic). + /// Run via `heaven.sh soak`; tune with `FUZZ_SOAK_SEEDS` / `FUZZ_SOAK_ROUNDS` / + /// `FUZZ_SOAK_OPS`, and set `FUZZ_PER_OP_CHECK=1` for per-op tracing. + #[test] + #[ignore = "long-horizon soak — run via `heaven.sh soak`"] + fn intent_fuzz_soak() { + let seeds: u64 = std::env::var("FUZZ_SOAK_SEEDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(500); + let rounds: usize = std::env::var("FUZZ_SOAK_ROUNDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8); + let ops: usize = std::env::var("FUZZ_SOAK_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(16); + for seed in 1..=seeds { + let peers = 2 + (seed % 3) as usize; // 2–4 peers + run_fuzz(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15), peers, rounds, ops); + // The checkpoint op (variant 15) writes one `.db8` per peer-Arc per seed + // into the tmpfs checkpoint dir and nothing else deletes them — a long + // soak exhausts the /tmp user QUOTA and the checkpoint `.expect()` then + // fails looking exactly like a fuzz failure (it did, twice, 2026-07-10). + // Reclaim between seeds; recreation is one mkdir. + let _ = std::fs::remove_dir_all(std::env::temp_dir().join(format!("flowstate-fuzz-ckpt-{}", std::process::id()))); + if seed % 25 == 0 { + eprintln!("[soak] intent_fuzz {seed}/{seeds}"); + } + } + } + + /// Spec §13.13 — zero-patch frontier churn is architecturally inert. A remote + /// update that advances the Loro frontier without changing the projection must + /// not disturb local typing in any way: no rejection, no recovery, no replay. + #[test] + fn zero_patch_frontier_churn_is_inert() { + let mut peers: Vec = (0..2).map(|_| Peer::new("churn", 2)).collect(); + sync_all(&mut peers); + let paragraph = peers[0].projection().ids.paragraph_ids[0]; + + for i in 0..10 { + // Peer 1 produces a frontier-advancing, projection-neutral update + // (metadata touch: register_replica writes bookkeeping maps only). + let guard = peers[1].gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::register_replica(guard.doc(), Some(0x77)).expect("metadata commit"); + drop(guard); + let bytes = peers[1].export_updates_since(&peers[0].synced_vv[1].clone()); + peers[0].synced_vv[1] = peers[1].state_vv(); + peers[0].import(&bytes); + + // Immediately type locally: must commit cleanly against current state. + let outcome = peers[0] + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("{i}"), + style_override: None, + }) + .expect("local insert must be untouched by frontier-only churn"); + let commit = outcome.commit(); + assert!(!commit.counters.full_rebuild, "frontier churn must not force local rebuilds"); + } + // Both replicas seeded their own sentinel before the exchange, so the + // absolute text shape carries two boundary newlines; the property under + // test is that every digit landed, in order, at the anchored paragraph. + assert!( + peers[0].body_text().contains("0123456789"), + "digits must land contiguously in order: {:?}", + peers[0].body_text() + ); + } +} diff --git a/crates/flowstate-collab/tests/mass_op_chunking.rs b/crates/flowstate-collab/tests/mass_op_chunking.rs new file mode 100644 index 00000000..880745d2 --- /dev/null +++ b/crates/flowstate-collab/tests/mass_op_chunking.rs @@ -0,0 +1,129 @@ +//! §A14.1.1 gates: a MASS local commit publishes as bounded slices; a +//! receiver imports them as separate small gate holds (the priority lane +//! admits keystrokes between them) and converges to the sender byte-for-byte. + +#[cfg(test)] +mod tests { + mod tests { + use flowstate_collab::crdt_runtime::{CrdtRuntime, RuntimeEvent, SemanticCommand}; + use flowstate_document::{RunStyles, loro_schema::body_text}; + use loro::LoroDoc; + + fn mass_commit_events() -> (CrdtRuntime, Vec, Vec>) { + let mut runtime = CrdtRuntime::new_empty("Mass chunking").expect("runtime"); + // Receivers in the field share history up to the sender's last publish; + // capture that baseline (genesis/schema ops) before the mass commit. + let baseline = runtime + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("baseline export"); + // One mass commit: a single ~40k-char insert (40k op atoms — well past + // the 2048-atom slice budget). + let big = "mass chunked payload ".repeat(2_000); + let events = runtime + .command(SemanticCommand::InsertText { + unicode_index: 1, + text: big, + styles: RunStyles::default(), + }) + .expect("mass insert"); + let updates: Vec> = events + .iter() + .filter_map(|event| match event { + RuntimeEvent::LocalUpdate { bytes, .. } => Some(bytes.clone()), + _ => None, + }) + .collect(); + (runtime, baseline, updates) + } + + #[test] + fn mass_local_commit_publishes_bounded_slices() { + let (runtime, baseline, updates) = mass_commit_events(); + assert!( + updates.len() >= 10, + "a 40k-atom commit must slice (got {} update event(s))", + updates.len() + ); + // Every slice imports standalone IN ORDER on a peer sharing the + // baseline, and the peer converges to the sender's exact text. + let peer = LoroDoc::new(); + peer.import(&baseline).expect("baseline import"); + for (ix, update) in updates.iter().enumerate() { + let status = peer + .import(update) + .unwrap_or_else(|error| panic!("slice {ix} import failed: {error}")); + assert!(status.pending.is_none(), "slice {ix} must be causally self-contained: {status:?}"); + } + assert_eq!(body_text(&peer).to_string(), body_text(runtime.doc()).to_string()); + } + + #[test] + fn chunked_slices_import_through_the_runtime_receiver() { + let (sender, baseline, updates) = mass_commit_events(); + let mut receiver = CrdtRuntime::from_doc( + { + let doc = LoroDoc::new(); + flowstate_document::loro_schema::configure_text_styles(&doc); + doc.import(&baseline).expect("baseline import"); + doc + }, + None, + None, + ) + .expect("receiver runtime"); + let mut import_holds_over_budget = 0; + for update in &updates { + let t = std::time::Instant::now(); + receiver + .import_remote_updates(&[update.as_slice()]) + .expect("receiver import"); + // Counter-gate (generous for shared-machine noise): one slice's whole + // import+derive must stay far below the old ~240ms mass hold. + if t.elapsed() > std::time::Duration::from_millis(50) { + import_holds_over_budget += 1; + } + } + assert_eq!(import_holds_over_budget, 0, "every slice import must be a small hold"); + assert_eq!(body_text(receiver.doc()).to_string(), body_text(sender.doc()).to_string()); + } + + #[test] + fn each_slice_emits_a_projection_update() { + // §A14.2: the receiver paints PROGRESSIVELY — every chunked slice import + // yields its own projection event, so the user sees the mass op land in + // increments instead of one 240ms freeze. + let (sender, baseline, updates) = mass_commit_events(); + let mut receiver = CrdtRuntime::from_doc( + { + let doc = LoroDoc::new(); + flowstate_document::loro_schema::configure_text_styles(&doc); + doc.import(&baseline).expect("baseline import"); + doc + }, + None, + None, + ) + .expect("receiver runtime"); + let mut slices_with_projection = 0; + for update in &updates { + let batches = receiver + .import_remote_updates(&[update.as_slice()]) + .expect("import"); + let has_projection = batches + .iter() + .flatten() + .any(|event| matches!(event, RuntimeEvent::ProjectionUpdated { .. } | RuntimeEvent::ProjectionPatched { .. })); + if has_projection { + slices_with_projection += 1; + } + } + assert!( + slices_with_projection >= updates.len() - 1, + "nearly every slice must paint ({slices_with_projection}/{} slices had a projection event)", + updates.len() + ); + assert_eq!(body_text(receiver.doc()).to_string(), body_text(sender.doc()).to_string()); + } + } +} diff --git a/crates/flowstate-collab/tests/network_pathology.rs b/crates/flowstate-collab/tests/network_pathology.rs new file mode 100644 index 00000000..bb800941 --- /dev/null +++ b/crates/flowstate-collab/tests/network_pathology.rs @@ -0,0 +1,268 @@ +//! Network-pathology convergence fuzz (Loro-first spec §13/§22). +//! +//! The intent/object fuzzes exchange updates losslessly and in order; real +//! transports do not. This harness delivers update chunks REORDERED, +//! DUPLICATED, and DELAYED across rounds while peers keep editing (inserts, +//! splits, undo/redo). Imports with missing causal dependencies must park as +//! PENDING inside Loro and recover when the gap arrives — never wedge, never +//! project half a batch. After the chaos, a clean anti-entropy pass must +//! converge every peer on identical canonical state AND a projection equal to +//! a fresh rematerialization of its own doc (`swarm_loopback` covers the real +//! iroh transport out-of-gate; this covers the semantics deterministically). + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{ + GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, SplitParagraphIntent, TextAnchor, WriteGate, WriteRejected, + }; + use flowstate_document::{DocumentProjection, ParagraphStyle, document_from_loro, paragraph_text}; + + struct Rng(u64); + + impl Rng { + fn new(seed: u64) -> Self { + Self(seed.max(1)) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, bound: usize) -> usize { + if bound == 0 { 0 } else { (self.next() % bound as u64) as usize } + } + } + + struct Peer { + handle: LocalDocHandle, + gate: Arc>, + /// Version vector snapshot at the last chunk export. + exported_vv: loro::VersionVector, + } + + impl Peer { + fn new(title: &str) -> Self { + let core = CrdtRuntime::new_empty(title).expect("runtime"); + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let exported_vv = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard.doc().state_vv() + }; + // The pre-export baseline must NOT include the startup commits, or the + // first chunk would omit each peer's seed/replica metadata and every + // import of it would pend forever. + let _ = exported_vv; + Self { + handle, + gate, + exported_vv: loro::VersionVector::default(), + } + } + + fn projection(&self) -> DocumentProjection { + self.handle.projection().expect("projection") + } + + fn fresh_canonical(&self) -> DocumentProjection { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + document_from_loro(guard.doc()).expect("fresh rematerialization") + } + + fn body_text(&self) -> String { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + } + + fn state_vv(&self) -> loro::VersionVector { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard.doc().state_vv() + } + + /// Export everything committed since the previous chunk export — one + /// "packet" as a gossip transport would carry it. + fn export_chunk(&mut self) -> Option> { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + let now = guard.doc().state_vv(); + if now == self.exported_vv { + return None; + } + let bytes = guard + .doc() + .export(loro::ExportMode::updates(&self.exported_vv)) + .expect("export"); + drop(guard); + self.exported_vv = now; + (!bytes.is_empty()).then_some(bytes) + } + + fn import(&self, bytes: &[u8]) { + let mut guard = self.gate.lock(GateHolder::ImportChunk).expect("gate"); + guard + .import_remote_update(bytes) + .expect("import never errors, pending is a status"); + } + } + + fn random_edit(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let projection = peer.projection(); + if projection.paragraphs.is_empty() { + return Ok(()); + } + let paragraph_ix = rng.below(projection.paragraphs.len()); + let paragraph = projection.ids.paragraph_ids[paragraph_ix]; + match rng.below(6) { + 0..=2 => peer + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!("n{step}"), + style_override: None, + }) + .map(|_| ()), + 3 => peer + .handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .map(|_| ()), + 4 => peer.handle.apply_undo().map(|_| ()), + _ => peer.handle.apply_redo().map(|_| ()), + } + } + + /// One in-flight packet: producer peer + payload + how many rounds it still + /// sits in the network before delivery is even attempted. + struct Packet { + from: usize, + bytes: Vec, + delay_rounds: usize, + } + + fn run_chaos(seed: u64, peers_n: usize, rounds: usize, ops_per_round: usize) { + let mut rng = Rng::new(seed); + let mut peers: Vec = (0..peers_n).map(|_| Peer::new("chaos")).collect(); + let mut network: Vec = Vec::new(); + + for round in 0..rounds { + // ---- Edits -------------------------------------------------------- + for op in 0..ops_per_round { + let peer_ix = rng.below(peers_n); + match random_edit(&mut rng, &peers[peer_ix], round * ops_per_round + op) { + Ok(()) | Err(WriteRejected::EmptyIntent | WriteRejected::StructureViolation(_) | WriteRejected::UnresolvedParagraph(_)) => {}, + Err(other) => panic!("seed {seed} round {round} op {op}: unexpected rejection {other}"), + } + } + + // ---- Publish: each peer's new history becomes one packet ---------- + for (from, peer) in peers.iter_mut().enumerate() { + if let Some(bytes) = peer.export_chunk() { + network.push(Packet { + from, + bytes, + delay_rounds: rng.below(3), + }); + } + } + + // ---- Chaotic delivery: shuffle, sometimes duplicate, delay -------- + // Deterministic Fisher-Yates on the in-flight set. + for ix in (1..network.len()).rev() { + network.swap(ix, rng.below(ix + 1)); + } + let mut undelivered = Vec::new(); + for mut packet in std::mem::take(&mut network) { + if packet.delay_rounds > 0 { + packet.delay_rounds -= 1; + undelivered.push(packet); + continue; + } + for (to, peer) in peers.iter().enumerate() { + if to != packet.from { + peer.import(&packet.bytes); + if rng.below(5) == 0 { + // Duplicate delivery: idempotent import is part of the contract. + peer.import(&packet.bytes); + } + } + } + } + network = undelivered; + } + + // ---- Drain the network, then clean anti-entropy to quiescence -------- + for packet in network { + for (to, peer) in peers.iter().enumerate() { + if to != packet.from { + peer.import(&packet.bytes); + } + } + } + for _pass in 0..8 { + let before: Vec<_> = peers.iter().map(Peer::state_vv).collect(); + for from in 0..peers_n { + let bytes = { + let guard = peers[from] + .gate + .lock(GateHolder::ExportUpdates) + .expect("gate"); + guard + .doc() + .export(loro::ExportMode::all_updates()) + .expect("export") + }; + for (to, peer) in peers.iter().enumerate() { + if to != from { + peer.import(&bytes); + } + } + } + if peers.iter().map(Peer::state_vv).collect::>() == before { + break; + } + } + + // ---- Convergence + self-consistency bar ------------------------------ + let reference_text = peers[0].body_text(); + for (ix, peer) in peers.iter().enumerate() { + let projection = peer.projection(); + let canonical = peer.fresh_canonical(); + assert_eq!( + projection.paragraphs.len(), + canonical.paragraphs.len(), + "seed {seed}: peer {ix} projection/canonical paragraph count diverged" + ); + for paragraph_ix in 0..projection.paragraphs.len() { + assert_eq!( + paragraph_text(&projection, paragraph_ix), + paragraph_text(&canonical, paragraph_ix), + "seed {seed}: peer {ix} paragraph {paragraph_ix} deviates from own canonical" + ); + } + assert_eq!(peer.state_vv(), peers[0].state_vv(), "seed {seed}: peer {ix} version vector diverged"); + assert_eq!(peer.body_text(), reference_text, "seed {seed}: peer {ix} body text diverged"); + } + } + + #[test] + fn reordered_duplicated_delayed_delivery_converges() { + for seed in [1, 7, 42, 99, 20260707] { + run_chaos(seed, 3, 6, 8); + } + } + + #[test] + fn two_peer_chaos_with_heavy_delay_converges() { + for seed in [3, 1337] { + run_chaos(seed, 2, 10, 6); + } + } +} diff --git a/crates/flowstate-collab/tests/object_table_convergence.rs b/crates/flowstate-collab/tests/object_table_convergence.rs new file mode 100644 index 00000000..ef8c2951 --- /dev/null +++ b/crates/flowstate-collab/tests/object_table_convergence.rs @@ -0,0 +1,1312 @@ +//! N-peer structural convergence over the full object + table intent surface +//! (Loro-first spec §13.7), rebuilt from the raw-command-era multi-peer and +//! table convergence suites. +//! +//! Coverage carried over from the deleted `multi_peer_convergence_tests.rs` / +//! `table_convergence_tests.rs` op inventory, now driven exclusively through +//! `LocalDocHandle` intents: text insert (incl. intra-paragraph U+2028 soft +//! breaks) / delete / split / join / run marks / paragraph style, object +//! insert / delete / move / replace, image alt-text / caption / layout, +//! equation source edits, rich-fragment paste, and all 9 table ops +//! (row insert/delete/move, column insert/delete/move, cell replace, cell +//! span, column width) with durable §P2b identities. +//! +//! Convergence is the PROPERTY: intents race across peers (rejections from +//! stale identities are legal outcomes, I-15); after every full-mesh sync round +//! all peers must agree on body text, projection paragraphs/runs/styles, +//! block-id sequences, block kinds, and table topology — and at the end each +//! peer's incrementally maintained projection must equal a fresh +//! `document_from_loro` rebuild of its own doc (materializer equivalence). + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{ + DeleteBlocksIntent, DeleteRangeIntent, FragmentBlock, GateHolder, InsertObjectIntent, InsertRichFragmentIntent, InsertTextIntent, + JoinParagraphsIntent, LocalDocHandle, LocalWriteConfig, MoveBlockIntent, ReplaceEquationSourceRangeIntent, ReplaceImageAltTextIntent, + ReplaceImageCaptionIntent, ReplaceObjectIntent, SetImageLayoutIntent, SetMarksIntent, SetParagraphStyleIntent, SplitParagraphIntent, + TableIntent, TextAnchor, WriteGate, WriteRejected, + }; + use flowstate_document::{ + AssetId, Block, BlockId, CellId, ColumnId, DocumentProjection, InputBlock, InputBlockAlignment, InputEquationBlock, InputEquationDisplay, + InputEquationSyntax, InputImageBlock, InputImageSizing, InputParagraph, InputRun, InputTableBlock, InputTableCell, InputTableCellBlock, + InputTableColumn, InputTableColumnWidth, InputTableRow, InputTableStyle, ParagraphStyle, RowId, RunSemanticStyle, RunStyles, TableBlock, + TableCell, TableCellBlock, document_from_loro, paragraph_text, paragraph_text_len, + }; + use uuid::Uuid; + + // --------------------------------------------------------------------------- + // Harness (the intent_fuzz.rs Peer/sync_all pattern) + // --------------------------------------------------------------------------- + + /// Deterministic xorshift PRNG — reproducible fuzz, no wall-clock dependence. + /// (Editor-minted durable ids inside payloads are uuid-minted, matching the + /// production write path; determinism here is about the op SEQUENCE.) + struct Rng(u64); + + impl Rng { + fn new(seed: u64) -> Self { + Self(seed.max(1)) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + fn below(&mut self, bound: usize) -> usize { + if bound == 0 { 0 } else { (self.next() % bound as u64) as usize } + } + } + + struct Peer { + handle: LocalDocHandle, + gate: Arc>, + synced_vv: Vec, + } + + impl Peer { + fn new(title: &str, peer_count: usize) -> Self { + let core = CrdtRuntime::new_empty(title).expect("runtime"); + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + Self { + handle, + gate, + synced_vv: vec![loro::VersionVector::default(); peer_count], + } + } + + fn projection(&self) -> DocumentProjection { + self.handle.projection().expect("projection") + } + + fn export_updates_since(&self, vv: &loro::VersionVector) -> Vec { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard + .doc() + .export(loro::ExportMode::updates(vv)) + .expect("export") + } + + fn state_vv(&self) -> loro::VersionVector { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + guard.doc().state_vv() + } + + fn import(&self, bytes: &[u8]) { + if bytes.is_empty() { + return; + } + let mut guard = self.gate.lock(GateHolder::ImportChunk).expect("gate"); + guard.import_remote_update(bytes).expect("import"); + } + + fn body_text(&self) -> String { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + } + + /// Fresh full rebuild of this peer's canonical Loro state (the materializer + /// equivalence reference). + fn fresh_projection(&self) -> DocumentProjection { + let guard = self.gate.lock(GateHolder::ExportUpdates).expect("gate"); + document_from_loro(guard.doc()).expect("document_from_loro") + } + } + + /// Full-mesh sync, iterated to quiescence (mirror of `intent_fuzz)`: + /// importing an undo can COMMIT convergent repairs (records whose cursor + /// anchors died with the deleted content), and those repair updates need + /// delivery too. A bounded pass count keeps a non-converging repair loop + /// from masquerading as convergence. + fn sync_all(peers: &mut [Peer]) { + for _pass in 0..8 { + let before: Vec<_> = peers.iter().map(Peer::state_vv).collect(); + for from in 0..peers.len() { + let from_vv_now = peers[from].state_vv(); + for to in 0..peers.len() { + if from == to { + continue; + } + let since = peers[to].synced_vv[from].clone(); + let bytes = peers[from].export_updates_since(&since); + peers[to].import(&bytes); + peers[to].synced_vv[from] = from_vv_now.clone(); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() + && let Err(reason) = projections_agree(&peers[to].projection(), &peers[to].fresh_projection()) + { + panic!("sync pass {_pass}: peer {to} deviates from own canonical after importing from {from}: {reason}"); + } + } + } + if peers.iter().map(Peer::state_vv).collect::>() == before { + return; + } + } + panic!("sync_all did not quiesce within 8 passes (non-converging repair loop?)"); + } + + // --------------------------------------------------------------------------- + // Structural projection comparison (extends intent_fuzz's projections_agree + // with block kinds + table row/column id sequences + cell texts) + // --------------------------------------------------------------------------- + + fn block_kind(block: &Block) -> &'static str { + match block { + Block::Paragraph(_) => "paragraph", + Block::Image(_) => "image", + Block::Equation(_) => "equation", + Block::Table(_) => "table", + } + } + + fn cell_text(cell: &TableCell) -> String { + cell + .blocks + .iter() + .filter_map(|block| match block { + TableCellBlock::Paragraph(paragraph) => Some(paragraph.text.as_str()), + TableCellBlock::Table(_) => None, + }) + .collect() + } + + fn tables_agree(left: &TableBlock, right: &TableBlock) -> Result<(), String> { + let left_rows: Vec = left.rows.iter().map(|row| row.id).collect(); + let right_rows: Vec = right.rows.iter().map(|row| row.id).collect(); + if left_rows != right_rows { + return Err(format!("table row id sequences differ: {left_rows:?} != {right_rows:?}")); + } + let left_columns: Vec = left.columns.iter().map(|column| column.id).collect(); + let right_columns: Vec = right.columns.iter().map(|column| column.id).collect(); + if left_columns != right_columns { + return Err(format!("table column id sequences differ: {left_columns:?} != {right_columns:?}")); + } + for (row_ix, (left_row, right_row)) in left.rows.iter().zip(&right.rows).enumerate() { + if left_row.cells.len() != right_row.cells.len() { + return Err(format!( + "table row {row_ix} cell count {} != {}", + left_row.cells.len(), + right_row.cells.len() + )); + } + for (col_ix, (left_cell, right_cell)) in left_row.cells.iter().zip(&right_row.cells).enumerate() { + if (left_cell.row_id, left_cell.column_id) != (right_cell.row_id, right_cell.column_id) { + return Err(format!("table cell ({row_ix},{col_ix}) coordinates differ")); + } + if (left_cell.row_span, left_cell.col_span) != (right_cell.row_span, right_cell.col_span) { + return Err(format!("table cell ({row_ix},{col_ix}) spans differ")); + } + if cell_text(left_cell) != cell_text(right_cell) { + return Err(format!( + "table cell ({row_ix},{col_ix}) text {:?} != {:?}", + cell_text(left_cell), + cell_text(right_cell) + )); + } + } + } + Ok(()) + } + + fn blocks_agree(left: &DocumentProjection, right: &DocumentProjection) -> Result<(), String> { + if left.blocks.len() != right.blocks.len() { + return Err(format!("block count {} != {}", left.blocks.len(), right.blocks.len())); + } + for ix in 0..left.blocks.len() { + let (left_block, right_block) = (&left.blocks[ix], &right.blocks[ix]); + if block_kind(left_block) != block_kind(right_block) { + return Err(format!("block[{ix}] kind {} != {}", block_kind(left_block), block_kind(right_block))); + } + match (left_block, right_block) { + (Block::Image(left_image), Block::Image(right_image)) => { + if left_image.alt_text != right_image.alt_text + || left_image.sizing != right_image.sizing + || left_image.alignment != right_image.alignment + || left_image.caption.is_some() != right_image.caption.is_some() + { + return Err(format!("block[{ix}] image metadata differs")); + } + }, + (Block::Equation(left_equation), Block::Equation(right_equation)) => { + if left_equation.source != right_equation.source { + return Err(format!( + "block[{ix}] equation source {:?} != {:?}", + left_equation.source, right_equation.source + )); + } + }, + (Block::Table(left_table), Block::Table(right_table)) => { + tables_agree(left_table, right_table).map_err(|reason| format!("block[{ix}]: {reason}"))?; + }, + _ => {}, + } + } + Ok(()) + } + + fn projections_agree(left: &DocumentProjection, right: &DocumentProjection) -> Result<(), String> { + if left.paragraphs.len() != right.paragraphs.len() { + return Err(format!("paragraph count {} != {}", left.paragraphs.len(), right.paragraphs.len())); + } + for ix in 0..left.paragraphs.len() { + if paragraph_text(left, ix) != paragraph_text(right, ix) { + return Err(format!( + "paragraph[{ix}] text {:?} != {:?}", + paragraph_text(left, ix), + paragraph_text(right, ix) + )); + } + if left.paragraphs[ix].style != right.paragraphs[ix].style { + return Err(format!( + "paragraph[{ix}] style differs: left {:?} right {:?} (kinds: {:?})", + left.paragraphs[ix].style, + right.paragraphs[ix].style, + left.blocks.iter().map(block_kind).collect::>() + )); + } + if left.paragraphs[ix].runs != right.paragraphs[ix].runs { + return Err(format!( + "paragraph[{ix}] runs differ: {:?} != {:?}", + left.paragraphs[ix].runs, right.paragraphs[ix].runs + )); + } + } + if left.ids.paragraph_ids != right.ids.paragraph_ids { + return Err(format!( + "paragraph ids differ:\n left {:?}\n right {:?}", + left.ids.paragraph_ids, right.ids.paragraph_ids + )); + } + if left.ids.block_ids != right.ids.block_ids { + return Err(format!( + "block ids differ:\n left {:?}\n right {:?}", + left.ids.block_ids, right.ids.block_ids + )); + } + blocks_agree(left, right) + } + + fn assert_converged(peers: &[Peer], context: &str) { + // Self-consistency first (classifier, mirror of intent_fuzz): a peer whose + // maintained projection deviates from a fresh rematerialization of its OWN + // canonical state is a derivation bug; self-consistent peers that disagree + // with each other are CRDT-level divergence. + for (ix, peer) in peers.iter().enumerate() { + if let Err(reason) = projections_agree(&peer.projection(), &peer.fresh_projection()) { + panic!("{context}: peer {ix} projection deviates from own canonical: {reason}"); + } + } + let reference_text = peers[0].body_text(); + let reference = peers[0].projection(); + for (ix, peer) in peers.iter().enumerate().skip(1) { + assert_eq!(peer.body_text(), reference_text, "{context}: peer {ix} body text diverged"); + if let Err(reason) = projections_agree(&peer.projection(), &reference) { + panic!("{context}: peer {ix} projection diverged: {reason}"); + } + } + } + + /// Materializer equivalence: each peer's incrementally maintained projection + /// equals a fresh full `document_from_loro` rebuild of its own doc. + fn assert_materializer_equivalence(peers: &[Peer], context: &str) { + for (ix, peer) in peers.iter().enumerate() { + if let Err(reason) = projections_agree(&peer.projection(), &peer.fresh_projection()) { + panic!("{context}: peer {ix} incremental projection != fresh document_from_loro rebuild: {reason}"); + } + } + } + + // --------------------------------------------------------------------------- + // Payload builders (the old suites' fixture builders, editor-minted ids via + // uuid exactly like the production write path) + // --------------------------------------------------------------------------- + + fn input_paragraph(text: &str) -> InputParagraph { + InputParagraph { + style: ParagraphStyle::Normal, + runs: if text.is_empty() { + Vec::new() + } else { + vec![InputRun { + text: text.to_string(), + styles: RunStyles::default(), + }] + }, + } + } + + fn image_input(rng: &mut Rng) -> InputBlock { + InputBlock::Image(InputImageBlock { + asset_id: AssetId(1), + alt_text: format!("img{}", rng.below(100)), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Left, + external_url: None, + }) + } + + fn equation_input(rng: &mut Rng) -> InputBlock { + InputBlock::Equation(InputEquationBlock { + source: format!("x^{}", rng.below(10)), + syntax: InputEquationSyntax::Latex, + display: InputEquationDisplay::Display, + }) + } + + fn object_input(rng: &mut Rng) -> InputBlock { + if rng.below(2) == 0 { image_input(rng) } else { equation_input(rng) } + } + + fn table_cell_input(row_id: RowId, column_id: ColumnId, text: &str) -> InputTableCell { + InputTableCell { + // §P2b law: a cell's identity IS its coordinate (deterministic mix) — + // never a random id, or writers keyed by coordinate and readers keyed by + // record diverge into duplicate cells. + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks: vec![InputTableCellBlock::Paragraph(input_paragraph(text))], + row_span: 1, + col_span: 1, + } + } + + fn fresh_row_input(columns: &[ColumnId], text: &str) -> InputTableRow { + let row_id = RowId(Uuid::new_v4().as_u128()); + InputTableRow { + id: row_id, + cells: columns + .iter() + .map(|column| table_cell_input(row_id, *column, text)) + .collect(), + } + } + + /// A `rows`x`columns` table input with uuid-minted durable ids and per-cell + /// coordinate text (`r{r}c{c}`), the old `table_fixture` in miniature. + fn table_input(rows: usize, columns: usize) -> InputBlock { + let column_ids: Vec = (0..columns) + .map(|_| ColumnId(Uuid::new_v4().as_u128())) + .collect(); + let input_rows: Vec = (0..rows) + .map(|r| { + let row_id = RowId(Uuid::new_v4().as_u128()); + InputTableRow { + id: row_id, + cells: column_ids + .iter() + .enumerate() + .map(|(c, column)| table_cell_input(row_id, *column, &format!("r{r}c{c}"))) + .collect(), + } + }) + .collect(); + InputBlock::Table(InputTableBlock { + rows: input_rows, + columns: column_ids + .into_iter() + .map(|id| InputTableColumn { + id, + width: InputTableColumnWidth::Auto, + }) + .collect(), + style: InputTableStyle { header_row: true }, + }) + } + + fn first_table(projection: &DocumentProjection) -> Option<(BlockId, TableBlock)> { + projection + .blocks + .iter() + .enumerate() + .find_map(|(ix, block)| match block { + Block::Table(table) => Some((projection.ids.block_ids[ix], table.clone())), + _ => None, + }) + } + + fn object_block_indices(projection: &DocumentProjection) -> Vec { + (0..projection.blocks.len()) + .filter(|ix| !matches!(projection.blocks[*ix], Block::Paragraph(_))) + .collect() + } + + // --------------------------------------------------------------------------- + // Seeding (through the intent API — peer 0 builds the fixture, mesh syncs) + // --------------------------------------------------------------------------- + + /// The old `structural_fixture` in miniature: paragraphs with editable words, + /// objects flanked by empty paragraphs (projection coalescing vs live-body + /// coordinates), and an intra-paragraph U+2028 soft break. Built as ONE rich + /// fragment (which also exercises the compound paste intent). + fn seed_structural(peer: &Peer, rng: &mut Rng) { + let projection = peer.projection(); + let paragraph = projection.ids.paragraph_ids[0]; + peer + .handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, 0), + blocks: vec![ + FragmentBlock::Paragraph(input_paragraph("Introduction with several words to edit.")), + FragmentBlock::Object(image_input(rng)), + FragmentBlock::Paragraph(input_paragraph("")), + FragmentBlock::Paragraph(input_paragraph("Text after the first image.")), + FragmentBlock::Paragraph(input_paragraph("Left of soft break\u{2028}right of soft break.")), + FragmentBlock::Paragraph(input_paragraph("alpha bravo charlie")), + FragmentBlock::Object(equation_input(rng)), + FragmentBlock::Paragraph(input_paragraph("")), + FragmentBlock::Paragraph(input_paragraph("Closing remarks paragraph.")), + ], + }) + .expect("seed structural fixture"); + } + + /// The old `table_fixture`: a paragraph, a 2x3 table, a trailing paragraph. + fn seed_table(peer: &Peer, _rng: &mut Rng) { + let projection = peer.projection(); + let paragraph = projection.ids.paragraph_ids[0]; + peer + .handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, 0), + blocks: vec![ + FragmentBlock::Paragraph(input_paragraph("Above the table.")), + FragmentBlock::Object(table_input(2, 3)), + FragmentBlock::Paragraph(input_paragraph("Below the table.")), + ], + }) + .expect("seed table fixture"); + } + + // --------------------------------------------------------------------------- + // Random intent generators + // --------------------------------------------------------------------------- + + fn random_styles(rng: &mut Rng) -> RunStyles { + let mut styles = RunStyles::default(); + if rng.below(2) == 0 { + styles.semantic = RunSemanticStyle::Custom((rng.below(4) + 1) as u8); + } + if rng.below(3) == 0 { + styles.direct_underline = true; + } + if rng.below(4) == 0 { + styles.strikethrough = true; + } + styles + } + + /// Paragraph/text coordinate stress: inserts (with occasional intra-paragraph + /// U+2028 soft breaks — the coordinate paths that broke on the real doc), + /// deletes, splits, joins, run marks, paragraph styles. Byte hints are raw and + /// clamped to char boundaries by the resolver. + fn random_text_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let projection = peer.projection(); + if projection.paragraphs.is_empty() { + return Ok(()); + } + let paragraph_ix = rng.below(projection.paragraphs.len()); + let paragraph = projection.ids.paragraph_ids[paragraph_ix]; + let text_len = paragraph_text_len(&projection.paragraphs[paragraph_ix]); + let byte = rng.below(text_len + 1); + let text_arm = rng.below(10); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() { + eprintln!("step {step}: text sub-arm {text_arm} paragraph_ix {paragraph_ix} byte {byte}"); + } + match text_arm { + 0..=3 => { + let text = if rng.below(6) == 0 { "\u{2028}".to_string() } else { format!("s{step}") }; + peer + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, byte), + text, + style_override: (rng.below(4) == 0).then(|| random_styles(rng)), + }) + .map(|_| ()) + }, + 4 => { + if text_len == 0 { + return Ok(()); + } + let start = rng.below(text_len); + let end = (start + 1 + rng.below(3)).min(text_len); + peer + .handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(paragraph, start), + end: TextAnchor::new(paragraph, end), + }) + .map(|_| ()) + }, + 5 if projection.paragraphs.len() < 32 => peer + .handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(paragraph, byte), + inherited_style: if rng.below(2) == 0 { + ParagraphStyle::Normal + } else { + ParagraphStyle::Custom(1) + }, + }) + .map(|_| ()), + 6 => { + if paragraph_ix + 1 >= projection.paragraphs.len() { + return Ok(()); + } + let second = projection.ids.paragraph_ids[paragraph_ix + 1]; + peer + .handle + .join_paragraphs(JoinParagraphsIntent { first: paragraph, second }) + .map(|_| ()) + }, + 7..=8 => { + if text_len == 0 { + return Ok(()); + } + let start = rng.below(text_len); + let end = (start + 1 + rng.below(4)).min(text_len); + peer + .handle + .set_marks(SetMarksIntent { + start: TextAnchor::new(paragraph, start), + end: TextAnchor::new(paragraph, end), + styles: random_styles(rng), + }) + .map(|_| ()) + }, + _ => peer + .handle + .set_paragraph_style(SetParagraphStyleIntent { + paragraph, + style: if rng.below(2) == 0 { + ParagraphStyle::Normal + } else { + ParagraphStyle::Custom((rng.below(3) + 1) as u8) + }, + }) + .map(|_| ()), + } + } + + /// Object block-structure ops (the old `object_structural_command` surface): + /// insert/delete/move/replace at identity-anchored positions plus image + /// alt/caption/layout, equation source edits, and rich-fragment paste. + fn random_object_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let projection = peer.projection(); + if projection.paragraphs.is_empty() { + return Ok(()); + } + let paragraph_ix = rng.below(projection.paragraphs.len()); + let paragraph = projection.ids.paragraph_ids[paragraph_ix]; + let text_len = paragraph_text_len(&projection.paragraphs[paragraph_ix]); + let objects = object_block_indices(&projection); + let pick_object = |rng: &mut Rng| objects.get(rng.below(objects.len().max(1))).copied(); + let images: Vec = (0..projection.blocks.len()) + .filter(|ix| matches!(projection.blocks[*ix], Block::Image(_))) + .collect(); + let equations: Vec = (0..projection.blocks.len()) + .filter(|ix| matches!(projection.blocks[*ix], Block::Equation(_))) + .collect(); + let object_arm = rng.below(9); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() { + eprintln!("step {step}: object sub-arm {object_arm} paragraph_ix {paragraph_ix}"); + } + match object_arm { + // InsertObject at an identity-anchored position (byte 0 = before the + // paragraph's block, byte > 0 = after it — both positions fuzzed). + 0..=1 => { + let byte = if rng.below(2) == 0 { 0 } else { text_len }; + let block = object_input(rng); + peer + .handle + .insert_object(InsertObjectIntent { + at: TextAnchor::new(paragraph, byte), + block, + }) + .map(|_| ()) + }, + 2 => { + let Some(object_ix) = pick_object(rng) else { return Ok(()) }; + peer + .handle + .delete_blocks(DeleteBlocksIntent { + blocks: vec![projection.ids.block_ids[object_ix]], + }) + .map(|_| ()) + }, + 3 => { + let Some(object_ix) = pick_object(rng) else { return Ok(()) }; + let before = if rng.below(4) == 0 { + None // move to document end + } else { + Some(projection.ids.block_ids[rng.below(projection.blocks.len())]) + }; + peer + .handle + .move_block(MoveBlockIntent { + block: projection.ids.block_ids[object_ix], + before, + }) + .map(|_| ()) + }, + 4 => { + let Some(object_ix) = pick_object(rng) else { return Ok(()) }; + let after = object_input(rng); + peer + .handle + .replace_object(ReplaceObjectIntent { + block: projection.ids.block_ids[object_ix], + after, + }) + .map(|_| ()) + }, + 5 => { + let Some(&image_ix) = images.get(rng.below(images.len().max(1))) else { + return Ok(()); + }; + let image = projection.ids.block_ids[image_ix]; + match rng.below(3) { + 0 => peer + .handle + .replace_image_alt_text(ReplaceImageAltTextIntent { + image, + text: format!("alt-{step}"), + }) + .map(|_| ()), + 1 => peer + .handle + .replace_image_caption(ReplaceImageCaptionIntent { + image, + caption: (rng.below(2) == 0).then(|| input_paragraph(&format!("cap-{step}"))), + }) + .map(|_| ()), + _ => peer + .handle + .set_image_layout(SetImageLayoutIntent { + image, + sizing: match rng.below(3) { + 0 => InputImageSizing::Intrinsic, + 1 => InputImageSizing::FitWidth, + _ => InputImageSizing::Fixed { + width_px: 100 + rng.below(200) as u32, + height_px: None, + }, + }, + alignment: match rng.below(3) { + 0 => InputBlockAlignment::Left, + 1 => InputBlockAlignment::Center, + _ => InputBlockAlignment::Right, + }, + }) + .map(|_| ()), + } + }, + 6 => { + let Some(&equation_ix) = equations.get(rng.below(equations.len().max(1))) else { + return Ok(()); + }; + peer + .handle + .replace_equation_source_range(ReplaceEquationSourceRangeIntent { + equation: projection.ids.block_ids[equation_ix], + range: 0..0, + text: format!("+{}", rng.below(10)), + }) + .map(|_| ()) + }, + // Rich-fragment paste: paragraph + object + paragraph in one compound + // intent (one gate hold, one commit). + _ => { + let object = object_input(rng); + peer + .handle + .insert_rich_fragment(InsertRichFragmentIntent { + at: TextAnchor::new(paragraph, rng.below(text_len + 1)), + blocks: vec![ + FragmentBlock::Paragraph(input_paragraph(&format!("frag{step}a"))), + FragmentBlock::Object(object), + FragmentBlock::Paragraph(input_paragraph(&format!("frag{step}b"))), + ], + }) + .map(|_| ()) + }, + } + } + + /// All 9 table ops over the first table (durable §P2b identities throughout). + fn random_table_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let projection = peer.projection(); + let Some((table_id, table)) = first_table(&projection) else { + return random_text_intent(rng, peer, step); + }; + if table.rows.is_empty() || table.columns.is_empty() { + return Ok(()); + } + let row = table.rows[rng.below(table.rows.len())].id; + let column = table.columns[rng.below(table.columns.len())].id; + let column_ids: Vec = table.columns.iter().map(|c| c.id).collect(); + let table_arm = rng.below(9); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() { + eprintln!("step {step}: table sub-arm {table_arm} row {row:?} column {column:?}"); + } + let intent = match table_arm { + 0 => TableIntent::InsertRow { + table: table_id, + after_row: if rng.below(3) == 0 { None } else { Some(row) }, + row: fresh_row_input(&column_ids, "new"), + }, + 1 if table.rows.len() > 1 => TableIntent::DeleteRow { table: table_id, row }, + 2 if table.rows.len() > 1 => TableIntent::MoveRow { + table: table_id, + row, + after_row: if rng.below(3) == 0 { + None + } else { + Some(table.rows[rng.below(table.rows.len())].id) + }, + }, + 3 => TableIntent::InsertColumn { + table: table_id, + after_column: if rng.below(3) == 0 { None } else { Some(column) }, + width: InputTableColumnWidth::Auto, + }, + 4 if table.columns.len() > 1 => TableIntent::DeleteColumn { table: table_id, column }, + 5 if table.columns.len() > 1 => TableIntent::MoveColumn { + table: table_id, + column, + after_column: if rng.below(3) == 0 { + None + } else { + Some(table.columns[rng.below(table.columns.len())].id) + }, + }, + 6 => TableIntent::ReplaceCell { + table: table_id, + row, + column, + cell: table_cell_input(row, column, &format!("edit{step}")), + }, + 7 => TableIntent::SetCellSpan { + table: table_id, + row, + column, + row_span: 1 + rng.below(2) as u16, + column_span: 1 + rng.below(2) as u16, + }, + _ => TableIntent::SetColumnWidth { + table: table_id, + column, + width: match rng.below(3) { + 0 => InputTableColumnWidth::Auto, + 1 => InputTableColumnWidth::FixedPx(50 + rng.below(200) as u32), + _ => InputTableColumnWidth::Fraction(1 + rng.below(4) as u32), + }, + }, + }; + peer.handle.table_op(intent).map(|_| ()) + } + + /// ~60% text coordinate stress / ~40% object block-structure ops. + fn random_structural_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let arm = rng.below(10); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() { + eprintln!("step {step}: structural arm {arm}"); + } + match arm { + 0..=4 => random_text_intent(rng, peer, step), + // Undo/redo of OBJECT-structural history under concurrency (spec §10): + // intent_fuzz covers body-text undo; this leg covers undoing object + // inserts/moves/replaces and their transforms against remote edits. + 5 => peer.handle.apply_undo().map(|_| ()), + 6 => peer.handle.apply_redo().map(|_| ()), + _ => random_object_intent(rng, peer, step), + } + } + + /// ~50% text stress / ~35% table ops / ~15% undo-redo (durable-id table + /// history under undo transforms). + fn random_table_mix_intent(rng: &mut Rng, peer: &Peer, step: usize) -> Result<(), WriteRejected> { + let arm = rng.below(100); + if std::env::var("FUZZ_PER_OP_CHECK").is_ok() { + eprintln!("step {step}: table-mix arm {arm}"); + } + match arm { + 0..=49 => random_text_intent(rng, peer, step), + 50..=57 => peer.handle.apply_undo().map(|_| ()), + 58..=64 => peer.handle.apply_redo().map(|_| ()), + _ => random_table_intent(rng, peer, step), + } + } + + // --------------------------------------------------------------------------- + // Fuzz driver + // --------------------------------------------------------------------------- + + /// Rejections from stale identities, empty/degenerate intents, structural + /// refusals, and compensated no-op mutations are legal fuzz outcomes (I-15 / + /// I-10) — only convergence is asserted. Wedge states must fail loudly. + fn tolerate(result: Result<(), WriteRejected>, seed: u64, round: usize, op: usize) { + match result { + Ok(()) => {}, + Err(fatal @ (WriteRejected::GatePoisoned | WriteRejected::CompensationFailed { .. })) => { + panic!("seed {seed} round {round} op {op}: doc wedged: {fatal}") + }, + Err(_) => {}, + } + } + + type IntentGenerator = fn(&mut Rng, &Peer, usize) -> Result<(), WriteRejected>; + type Seeder = fn(&Peer, &mut Rng); + + fn run_fuzz(seed: u64, peers_n: usize, rounds: usize, ops_per_round: usize, seeder: Seeder, generator: IntentGenerator) { + let mut rng = Rng::new(seed); + let mut peers: Vec = (0..peers_n) + .map(|_| Peer::new("structural fuzz", peers_n)) + .collect(); + // Converge the mergeable empty seeds, build the fixture on peer 0 through + // the intent API, and converge again so every peer shares the identities. + sync_all(&mut peers); + seeder(&peers[0], &mut rng); + sync_all(&mut peers); + assert_converged(&peers, &format!("seed {seed} post-fixture")); + + let per_op_check = std::env::var("FUZZ_PER_OP_CHECK").is_ok(); + for round in 0..rounds { + for op in 0..ops_per_round { + let peer_ix = rng.below(peers_n); + let step = round * ops_per_round + op; + tolerate(generator(&mut rng, &peers[peer_ix], step), seed, round, op); + if per_op_check { + let maintained = peers[peer_ix].projection(); + let fresh = peers[peer_ix].fresh_projection(); + eprintln!( + " after step {step}: maintained={:?} fresh={:?}", + maintained + .ids + .paragraph_ids + .iter() + .map(|id| id.0 % 100000) + .collect::>(), + fresh + .ids + .paragraph_ids + .iter() + .map(|id| id.0 % 100000) + .collect::>(), + ); + if let Err(reason) = projections_agree(&maintained, &fresh) { + let fresh_again = peers[peer_ix].fresh_projection(); + let determinism = match projections_agree(&fresh, &fresh_again) { + Ok(()) => "fresh is deterministic".to_string(), + Err(inner) => format!("fresh is NON-DETERMINISTIC: {inner}"), + }; + panic!("seed {seed} round {round} op {op} (peer {peer_ix}): projection deviates from own canonical ({determinism}): {reason}"); + } + } + } + sync_all(&mut peers); + assert_converged(&peers, &format!("seed {seed} round {round}")); + } + assert_materializer_equivalence(&peers, &format!("seed {seed} final")); + } + + // --------------------------------------------------------------------------- + // Fuzz suites + // --------------------------------------------------------------------------- + + /// Object block-structure convergence: insert/delete/move/replace + image and + /// equation metadata + rich fragments over the structural fixture (objects + /// flanked by empties, soft breaks) — the coordinate/coalescing paths blank + /// docs structurally cannot reach. + #[test] + fn npeer_object_structural_fuzz_converges() { + for seed in [0xA1, 0xB2, 0xC3] { + run_fuzz(seed, 2, 6, 10, seed_structural, random_structural_intent); + } + run_fuzz(0x1111, 3, 5, 8, seed_structural, random_structural_intent); + } + + /// Full table-op convergence: the 9 table intents with durable row/column/cell + /// identities plus text stress, seeded from the 2x3 table fixture. + #[test] + fn npeer_table_op_fuzz_converges() { + for seed in [7, 42, 20260707] { + run_fuzz(seed, 2, 6, 10, seed_table, random_table_mix_intent); + } + run_fuzz(0x2222, 3, 5, 8, seed_table, random_table_mix_intent); + } + + /// §act-eleven C10: long-horizon object/table soak — see `intent_fuzz_soak`. + #[test] + #[ignore = "long-horizon soak — run via `heaven.sh soak`"] + fn object_table_fuzz_soak() { + let seeds: u64 = std::env::var("FUZZ_SOAK_SEEDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(500); + let rounds: usize = std::env::var("FUZZ_SOAK_ROUNDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8); + let ops: usize = std::env::var("FUZZ_SOAK_OPS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(12); + for seed in 1..=seeds { + let peers = 2 + (seed % 2) as usize; // 2–3 peers + let mixed = seed.wrapping_mul(0xD1B5_4A32_D192_ED03); + if seed % 2 == 0 { + run_fuzz(mixed, peers, rounds, ops, seed_structural, random_structural_intent); + } else { + run_fuzz(mixed, peers, rounds, ops, seed_table, random_table_mix_intent); + } + if seed % 25 == 0 { + eprintln!("[soak] object_table {seed}/{seeds}"); + } + } + } + + // --------------------------------------------------------------------------- + // Directed concurrent regressions (semantics carried from the old suites) + // --------------------------------------------------------------------------- + + /// Two peers with one seeded table, converged and ready for concurrent edits. + fn table_pair() -> (Vec, BlockId, TableBlock) { + let mut rng = Rng::new(1); + let mut peers: Vec = (0..2).map(|_| Peer::new("table directed", 2)).collect(); + sync_all(&mut peers); + seed_table(&peers[0], &mut rng); + sync_all(&mut peers); + let projection = peers[0].projection(); + let (table_id, table) = first_table(&projection).expect("seeded table"); + (peers, table_id, table) + } + + fn projected_table(peer: &Peer) -> TableBlock { + first_table(&peer.projection()).expect("table present").1 + } + + fn row_texts(table: &TableBlock) -> Vec { + table + .rows + .iter() + .map(|row| { + row + .cells + .iter() + .map(cell_text) + .collect::>() + .join("|") + }) + .collect() + } + + /// Both peers insert a row after the SAME anchor row concurrently: both rows + /// and both originals must survive the merge (durable-id row identity). + #[test] + fn concurrent_row_inserts_after_same_anchor_both_survive() { + let (mut peers, table_id, table) = table_pair(); + let anchor = table.rows[0].id; + let column_ids: Vec = table.columns.iter().map(|c| c.id).collect(); + + peers[0] + .handle + .table_op(TableIntent::InsertRow { + table: table_id, + after_row: Some(anchor), + row: fresh_row_input(&column_ids, "A"), + }) + .expect("peer 0 row insert"); + peers[1] + .handle + .table_op(TableIntent::InsertRow { + table: table_id, + after_row: Some(anchor), + row: fresh_row_input(&column_ids, "B"), + }) + .expect("peer 1 row insert"); + sync_all(&mut peers); + assert_converged(&peers, "concurrent row inserts"); + assert_materializer_equivalence(&peers, "concurrent row inserts"); + + let merged = projected_table(&peers[0]); + assert_eq!(merged.rows.len(), 4, "both inserted rows plus the two originals must survive"); + let texts = row_texts(&merged); + assert!(texts.contains(&"r0c0|r0c1|r0c2".to_string()), "original first row survives: {texts:?}"); + assert!(texts.contains(&"r1c0|r1c1|r1c2".to_string()), "original second row survives: {texts:?}"); + assert!(texts.contains(&"A|A|A".to_string()), "peer 0's row survives: {texts:?}"); + assert!(texts.contains(&"B|B|B".to_string()), "peer 1's row survives: {texts:?}"); + } + + /// Concurrent rewrites of DISTINCT durable cells are independent: both edits + /// survive, untouched cells stay put. + #[test] + fn concurrent_distinct_cell_edits_are_independent() { + let (mut peers, table_id, table) = table_pair(); + let (r1, r2) = (table.rows[0].id, table.rows[1].id); + let (c1, c3) = (table.columns[0].id, table.columns[2].id); + + peers[0] + .handle + .table_op(TableIntent::ReplaceCell { + table: table_id, + row: r1, + column: c1, + cell: table_cell_input(r1, c1, "A-EDIT"), + }) + .expect("peer 0 cell edit"); + peers[1] + .handle + .table_op(TableIntent::ReplaceCell { + table: table_id, + row: r2, + column: c3, + cell: table_cell_input(r2, c3, "B-EDIT"), + }) + .expect("peer 1 cell edit"); + sync_all(&mut peers); + assert_converged(&peers, "concurrent distinct cell edits"); + assert_materializer_equivalence(&peers, "concurrent distinct cell edits"); + + let merged = projected_table(&peers[0]); + assert_eq!(cell_text(&merged.rows[0].cells[0]), "A-EDIT", "peer 0's cell edit survives"); + assert_eq!(cell_text(&merged.rows[1].cells[2]), "B-EDIT", "peer 1's cell edit survives"); + assert_eq!(cell_text(&merged.rows[0].cells[1]), "r0c1", "untouched cell unchanged"); + assert_eq!(cell_text(&merged.rows[1].cells[0]), "r1c0", "untouched cell unchanged"); + } + + /// FS-010: concurrent add-row x add-column leaves the (new row, new column) + /// crossing uncreated by either peer — topology repair must synthesize it + /// deterministically as ONE empty paragraph so every peer reads an identical + /// full grid. + #[test] + fn concurrent_add_row_and_add_column_repairs_cross_cell() { + let (mut peers, table_id, table) = table_pair(); + let base_rows: Vec = table.rows.iter().map(|r| r.id).collect(); + let base_columns: Vec = table.columns.iter().map(|c| c.id).collect(); + + peers[0] + .handle + .table_op(TableIntent::InsertRow { + table: table_id, + after_row: Some(base_rows[1]), + row: fresh_row_input(&base_columns, "r"), + }) + .expect("peer 0 row insert"); + peers[1] + .handle + .table_op(TableIntent::InsertColumn { + table: table_id, + after_column: Some(base_columns[2]), + width: InputTableColumnWidth::Auto, + }) + .expect("peer 1 column insert"); + sync_all(&mut peers); + assert_converged(&peers, "add-row x add-column repair"); + assert_materializer_equivalence(&peers, "add-row x add-column repair"); + + let merged = projected_table(&peers[0]); + assert_eq!(merged.rows.len(), 3, "two originals plus the inserted row"); + assert_eq!(merged.columns.len(), 4, "three originals plus the inserted column"); + for (row_ix, row) in merged.rows.iter().enumerate() { + assert_eq!(row.cells.len(), 4, "row {row_ix} must be a full 4-column grid after repair"); + } + let new_row_ix = merged + .rows + .iter() + .position(|row| !base_rows.contains(&row.id)) + .expect("inserted row present"); + let new_column_ix = merged + .columns + .iter() + .position(|column| !base_columns.contains(&column.id)) + .expect("inserted column present"); + let cross = &merged.rows[new_row_ix].cells[new_column_ix]; + assert_eq!( + cross.blocks.len(), + 1, + "synthesized cross cell must be one empty paragraph, not a doubled flow (got {} blocks)", + cross.blocks.len() + ); + assert_eq!(cell_text(cross), "", "synthesized cross cell must be empty"); + } + + /// Concurrent same-paragraph edits against the same base: both inserts and one + /// of the two concurrent paragraph styles must survive (the old two-peer + /// concurrent-paragraph regression, intent-flavored). + #[test] + fn concurrent_same_paragraph_edits_converge() { + let mut peers: Vec = (0..2).map(|_| Peer::new("same paragraph", 2)).collect(); + sync_all(&mut peers); + let paragraph = peers[0].projection().ids.paragraph_ids[0]; + peers[0] + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "hello world".into(), + style_override: None, + }) + .expect("seed text"); + sync_all(&mut peers); + + peers[0] + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "A>".into(), + style_override: None, + }) + .expect("peer 0 insert"); + peers[0] + .handle + .set_paragraph_style(SetParagraphStyleIntent { + paragraph, + style: ParagraphStyle::Custom(2), + }) + .expect("peer 0 style"); + peers[1] + .handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: "hello world = (0..1).map(|_| Peer::new("undo-table", 1)).collect(); + seed_table(&peers[0], &mut rng); + let peer = &peers[0]; + let check = |label: &str| { + if let Err(reason) = projections_agree(&peer.projection(), &peer.fresh_projection()) { + panic!("{label}: projection deviates from own canonical: {reason}"); + } + }; + check("post-seed"); + + let (table_id, table) = first_table(&peer.projection()).expect("table"); + let column0 = table.columns[0].id; + let row0 = table.rows[0].id; + + peer + .handle + .table_op(TableIntent::InsertColumn { + table: table_id, + after_column: Some(column0), + width: InputTableColumnWidth::Auto, + }) + .expect("insert column"); + check("post-insert-column"); + peer.handle.apply_undo().expect("undo insert column"); + check("post-undo"); + peer.handle.apply_redo().expect("redo insert column"); + check("post-redo"); + + peer + .handle + .table_op(TableIntent::SetCellSpan { + table: table_id, + row: row0, + column: column0, + row_span: 2, + column_span: 2, + }) + .expect("set cell span"); + check("post-set-span"); + + peer.handle.apply_undo().expect("undo set span"); + check("post-undo-span"); + peer.handle.apply_redo().expect("redo set span"); + check("post-redo-span"); + } + + /// TEMP shrink harness: single-peer sweep (no imports) to shrink the fuzz + /// finding to a local-only sequence. + #[test] + #[ignore = "shrink harness: single-peer seed sweep, run on demand with FUZZ_PER_OP_CHECK=1"] + fn scratch_single_peer_table_sweep() { + for seed in 1..200 { + run_fuzz(seed, 1, 3, 12, seed_table, random_table_mix_intent); + } + } + + #[test] + #[ignore = "shrink harness: single-peer seed sweep, run on demand with FUZZ_PER_OP_CHECK=1"] + fn scratch_single_peer_object_sweep() { + for seed in 1..200 { + run_fuzz(seed, 1, 3, 12, seed_structural, random_structural_intent); + } + } + + /// Minimal repro (from the single-peer fuzz shrink, seed 1): undo the + /// seeded fragment, rebuild some text, then insert an object at the FIRST + /// paragraph's byte 0. The maintained projection keeps the seeded root + /// paragraph id while a fresh rematerialization elects a fabricated + /// interstitial identity for the after-object row. + #[test] + fn object_insert_at_paragraph_start_after_undo_stays_canonical() { + let mut rng = Rng::new(1); + let peers: Vec = (0..1).map(|_| Peer::new("obj-undo", 1)).collect(); + let peer = &peers[0]; + seed_structural(peer, &mut rng); + peer.handle.apply_undo().expect("undo the seeded fixture"); + + let check = |label: &str| { + if let Err(reason) = projections_agree(&peer.projection(), &peer.fresh_projection()) { + panic!("{label}: projection deviates from own canonical: {reason}"); + } + }; + check("post-undo"); + + let projection = peer.projection(); + let paragraph = projection.ids.paragraph_ids[0]; + peer + .handle + .insert_object(InsertObjectIntent { + at: TextAnchor::new(paragraph, 0), + block: { + let mut object_rng = Rng::new(42); + object_input(&mut object_rng) + }, + }) + .expect("insert object at paragraph start"); + check("post-insert-object"); + } +} diff --git a/crates/flowstate-collab/tests/shallow_history.rs b/crates/flowstate-collab/tests/shallow_history.rs new file mode 100644 index 00000000..c5d2ced3 --- /dev/null +++ b/crates/flowstate-collab/tests/shallow_history.rs @@ -0,0 +1,308 @@ +//! §A12.1.3 slice 4 gates (design G2): a SHALLOW session must serve +//! far-behind peers from reconstructed full history (never the silently +//! truncating live export), and must durably absorb remote ops that depend +//! on pre-root history (the "rebase reopen" contract) instead of failing or +//! dropping them. + +#[cfg(test)] +mod tests { + mod tests { + use flowstate_collab::crdt_runtime::{CrdtRuntime, RuntimeEvent}; + use flowstate_collab::local_write::{InsertTextIntent, LocalDocHandle, LocalWriteConfig, TextAnchor}; + use flowstate_document::{DocumentPackage, loro_schema::body_text, loro_schema::new_loro_document}; + use loro::{ExportMode, LoroDoc}; + + /// Full source doc with pre-root history + a package carrying a shallow + /// chunk rooted mid-history, plus a far-behind peer forked BEFORE the root. + fn shallow_session_with_behind_peer() -> (CrdtRuntime, LoroDoc, LoroDoc) { + let source = new_loro_document("Shallow serve").expect("doc"); + let text = body_text(&source); + text + .insert(text.len_unicode(), "ancient shared history") + .expect("insert"); + source.commit(); + + // The far-behind peer last synced BEFORE the shallow root. + let behind = LoroDoc::new(); + behind + .import(&source.export(ExportMode::Snapshot).expect("snapshot")) + .expect("behind bootstrap"); + + text + .insert(text.len_unicode(), " | past the root") + .expect("insert"); + source.commit(); + let root = source.state_frontiers(); + text + .insert(text.len_unicode(), " | recent window") + .expect("insert"); + source.commit(); + + let mut package = DocumentPackage::from_loro_snapshot(&source, "Shallow serve").expect("package"); + package.compact_to_snapshot(&source).expect("compact"); + package + .record_shallow_snapshot(&source, &root) + .expect("record shallow"); + let live = package + .load_loro_doc_shallow() + .expect("shallow load") + .expect("shallow chunk present"); + assert!(live.is_shallow()); + let runtime = CrdtRuntime::from_doc(live, Some(package), None).expect("runtime"); + (runtime, source, behind) + } + + #[test] + fn below_root_pull_is_served_from_full_history() { + let (runtime, source, behind) = shallow_session_with_behind_peer(); + + // The live doc alone would silently truncate this export; the runtime + // must serve it from the reconstructed full history. + let updates = runtime + .export_updates_for(&behind.oplog_vv()) + .expect("below-root serve"); + let status = behind.import(&updates).expect("behind import"); + assert!(status.pending.is_none(), "the served range must be complete: {status:?}"); + assert_eq!( + body_text(&behind).to_string(), + body_text(&source).to_string(), + "the far-behind peer must converge to the tip" + ); + } + + #[test] + fn below_root_import_is_absorbed_durably() { + let (mut runtime, _source, behind) = shallow_session_with_behind_peer(); + + // The behind peer typed while offline: its ops depend on pre-root history. + let behind_text = body_text(&behind); + behind_text + .insert(behind_text.len_unicode(), " | offline peer edit") + .expect("behind edit"); + behind.commit(); + let foreign = behind + .export(ExportMode::updates(&loro::VersionVector::default())) + .expect("behind export"); + + let live_text_before = body_text(runtime.doc()).to_string(); + let events = runtime + .import_remote_updates(&[&foreign]) + .expect("below-root import must absorb, not fail"); + assert!( + events + .iter() + .flatten() + .any(|event| matches!(event, RuntimeEvent::HistoryRebaseRequired { .. })), + "the session must be told a reopen is required: {events:?}" + ); + // The in-memory shallow doc is untouched... + assert_eq!(body_text(runtime.doc()).to_string(), live_text_before); + // ...but the PACKAGE now carries the merged full history. + let merged = runtime + .package() + .expect("runtime keeps its package") + .load_loro_doc() + .expect("merged package loads full"); + assert!(!merged.is_shallow()); + let merged_text = body_text(&merged).to_string(); + assert!( + merged_text.contains("offline peer edit"), + "the foreign edit must be durably merged: {merged_text:?}" + ); + assert!(merged_text.contains("recent window"), "local history must survive the merge"); + } + + #[test] + fn shallow_session_undo_and_edits_work() { + // G5 smoke: a session opened from a shallow snapshot types and undoes on + // the normal write path; every checkout target is post-root by + // construction (the root is fixed at open). + let (runtime, _source, _behind) = shallow_session_with_behind_peer(); + assert!(runtime.doc().is_shallow()); + let (handle, _gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + let before = handle.projection().expect("projection"); + let before_text = flowstate_document::paragraph_text(&before, 0); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: " typed on the shallow session".to_string(), + style_override: None, + }) + .expect("insert on shallow session"); + let outcome = handle.apply_undo().expect("undo on shallow session"); + assert!(outcome.applied, "undo must apply on a shallow session"); + let after = handle.projection().expect("projection"); + assert_eq!(flowstate_document::paragraph_text(&after, 0), before_text); + } + + #[test] + fn shallow_session_converges_over_repeated_exchange() { + // G7 (targeted): repeated bidirectional exchange between a shallow + // session and an in-window peer stays convergent. + let (mut runtime, source, _behind) = shallow_session_with_behind_peer(); + // Peer bootstraps from the CURRENT tip (inside the window). + let peer = LoroDoc::new(); + peer + .import(&source.export(ExportMode::Snapshot).expect("snapshot")) + .expect("peer bootstrap"); + + for round in 0..5 { + // Peer edit → shallow session. + let peer_from = peer.oplog_vv(); + let peer_text = body_text(&peer); + peer_text + .insert(peer_text.len_unicode(), &format!(" | peer round {round}")) + .expect("peer edit"); + peer.commit(); + let update = peer + .export(ExportMode::updates(&peer_from)) + .expect("peer export"); + runtime + .import_remote_updates(&[&update]) + .expect("session import"); + + // Shallow session edit → peer. + let session_from = peer.oplog_vv(); + let session_text = body_text(runtime.doc()); + session_text + .insert(session_text.len_unicode(), &format!(" | session round {round}")) + .expect("session edit"); + runtime.doc().commit(); + let update = runtime + .export_updates_for(&session_from) + .expect("session export"); + peer.import(&update).expect("peer import"); + + assert_eq!( + body_text(runtime.doc()).to_string(), + body_text(&peer).to_string(), + "round {round} must converge" + ); + } + } + + #[test] + fn routine_checkpoint_skips_consolidation_but_stays_reopenable() { + // §A12.4.1: a revision checkpoint below the segment threshold keeps the + // old full snapshot + segment chain (no full-history export) yet the + // written package reopens to the same tip, with a fresh shallow + // accelerator. + let (mut runtime, _source, _behind) = shallow_session_with_behind_peer(); + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("incremental.db8"); + // (Edits go through the persist path in real sessions, keeping the + // package tip chained; the checkpoint's own meta/revision commit is the + // delta under test here.) + let snapshots_before = runtime.package().expect("package").loro_snapshots.len(); + let (job, _events) = runtime + .begin_checkpoint("Routine save", Some(path.clone())) + .expect("begin") + .expect("package present"); + let (package, wrote) = job.run(); + assert!(wrote.expect("job io"), "checkpoint must reach disk"); + assert_eq!( + package.loro_snapshots.len(), + snapshots_before, + "no full-history export below the threshold (a shallow session also skips the revision-load accelerator chunk)" + ); + assert!(!package.loro_update_segments.is_empty(), "the segment chain must survive"); + // §A13.4.3: the accelerator deliberately LAGS on routine checkpoints + // (re-minting costs a ~200ms shallow export; staleness only costs the + // next open a cheap segment replay). Present is the contract; the + // reopen assertions below prove the lagging chunk still reconstructs + // the tip. + assert!(package.latest_shallow_snapshot().is_some(), "accelerator chunk must survive"); + runtime.finish_checkpoint(package, true); + + let reopened = DocumentPackage::read(&path).expect("reopen"); + let full = reopened.load_loro_doc().expect("full load"); + assert_eq!( + body_text(&full).to_string(), + body_text(runtime.doc()).to_string(), + "the incremental checkpoint must reopen to the live tip" + ); + let shallow = reopened + .load_loro_doc_shallow() + .expect("shallow load") + .expect("chunk"); + assert_eq!(body_text(&shallow).to_string(), body_text(&full).to_string()); + } + + #[test] + fn shallow_session_slow_path_undo_is_bounded() { + // §A13.3.0: the slow-path (UndoManager) undo on a SHALLOW-opened doc. + // The 1.0-1.6s tail was measured on full-history in-memory docs; the + // shallow root must bound every walk to the session window. Remote + // chatter lands INSIDE the typed hull so the recorded fast path + // truncates and undo falls through to the checkout-based path. + let (runtime, source, _behind) = shallow_session_with_behind_peer(); + assert!(runtime.doc().is_shallow()); + let peer = LoroDoc::new(); + peer + .import(&source.export(ExportMode::Snapshot).expect("snapshot")) + .expect("peer bootstrap"); + let (handle, gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let paragraph = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..40 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, usize::MAX), + text: format!(" typed{i}"), + style_override: None, + }) + .expect("typed edit"); + if i % 4 == 0 { + // Remote chatter at the head of the same paragraph — inside the + // recorded hull, forcing truncation of the fast-path stacks. + let from = peer.oplog_vv(); + { + let guard = gate + .lock(flowstate_collab::local_write::GateHolder::ExportUpdates) + .expect("gate"); + let update = guard + .doc() + .export(ExportMode::updates(&peer.oplog_vv())) + .expect("session export"); + drop(guard); + peer.import(&update).expect("peer catch-up") + }; + let peer_text = body_text(&peer); + // INSIDE the typed tail (hull-overlapping): forces at-coordinate + // truncation of the recorded stacks, so undo falls to the slow path. + let at = peer_text.len_unicode().saturating_sub(2); + peer_text.insert(at, "r").expect("peer edit"); + peer.commit(); + let update = peer + .export(ExportMode::updates(&from)) + .expect("peer export"); + let mut guard = gate + .lock(flowstate_collab::local_write::GateHolder::ImportChunk) + .expect("gate"); + guard + .import_remote_updates(&[&update]) + .expect("session import"); + } + } + // Undo the whole session; time the worst step (slow path engages once + // the truncated stacks run dry). + let mut worst = std::time::Duration::ZERO; + let mut applied = 0; + for _ in 0..40 { + let t = std::time::Instant::now(); + let outcome = handle.apply_undo().expect("undo"); + worst = worst.max(t.elapsed()); + if !outcome.applied { + break; + } + applied += 1; + } + eprintln!("[shallow-undo-timing] applied={applied} worst_step={worst:?}"); + assert!(applied > 0, "at least one undo must apply"); + assert!( + worst < std::time::Duration::from_millis(2000), + "slow-path undo on a shallow session took {worst:?} — the root is not bounding the walk" + ); + } + } +} diff --git a/crates/flowstate-collab/tests/swarm_loopback.rs b/crates/flowstate-collab/tests/swarm_loopback.rs new file mode 100644 index 00000000..309e8504 --- /dev/null +++ b/crates/flowstate-collab/tests/swarm_loopback.rs @@ -0,0 +1,320 @@ +#![cfg(test)] + +use std::time::Duration; + +use anyhow::{Context as _, Result, bail, ensure}; +use async_channel::Receiver; +use flowstate_collab::{ + DIRECT_ALPN, SessionAdmission, SessionId, + net::{ + NetEvent, PublishPayload, auth, + direct::{self, DirectProto, DirectServeRequest, DirectServeState, DirectSessionHandler}, + swarm::SwarmHandle, + }, + proto_direct::DirectRequest, + proto_gossip::{GOSSIP_INLINE_LIMIT, GossipMsg}, + ticket::SessionTicket, +}; +use iroh::{Endpoint, EndpointAddr, Watcher as _, address_lookup::memory::MemoryLookup, endpoint::presets, protocol::Router}; +use iroh_gossip::net::Gossip; +use tokio::time::timeout; + +struct Peer { + endpoint: Endpoint, + gossip: Gossip, + direct_state: DirectServeState, + router: Router, + events: Receiver, + swarm: Option, +} + +impl Peer { + async fn spawn(lookup: MemoryLookup) -> Result { + let endpoint = Endpoint::builder(presets::Minimal) + .address_lookup(lookup) + .clear_ip_transports() + .bind_addr("127.0.0.1:0")? + .bind() + .await + .context("binding loopback endpoint failed")?; + let gossip = Gossip::builder().spawn(endpoint.clone()); + let direct_state = DirectServeState::default(); + let router = Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .accept(DIRECT_ALPN, DirectProto::new(direct_state.clone())) + .spawn(); + let (_, events) = async_channel::unbounded(); + + Ok(Self { + endpoint, + gossip, + direct_state, + router, + events, + swarm: None, + }) + } + + fn start_swarm(&mut self, session: SessionId, bootstrap: Vec) -> Result<()> { + let (evt_tx, evt_rx) = async_channel::unbounded(); + self.events = evt_rx; + self.swarm = Some(SwarmHandle::spawn( + self.endpoint.clone(), + self.gossip.clone(), + self.direct_state.clone(), + session, + bootstrap, + evt_tx, + )?); + Ok(()) + } + + async fn publish(&self, payload: PublishPayload) -> Result<()> { + let Some(swarm) = &self.swarm else { + bail!("peer swarm is not running"); + }; + swarm.publish(payload).await + } + + async fn stop_swarm(&mut self) { + if let Some(swarm) = self.swarm.take() { + swarm.stop().await; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "requires local UDP loopback and real iroh gossip timing"] +async fn swarm_loopback_net_subset() -> Result<()> { + let session = SessionId::new(); + let lookup = MemoryLookup::new(); + let mut a = Peer::spawn(lookup.clone()).await?; + let mut b = Peer::spawn(lookup.clone()).await?; + let mut c = Peer::spawn(lookup.clone()).await?; + let admission = SessionAdmission::generate(); + for peer in [&a, &b, &c] { + peer + .direct_state + .auth() + .configure_session(session, admission.clone()); + } + auth::install_local_admission(session, admission.clone()); + + let a_addr = wait_addr(&a.endpoint).await; + let b_addr = wait_addr(&b.endpoint).await; + let c_addr = wait_addr(&c.endpoint).await; + ensure!(!a_addr.is_empty(), "A endpoint did not expose loopback addresses"); + ensure!(!b_addr.is_empty(), "B endpoint did not expose loopback addresses"); + ensure!(!c_addr.is_empty(), "C endpoint did not expose loopback addresses"); + let addrs = [a_addr.clone(), b_addr.clone(), c_addr.clone()]; + for addr in addrs.iter().cloned() { + lookup.add_endpoint_info(addr); + } + for endpoint in [&a.endpoint, &b.endpoint, &c.endpoint] { + endpoint + .address_lookup()? + .add(MemoryLookup::from_endpoint_info(addrs.iter().cloned())); + } + + install_snapshot_handler(&a, session, b"snapshot-from-a".to_vec()).await; + let snapshot = direct::pull_with_endpoint( + &b.endpoint, + DirectRequest::Snapshot { session }, + vec![a.endpoint.id()], + Duration::from_secs(5), + ) + .await?; + assert_eq!(snapshot, b"snapshot-from-a".to_vec()); + + // B received the same session bearer secret as A, so its invite is not an + // owner proxy: C can authenticate directly to B and bootstrap from B even + // if A is unavailable. + install_snapshot_handler(&b, session, b"snapshot-from-b".to_vec()).await; + let b_invite = SessionTicket::new(session, vec![b_addr.clone()], "Shared brief".into(), admission.clone()); + let b_invite = SessionTicket::decode_text(&b_invite.encode_text())?; + let snapshot = direct::pull_with_endpoint( + &c.endpoint, + DirectRequest::Snapshot { session }, + vec![b_invite.bootstrap[0].id], + Duration::from_secs(5), + ) + .await?; + assert_eq!(snapshot, b"snapshot-from-b".to_vec()); + + a.start_swarm(session, Vec::new())?; + b.start_swarm(session, vec![a_addr.clone()])?; + c.start_swarm(session, vec![b_addr.clone()])?; + recv_neighbor(&a.events).await?; + recv_neighbor(&b.events).await?; + recv_neighbor(&c.events).await?; + + let small_update = b"a-to-b-to-c".to_vec(); + a.publish(PublishPayload::Update(small_update.clone())) + .await?; + recv_update(&b.events, &small_update).await?; + recv_update(&c.events, &small_update).await?; + + a.publish(PublishPayload::Presence(b"presence-a".to_vec())) + .await?; + b.publish(PublishPayload::Presence(b"presence-b".to_vec())) + .await?; + c.publish(PublishPayload::Presence(b"presence-c".to_vec())) + .await?; + recv_presence(&a.events).await?; + recv_presence(&b.events).await?; + recv_presence(&c.events).await?; + + let large_update = vec![7; GOSSIP_INLINE_LIMIT + 1]; + a.publish(PublishPayload::Update(large_update.clone())) + .await?; + let blob = recv_update_available(&b.events, large_update.len() as u64).await?; + let blob_bytes = direct::pull_with_endpoint( + &b.endpoint, + DirectRequest::Blob { session, blob }, + vec![a.endpoint.id()], + Duration::from_secs(5), + ) + .await?; + assert_eq!(blob_bytes, large_update); + + a.stop_swarm().await; + let b_update = b"b-after-a-left".to_vec(); + b.publish(PublishPayload::Update(b_update.clone())).await?; + recv_update(&c.events, &b_update).await?; + let c_update = b"c-after-a-left".to_vec(); + c.publish(PublishPayload::Update(c_update.clone())).await?; + recv_update(&b.events, &c_update).await?; + + c.stop_swarm().await; + c.start_swarm(session, vec![b_addr])?; + b.publish(PublishPayload::Digest { + vv: b"digest-after-resubscribe".to_vec(), + }) + .await?; + recv_digest(&c.events).await?; + + a.router + .shutdown() + .await + .context("shutting down A router failed")?; + b.router + .shutdown() + .await + .context("shutting down B router failed")?; + c.router + .shutdown() + .await + .context("shutting down C router failed")?; + auth::clear_local_admission(session); + Ok(()) +} + +async fn install_snapshot_handler(peer: &Peer, session: SessionId, snapshot: Vec) { + let (request_tx, request_rx) = async_channel::unbounded(); + peer + .direct_state + .register_handler(session, DirectSessionHandler::new(request_tx, None)) + .await; + tokio::spawn(async move { + while let Ok(request) = request_rx.recv().await { + if let DirectServeRequest::Snapshot { reply } = request { + let _ = reply.send(Ok(snapshot.clone())).await; + } + } + }); +} + +async fn wait_addr(endpoint: &Endpoint) -> EndpointAddr { + let mut watcher = endpoint.watch_addr(); + let wait = async { + loop { + let addr = watcher.get(); + if !addr.is_empty() { + return addr; + } + if watcher.updated().await.is_err() { + return endpoint.addr(); + } + } + }; + timeout(Duration::from_secs(3), wait) + .await + .unwrap_or_else(|_| endpoint.addr()) +} + +async fn recv_update(events: &Receiver, expected: &[u8]) -> Result<()> { + recv_matching( + events, + "update", + |event| matches!(event, NetEvent::Gossip { msg: GossipMsg::Update(bytes), .. } if bytes == expected), + ) + .await + .map(|_| ()) +} + +async fn recv_presence(events: &Receiver) -> Result<()> { + recv_matching(events, "presence", |event| { + matches!( + event, + NetEvent::Gossip { + msg: GossipMsg::Presence(_), + .. + } + ) + }) + .await + .map(|_| ()) +} + +async fn recv_neighbor(events: &Receiver) -> Result<()> { + recv_matching(events, "neighbor", |event| matches!(event, NetEvent::NeighborUp { .. })) + .await + .map(|_| ()) +} + +async fn recv_update_available(events: &Receiver, expected_len: u64) -> Result { + let event = recv_matching( + events, + "update-available", + |event| matches!(event, NetEvent::Gossip { msg: GossipMsg::UpdateAvailable { len, .. }, .. } if *len == expected_len), + ) + .await?; + let NetEvent::Gossip { + msg: GossipMsg::UpdateAvailable { blob, .. }, + .. + } = event + else { + bail!("matched event was not update-available"); + }; + Ok(blob) +} + +async fn recv_digest(events: &Receiver) -> Result<()> { + recv_matching(events, "digest", |event| { + matches!( + event, + NetEvent::Gossip { + msg: GossipMsg::Digest { .. }, + .. + } + ) + }) + .await + .map(|_| ()) +} + +async fn recv_matching(events: &Receiver, label: &str, matches: F) -> Result +where + F: Fn(&NetEvent) -> bool, +{ + timeout(Duration::from_secs(10), async { + loop { + let event = events.recv().await.context("event channel closed")?; + if matches(&event) { + return Ok(event); + } + } + }) + .await + .with_context(|| format!("timed out waiting for {label}"))? +} diff --git a/crates/flowstate-collab/tests/undo_overlap_probe.rs b/crates/flowstate-collab/tests/undo_overlap_probe.rs new file mode 100644 index 00000000..922d8945 --- /dev/null +++ b/crates/flowstate-collab/tests/undo_overlap_probe.rs @@ -0,0 +1,110 @@ +//! §A14.3.2 frequency probe: is the slow-path undo hit by MARK-overlap +//! (rewritable, worth fixing) or only by destructive CONTENT/structural +//! overlap (a genuine Loro checkout-diff floor)? + +#[cfg(test)] +mod tests { + mod tests { + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, SetParagraphStylesIntent, TextAnchor}; + use flowstate_document::{ParagraphStyle, loro_schema::body_text}; + use loro::LoroDoc; + + fn seeded() -> ( + LocalDocHandle, + std::sync::Arc>, + LoroDoc, + ) { + let core = CrdtRuntime::new_empty("undo overlap").expect("runtime"); + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let para = handle.projection().expect("projection").ids.paragraph_ids[0]; + for i in 0..200 { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(para, usize::MAX), + text: format!("word{i} "), + style_override: None, + }) + .expect("seed"); + } + let peer = { + let g = gate.lock(GateHolder::ExportUpdates).expect("gate"); + let doc = LoroDoc::new(); + doc + .import( + &g.doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("export"), + ) + .expect("peer boot"); + doc + }; + (handle, gate, peer) + } + + fn push_peer(gate: &std::sync::Arc>, peer: &LoroDoc, from: &loro::VersionVector) { + let update = peer + .export(loro::ExportMode::updates(from)) + .expect("peer export"); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&update) + .expect("import"); + } + + #[test] + fn mark_overlap_undo_timing() { + // A restyles a paragraph range; B applies a DIFFERENT mark to an + // overlapping range; A undoes. Is A's undo fast (rebasable marks) or slow? + let (handle, gate, peer) = seeded(); + let ids = handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .clone(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids.to_vec(), + style: ParagraphStyle::Custom(3), + }) + .expect("A restyle"); + // B: a mark on the body (bold-ish via a run style) overlapping A's range. + let from = peer.oplog_vv(); + let body = body_text(&peer); + let _ = body.mark(0..50, "bold", loro::LoroValue::Bool(true)); + peer.commit(); + push_peer(&gate, &peer, &from); + let t = std::time::Instant::now(); + let outcome = handle.apply_undo().expect("undo"); + eprintln!("[undo-overlap] mark_overlap undo: {:?} applied={}", t.elapsed(), outcome.applied); + } + + #[test] + fn content_overlap_undo_timing() { + // Baseline: content insert overlapping A's restyled range (the bench shape). + let (handle, gate, peer) = seeded(); + let ids = handle + .projection() + .expect("projection") + .ids + .paragraph_ids + .clone(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: ids.to_vec(), + style: ParagraphStyle::Custom(3), + }) + .expect("A restyle"); + let from = peer.oplog_vv(); + let body = body_text(&peer); + body.insert(5, "INJECT").expect("peer insert"); + peer.commit(); + push_peer(&gate, &peer, &from); + let t = std::time::Instant::now(); + let outcome = handle.apply_undo().expect("undo"); + eprintln!("[undo-overlap] content_overlap undo: {:?} applied={}", t.elapsed(), outcome.applied); + } + } +} diff --git a/crates/flowstate-collab/tests/vendored_patch_guards.rs b/crates/flowstate-collab/tests/vendored_patch_guards.rs new file mode 100644 index 00000000..d1a1c2e7 --- /dev/null +++ b/crates/flowstate-collab/tests/vendored_patch_guards.rs @@ -0,0 +1,304 @@ +//! §act-eleven C6: workspace guards for the vendored loro-internal checkout +//! patches (flowstate-loro-vendor-patch #4–#7 — the richtext tracker/`id_to_cursor` +//! quadratics). The vendor tree carries its own perf-regression tests, but the +//! vendored crate cannot `cargo test` (workspace-detection + feature conflicts), +//! so those tests NEVER RUN — a Loro upgrade could silently drop a patch with no +//! failing test anywhere. This guard exercises the property through the PUBLIC +//! import path and asserts SCALING (a work ratio), not wall-clock, so it is +//! machine- and build-profile-independent in what it rejects: the unpatched +//! quadratic makes the ratio grow with N (16x at these sizes), the patched +//! amortized-linear behavior keeps it near the size ratio (4x). + +use anyhow::Result; +use flowstate_collab::crdt_runtime::CrdtRuntime; +use flowstate_collab::local_write::{GateHolder, InsertTextIntent, LocalDocHandle, LocalWriteConfig, TextAnchor}; +use std::time::Instant; + +/// One peer builds a large single-fragment body; a forked peer splits it with +/// `splits` interleaved single-char inserts (each lands mid-fragment, the +/// pathological shape); the first peer imports the whole divergence as ONE +/// blob, driving the vendored tracker checkout over every split. +fn interleaved_split_import_cost(splits: usize) -> Result { + let base_len: usize = 64 * 1024; + let runtime = CrdtRuntime::new_empty("patch-guard")?; + let (handle, gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let projection = handle.projection()?; + let paragraph = projection.ids.paragraph_ids[0]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(paragraph, 0), + text: "x".repeat(base_len), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("seed insert rejected: {error:?}"))?; + + // Forked peer: same history, then `splits` mid-fragment inserts. + let fork_bytes = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::Snapshot) + .expect("snapshot export") + }; + let peer = CrdtRuntime::new_empty("patch-guard-peer")?; + let (peer_handle, peer_gate) = LocalDocHandle::new(peer, LocalWriteConfig::default()); + { + let mut guard = peer_gate + .lock(GateHolder::ImportChunk) + .expect("gate healthy"); + guard.import_remote_update(&fork_bytes)? + }; + // Bidirectional base sync: a `new_empty` peer mints its OWN sentinel init + // ops, so the importer must see them too before the timed leg — otherwise + // the final convergence assert compares different baselines (the known + // one-way-exchange trap from the P3-deep validation). + { + let peer_init = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("peer init export") + }; + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard.import_remote_update(&peer_init)? + }; + let peer_vv_before = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + guard.doc().state_vv() + }; + let peer_projection = peer_handle.projection()?; + let peer_paragraph = *peer_projection + .ids + .paragraph_ids + .last() + .expect("peer paragraph"); + let stride = base_len / (splits + 1); + for split_ix in 0..splits { + peer_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_paragraph, (split_ix + 1) * stride + split_ix), + text: "y".into(), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("split insert {split_ix} rejected: {error:?}"))?; + } + let divergence = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + guard + .doc() + .export(loro::ExportMode::updates(&peer_vv_before)) + .expect("delta export") + }; + + let start = Instant::now(); + { + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard.import_remote_update(&divergence)? + }; + let elapsed = start.elapsed(); + + // Convergence stays the point: both peers must agree byte-for-byte. + let local = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate healthy"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + }; + let remote = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("gate healthy"); + flowstate_document::loro_schema::body_text(guard.doc()).to_string() + }; + assert_eq!(local, remote, "interleaved-split import diverged at splits={splits}"); + Ok(elapsed) +} + +/// §act-eleven A11.5 net: styled-divergence cost at `edits` ops. One peer types +/// `edits` single-char inserts at cycling positions, marking a short (20-char) +/// same-key range every 8th op — the shape that accumulates style-range +/// BOUNDARIES (2 per mark) so later inserts keep landing on the +/// `StyleRangeMap` boundary-intersection path. Returns (local edit time, +/// reconnect import time on the offline peer). +fn styled_divergence_cost(edits: usize) -> (std::time::Duration, std::time::Duration) { + let a = loro::LoroDoc::new(); + a.set_peer_id(1).expect("peer id"); + let b = loro::LoroDoc::new(); + b.set_peer_id(2).expect("peer id"); + let ta = a.get_text("t"); + ta.insert(0, &"base text ".repeat(200)).expect("seed"); + a.commit(); + b.import(&a.export(loro::ExportMode::all_updates()).expect("export")) + .expect("base import"); + let b_offline_vv = a.oplog_vv(); + b.get_text("t").insert(0, "b").expect("divergence seed"); + b.commit(); + + let edit_start = Instant::now(); + for edit_ix in 0..edits { + let pos = (edit_ix * 13) % ta.len_unicode(); + ta.insert(pos, "a").expect("insert"); + if edit_ix % 8 == 0 { + let end = (pos + 20).min(ta.len_unicode()); + if pos + 1 < end { + ta.mark(pos..end, "bold", true).expect("mark"); + } + } + a.commit(); + } + let edit_time = edit_start.elapsed(); + + let blob = a + .export(loro::ExportMode::updates(&b_offline_vv)) + .expect("divergence export"); + let import_start = Instant::now(); + b.import(&blob).expect("reconnect import"); + let import_time = import_start.elapsed(); + a.import( + &b.export(loro::ExportMode::updates(&a.oplog_vv())) + .expect("b export"), + ) + .expect("a import"); + b.import( + &a.export(loro::ExportMode::updates(&b.oplog_vv())) + .expect("a export"), + ) + .expect("b import"); + assert_eq!(ta.to_string(), b.get_text("t").to_string(), "styled divergence failed to converge"); + (edit_time, import_time) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// §stylemap pass (2026-07-09, follow-on to act-eleven A11.5): pins the + /// LAYERED `StyleValue` representation in the vendored `style_range_map.rs` + /// (persistent `im::OrdSet` base + small owned overlay + one-side-reuse + /// intersections + `max_common` for the keystroke path). Measured on this + /// shape at 16k→48k ops (3× input): + /// * EDIT (per-keystroke, the user-felt path): pre-pass 91.7× ratio + /// (5.39s absolute at 48k) → **15.4× (0.96s)** — the boundary-set copy + /// quadratic's constant is gone; the residual super-linearity is the + /// per-boundary intersection compare walk. + /// * IMPORT (reconnect): ratio 87× (3.5s at 48k; pre-pass 30.9×/1.2s — the + /// absolute regression on THIS adversarial shape is the accepted cost of + /// the edit win). The import curve is dominated by the annotate WALK + /// (every covered elem must record membership — 9M visits measured at + /// 48k), which NO per-elem representation can beat. + /// + /// Both residuals have one fix: a lazy-tag range structure (annotate as an + /// O(log) subtree tag instead of a per-elem walk) — the scheduled next + /// `StyleRangeMap` pass. When it lands, tighten BOTH bounds to < 8. + #[test] + fn styled_divergence_scales_within_layered_representation_bounds() { + let _ = styled_divergence_cost(1_000); // warm + let (small_edit, small_import) = styled_divergence_cost(16_000); + let (large_edit, large_import) = styled_divergence_cost(48_000); + let edit_ratio = large_edit.as_secs_f64() / small_edit.as_secs_f64().max(0.001); + let import_ratio = large_import.as_secs_f64() / small_import.as_secs_f64().max(0.001); + // Bounds = measured-at-landing + headroom. The pre-pass representation + // trips the edit bound at ~92×; a lost overlay/one-side-reuse patch trips + // it too. The import bound catches regression past the documented walk + // quadratic. + assert!( + edit_ratio < 30.0 && import_ratio < 140.0, + "styled-divergence scaling for 3x input: edit {edit_ratio:.1}x (small={small_edit:?}, large={large_edit:?}, bound 30), import {import_ratio:.1}x (small={small_import:?}, large={large_import:?}, bound 140)" + ); + } + + #[test] + fn interleaved_split_checkout_scales_subquadratically() -> Result<()> { + // Warm once so allocator/lazy-init noise doesn't pollute the small run. + let _ = interleaved_split_import_cost(16)?; + let small = interleaved_split_import_cost(128)?; + let large = interleaved_split_import_cost(512)?; + // Size ratio 4x. Patched (amortized-linear) checkout ⇒ time ratio ≈ 4; + // the unpatched quadratic ⇒ ≈ 16. The bound sits between with margin for + // noise; on a loaded machine the ABSOLUTE floor also protects against a + // degenerate tiny `small` inflating the ratio. + let ratio = large.as_secs_f64() / small.as_secs_f64().max(0.0005); + assert!( + ratio < 10.0, + "interleaved-split import scaled {ratio:.1}x for a 4x input (small={small:?}, large={large:?}) — a vendored checkout patch (loro-vendor-patch #4–#7) has regressed" + ); + Ok(()) + } + + /// §bimodal-undo fix (vendor patch #23 + the vendored generic-btree + /// `next_leaf_matching`): two peers each PREPEND `n` chars concurrently, then + /// a checkout crosses from one tip to the other. The LCA is the ROOT, so the + /// tracker rebuild feeds branch A, RETREATS it wholesale (marks every A + /// element FUTURE), then feeds branch B — whose inserts land at position 0 + /// with the entire retreated A block to their right. Upstream's linear + /// `origin_right` scan made every such insert O(block) ⇒ O(n²) per rebuild: + /// the 80-195s dirty-history undo regime, load/merge-boundary dependent. The + /// probe + `non_future_num` cache jump makes it O(n log n). + fn concurrent_prepend_checkout_cost(n: usize) -> (std::time::Duration, String) { + let a = loro::LoroDoc::new(); + a.set_peer_id(1).expect("peer id"); + let ta = a.get_text("t"); + for _ in 0..n { + ta.insert(0, "a").expect("A prepend"); + a.commit(); + } + let b = loro::LoroDoc::new(); + b.set_peer_id(2).expect("peer id"); + let tb = b.get_text("t"); + for _ in 0..n { + tb.insert(0, "b").expect("B prepend"); + b.commit(); + } + let a_tip = a.state_frontiers(); + let b_tip = b.state_frontiers(); + a.import( + &b.export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("B export"), + ) + .expect("A imports B"); + let merged = { + // Reference merged text from an independent replica (convergence oracle). + let c = loro::LoroDoc::new(); + c.import( + &a.export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("A export"), + ) + .expect("C imports merged"); + c.get_text("t").to_string() + }; + a.checkout(&a_tip).expect("checkout A tip"); + let t = Instant::now(); + // The timed leg: A-tip → B-tip crosses the root LCA — the tracker feeds + // both branches and the second faces the first as one giant future run. + a.checkout(&b_tip).expect("checkout B tip"); + let cost = t.elapsed(); + a.checkout_to_latest(); + assert_eq!( + a.get_text("t").to_string(), + merged, + "checkout round-trip must not corrupt the merged text" + ); + (cost, merged) + } + + #[test] + fn concurrent_prepend_checkout_scales_subquadratically() { + let _ = concurrent_prepend_checkout_cost(256); // warm + let (small, small_text) = concurrent_prepend_checkout_cost(4_000); + let (large, large_text) = concurrent_prepend_checkout_cost(12_000); + assert_eq!(small_text.len(), 8_000); + assert_eq!(large_text.len(), 24_000); + // Size ratio 3x. Near-linear ⇒ time ratio ≈ 3-4; the unpatched quadratic + // ⇒ ≥ 9 (measured far higher — the future-run Vec collection compounds). + let ratio = large.as_secs_f64() / small.as_secs_f64().max(0.0005); + assert!( + ratio < 8.0, + "concurrent-prepend checkout scaled {ratio:.1}x for a 3x input (small={small:?}, large={large:?}) — the future-run probe/jump (vendor patch #23, vendored generic-btree next_leaf_matching) has regressed" + ); + } +} diff --git a/crates/flowstate-collab/tests/warm_import_calculator.rs b/crates/flowstate-collab/tests/warm_import_calculator.rs new file mode 100644 index 00000000..7a77eda8 --- /dev/null +++ b/crates/flowstate-collab/tests/warm_import_calculator.rs @@ -0,0 +1,68 @@ +//! §perf-heaven T8.18 — the retained import calculator's eager-warm +//! (`LoroDoc::warm_import_diff_calculator`) pre-builds the per-container trackers +//! so the FIRST remote import of a session reuses the built `id_to_cursor` index +//! instead of constructing it cold on the receive path. +//! +//! Warming is a pure PERFORMANCE precomputation: `start_tracking` re-validates +//! each tracker on every real import and rebuilds it if it does not cover the +//! requested version, so a warmed peer MUST converge to exactly the same state as +//! a cold peer that applied the identical operations. This test pins that +//! invariant (the convergence fuzz separately proves the import path itself is +//! unchanged by the additive vendored method). + +#[cfg(test)] +mod tests { + use loro::{ExportMode, LoroDoc}; + + #[test] + fn warm_import_diff_calculator_preserves_convergence() { + let src = LoroDoc::new(); + src.set_peer_id(1).unwrap(); + let text = src.get_text("text"); + text.insert(0, "hello").unwrap(); + src.commit(); + let first_update = src.export(ExportMode::all_updates()).unwrap(); + let first_vv = src.oplog_vv(); + + text.insert(5, " world").unwrap(); + src.commit(); + let second_update = src.export(ExportMode::updates(&first_vv)).unwrap(); + + // A forward-extension import AFTER warming converges to src (and a second warm + // is idempotent). + let dst = LoroDoc::new(); + dst.set_peer_id(2).unwrap(); + dst.import(&first_update).unwrap(); + dst.inner().warm_import_diff_calculator(); + dst.inner().warm_import_diff_calculator(); + dst.import(&second_update).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + + // A CONCURRENT import after warming must match a COLD peer that applied the + // identical ops (same peer id ⇒ same op ids; the only difference is the warm). + let make_peer = |warm: bool| { + let peer = LoroDoc::new(); + peer.set_peer_id(3).unwrap(); + peer.import(&first_update).unwrap(); + if warm { + peer.inner().warm_import_diff_calculator(); + } + peer.get_text("text").insert(5, "!").unwrap(); + peer.commit(); + peer.import(&second_update).unwrap(); + peer + }; + let warmed = make_peer(true); + let cold = make_peer(false); + assert_eq!( + warmed.get_deep_value(), + cold.get_deep_value(), + "warmed and cold peers must converge to identical state" + ); + assert_eq!(warmed.oplog_frontiers(), cold.oplog_frontiers()); + + // Warming an empty doc is a no-op (must not panic). + LoroDoc::new().inner().warm_import_diff_calculator(); + } +} diff --git a/crates/flowstate-corpus/Cargo.toml b/crates/flowstate-corpus/Cargo.toml new file mode 100644 index 00000000..0af7648b --- /dev/null +++ b/crates/flowstate-corpus/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "flowstate-corpus" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +flowstate-collab = { version = "0.1.0", path = "../flowstate-collab" } +flowstate-document = { version = "0.1.0", path = "../flowstate-document" } +flowstate-docx = { version = "0.1.0", path = "../flowstate-docx" } +tempfile.workspace = true +gpui.workspace = true +gpui-component.workspace = true +gpui-flowtext = { version = "0.1.0", path = "../gpui-flowtext" } +hotpath.workspace = true +ignore.workspace = true +image = "0.25.10" +loro = "1.13" +mimalloc.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } + +# Profiler features (mirror the app's, for the `hotpath_bench` bin). Off by default, +# so `flowstate-corpus` (the sweep) and `perf_probe` build without any hotpath cost. +# Enable one for the profiler tables: `cargo run --release --features hotpath-alloc --bin hotpath_bench -- `. +[features] +hotpath = ["hotpath/hotpath", "flowstate-document/hotpath", "flowstate-docx/hotpath", "flowstate-collab/hotpath", "gpui-flowtext/hotpath"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "flowstate-document/hotpath-cpu", "flowstate-docx/hotpath-cpu", "flowstate-collab/hotpath-cpu", "gpui-flowtext/hotpath-cpu"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "flowstate-document/hotpath-alloc", "flowstate-docx/hotpath-alloc", "flowstate-collab/hotpath-alloc", "gpui-flowtext/hotpath-alloc"] + +[lints] +workspace = true diff --git a/crates/flowstate-corpus/src/bin/collab_bench.rs b/crates/flowstate-corpus/src/bin/collab_bench.rs new file mode 100644 index 00000000..0d45d024 --- /dev/null +++ b/crates/flowstate-corpus/src/bin/collab_bench.rs @@ -0,0 +1,619 @@ +//! Headless collab profiler bench — drives the real field hot paths through the actual +//! `CrdtRuntime` + `LocalDocHandle` write path on a real `.docx`, and dumps the same +//! hotpath tables `cargo survey` does. Complements `hotpath_bench` (which measures the +//! COLD reopen projection); this measures the WARM steady-state operations: +//! OPEN — docx -> projection -> canonical `CrdtRuntime`. +//! IMPORT — receiving-peer `import_remote_update` (a concurrent peer's edit). +//! KEYSTROKE — one `InsertTextIntent` through the write authority. +//! MASS-RESTYLE — one `SetParagraphStylesIntent` over EVERY paragraph (select-all). +//! UNDO / REDO — revert/replay the mass restyle. +//! +//! Run: `cargo run --release --features hotpath-alloc --bin collab_bench -- [ROUNDS]`. + +use std::alloc::{GlobalAlloc, Layout}; +use std::time::Instant; + +use flowstate_collab::crdt_runtime::CrdtRuntime; +use flowstate_collab::local_write::{GateHolder, LocalDocHandle, LocalWriteConfig}; +use flowstate_document::block_ix_scan_count; +use flowstate_document::instrument; +use gpui_flowtext::{ + DeleteRangeIntent, InsertTextIntent, JoinParagraphsIntent, ParagraphStyle, ReplaceMatch, ReplaceMatchesIntent, SetParagraphStylesIntent, + SplitParagraphIntent, TextAnchor, +}; + +/// Print the algorithmic-work counters accumulated during `label`'s op — the +/// §perf-heaven tripwires (whole-body delta builds, block-index scans, full +/// projections, cursor resolves). In `--release` (where these benches run) the +/// debug audit is OFF, so a warm op should show ZERO whole-body builds and ZERO +/// block-index scans; a non-zero number is a T2/T3 regression made visible. +fn report_work(label: &str, before: instrument::WorkCounts, before_scans: u64) { + let work = instrument::snapshot().since(before); + let scans = block_ix_scan_count().saturating_sub(before_scans); + eprintln!( + " {label:<12} work: body_to_delta={} block_ix_scans={} full_projections={} cursor_resolves={}", + work.body_to_delta_builds, scans, work.full_projections, work.cursor_pos_resolves, + ); +} + +struct FlowstateAllocator; + +impl Default for FlowstateAllocator { + fn default() -> Self { + Self + } +} + +// SAFETY: forwards every call unchanged to mimalloc, which upholds the contract. +unsafe impl GlobalAlloc for FlowstateAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: layout contract upheld by the caller; forwarded unchanged. + unsafe { mimalloc::MiMalloc.alloc(layout) } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: ptr/layout originate from this allocator; forwarded unchanged. + unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) } + } +} + +#[cfg(not(feature = "hotpath-alloc"))] +#[global_allocator] +static GLOBAL: FlowstateAllocator = FlowstateAllocator; + +#[hotpath::main(allocator = FlowstateAllocator)] +fn main() { + // Surface `tracing::warn!` bail diagnostics (regional-rematerializer bails + // etc.) on stderr when RUST_LOG is set — they are invisible otherwise. + if std::env::var_os("RUST_LOG").is_some() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .try_init(); + } + let mut args = std::env::args().skip(1); + let Some(path) = args + .next() + .or_else(|| std::env::var("FLOWSTATE_BENCH_FIXTURE").ok()) + else { + eprintln!( + "usage: collab_bench :> [ROUNDS] (or set FLOWSTATE_BENCH_FIXTURE)\n \ + synthetic kinds: equations (default), mixed, text — e.g. synthetic:equations:4000" + ); + std::process::exit(2); + }; + let rounds: usize = args.next().and_then(|v| v.parse().ok()).unwrap_or(20); + + // OPEN: source -> projection -> canonical runtime (body decoded, as after a live open). + // A `synthetic:*` selector builds an equation/object-bearing fixture in memory + // (§perf-heaven T7 — the corpus has no equation docs); otherwise convert a docx. + let open = Instant::now(); + let (projection, paragraph_count) = flowstate_corpus::fixtures::from_selector(&path).unwrap_or_else(|| { + let (projection, report) = flowstate_docx::convert_docx_to_document(&path).expect("convert docx"); + let count = report.paragraphs_imported; + (projection, count) + }); + let core = CrdtRuntime::from_document_projection(&projection, "collab-bench").expect("runtime"); + eprintln!( + "OPEN {:8.1} ms ({paragraph_count} paragraphs)", + open.elapsed().as_secs_f64() * 1000.0 + ); + + // A concurrent peer B forks the shared history and will feed edits to A. + let doc_b = core.doc().fork(); + doc_b.set_peer_id(0x00B0_B0B0).expect("distinct peer id"); + let mut peer_vv = doc_b.state_vv(); + let body_b = flowstate_document::loro_schema::body_text(&doc_b); + + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let live = handle.projection().expect("projection"); + let paragraph_ids = live.ids.paragraph_ids.clone(); + let first = *paragraph_ids.first().expect("at least one paragraph"); + + // IMPORT: receiving-peer cost — peer B prepends a char, A imports the delta. This is + // the field "another peer is typing" path (derive projection events on the receiver). + // §perf-heaven T1: the receive path is regional (derive_body_projection_events), so + // in --release the work report should show ZERO whole-body deltas per import; a + // non-zero number means the regional receive regressed to a full re-decode. + let import_work = instrument::snapshot(); + let import_scans = block_ix_scan_count(); + let mut import_ms = 0.0; + for _ in 0..rounds { + body_b.insert(1, "x").expect("peer insert"); + doc_b.commit(); + let update = doc_b + .export(loro::ExportMode::updates(&peer_vv)) + .expect("peer export"); + peer_vv = doc_b.state_vv(); + let t = Instant::now(); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&update) + .expect("import"); + import_ms += t.elapsed().as_secs_f64() * 1000.0; + } + eprintln!( + "IMPORT {:8.1} ms ({rounds} rounds, {:.2} ms/round)", + import_ms, + import_ms / rounds as f64 + ); + report_work("IMPORT", import_work, import_scans); + + // KEYSTROKE: local insert intent through the write authority. + let key_work = instrument::snapshot(); + let key_scans = block_ix_scan_count(); + let mut key_ms = 0.0; + for _ in 0..rounds { + let t = Instant::now(); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(first, 0), + text: "y".to_string(), + style_override: None, + }) + .expect("insert_text"); + key_ms += t.elapsed().as_secs_f64() * 1000.0; + } + eprintln!( + "KEYSTROKE {:8.1} ms ({rounds} rounds, {:.2} ms/keystroke)", + key_ms, + key_ms / rounds as f64 + ); + report_work("KEYSTROKE", key_work, key_scans); + + // MASS-RESTYLE: select-all paragraph-style change (one batched intent over every para). + let mass_work = instrument::snapshot(); + let mass_scans = block_ix_scan_count(); + let mass = Instant::now(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: paragraph_ids.to_vec(), + style: ParagraphStyle::Custom(2), + }) + .expect("set_paragraph_styles"); + eprintln!( + "MASS-RESTYLE {:8.1} ms ({} paragraphs)", + mass.elapsed().as_secs_f64() * 1000.0, + paragraph_ids.len() + ); + report_work("MASS-RESTYLE", mass_work, mass_scans); + + // §oom-leads #4: the structural rounds are COUNTER nets (fallback/whole-body + // tripwires), and REMOTE-STRUCT is a known ~340ms/op cliff until the remote + // regional fix lands — they'd swamp the hotpath share-gate's profile, so + // gate runs skip them (heaven.sh sets the env). + let skip_structural = std::env::var_os("FLOWSTATE_BENCH_SKIP_STRUCTURAL").is_some(); + if !skip_structural { + // STRUCTURAL (§oom-leads #4): split/join through the write authority plus a + // REMOTE structural import (peer B inserts a paragraph break). The §6-R + // regional rematerializer must serve all of these — `full_projections` + // in the work line is the fallback tripwire: every count here is a whole-doc + // rebuild (~warm-project cost) where a region-sized patch was intended. + let structural_work = instrument::snapshot(); + let structural_scans = block_ix_scan_count(); + let mut structural_ms = 0.0; + for round in 0..rounds { + let live = handle.projection().expect("projection"); + let mid_ix = live.ids.paragraph_ids.len() / 2 + (round % 3); + let mid = live.ids.paragraph_ids[mid_ix]; + let t = Instant::now(); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(mid, 1), + inherited_style: ParagraphStyle::Normal, + }) + .expect("split_paragraph"); + structural_ms += t.elapsed().as_secs_f64() * 1000.0; + let live = handle.projection().expect("projection"); + let first_id = live.ids.paragraph_ids[mid_ix]; + let second_id = live.ids.paragraph_ids[mid_ix + 1]; + let t = Instant::now(); + handle + .join_paragraphs(JoinParagraphsIntent { + first: first_id, + second: second_id, + }) + .expect("join_paragraphs"); + structural_ms += t.elapsed().as_secs_f64() * 1000.0; + } + eprintln!( + "STRUCTURAL {:8.1} ms ({rounds} split+join rounds, {:.2} ms/op)", + structural_ms, + structural_ms / (rounds as f64 * 2.0) + ); + report_work("STRUCTURAL", structural_work, structural_scans); + + // REMOTE-STRUCTURAL (§oom-leads #4): peer B is a REAL runtime driving the + // actual split_paragraph write path (registry records + sentinel mark — the + // true wire shape), then A imports the delta. The earlier synthetic rounds + // wrote bare/naked sentinels, which never carry registry records and forced + // the region resolver into permanent full-rebuild — realistic for the docx + // `w:br` shape (kept below as REMOTE-STRUCT-BARE) but NOT for peer splits. + let peer_runtime = CrdtRuntime::new_empty("collab-bench-peer").expect("peer runtime"); + let (peer_handle, peer_gate) = LocalDocHandle::new(peer_runtime, LocalWriteConfig::default()); + { + // Bidirectional init exchange (the known one-way sentinel trap). + let init_a = gate + .lock(GateHolder::ExportUpdates) + .expect("gate") + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("A init export"); + peer_gate + .lock(GateHolder::ImportChunk) + .expect("peer gate") + .import_remote_update(&init_a) + .expect("peer imports A"); + let init_b = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("peer gate") + .doc() + .export(loro::ExportMode::updates(&loro::VersionVector::default())) + .expect("B init export"); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&init_b) + .expect("A imports B init") + }; + let mut peer_handle_vv = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("peer gate"); + guard.doc().state_vv() + }; + let remote_structural_work = instrument::snapshot(); + let remote_structural_scans = block_ix_scan_count(); + let mut remote_structural_ms = 0.0; + for round in 0..rounds { + let peer_projection = peer_handle.projection().expect("peer projection"); + let mid_ix = peer_projection.ids.paragraph_ids.len() / 2 + (round % 3); + let mid = peer_projection.ids.paragraph_ids[mid_ix]; + peer_handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(mid, 1), + inherited_style: ParagraphStyle::Normal, + }) + .expect("peer split"); + let update = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("peer gate"); + let update = guard + .doc() + .export(loro::ExportMode::updates(&peer_handle_vv)) + .expect("peer export"); + peer_handle_vv = guard.doc().state_vv(); + update + }; + let t = Instant::now(); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&update) + .expect("import real split"); + remote_structural_ms += t.elapsed().as_secs_f64() * 1000.0; + } + eprintln!( + "REMOTE-SPLIT-REAL {:8.1} ms ({rounds} rounds, {:.2} ms/round)", + remote_structural_ms, + remote_structural_ms / rounds as f64 + ); + report_work("REMOTE-SPLIT-REAL", remote_structural_work, remote_structural_scans); + + // REMOTE-SPLIT-CONCUR: same real peer split, but A commits a local keystroke + // AFTER B's export and BEFORE the import, so the imported chunk is CONCURRENT + // with A's state — the diff arrives as origin `DiffMode::Import` (the field + // shape for two people typing) instead of `ImportGreaterUpdates` (the quiet + // strictly-ahead shape REMOTE-SPLIT-REAL measures). Exercises the vendored + // Import-origin per-key registry map diff. + let concurrent_work = instrument::snapshot(); + let concurrent_scans = block_ix_scan_count(); + let mut concurrent_ms = 0.0; + for round in 0..rounds { + let peer_projection = peer_handle.projection().expect("peer projection"); + let mid_ix = peer_projection.ids.paragraph_ids.len() / 2 + (round % 3); + let mid = peer_projection.ids.paragraph_ids[mid_ix]; + peer_handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(mid, 1), + inherited_style: ParagraphStyle::Normal, + }) + .expect("peer split"); + let update = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("peer gate"); + let update = guard + .doc() + .export(loro::ExportMode::updates(&peer_handle_vv)) + .expect("peer export"); + peer_handle_vv = guard.doc().state_vv(); + update + }; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(first, 0), + text: "z".to_string(), + style_override: None, + }) + .expect("concurrent local keystroke"); + let t = Instant::now(); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&update) + .expect("import concurrent split"); + concurrent_ms += t.elapsed().as_secs_f64() * 1000.0; + } + eprintln!( + "REMOTE-SPLIT-CONCUR {:8.1} ms ({rounds} rounds, {:.2} ms/round)", + concurrent_ms, + concurrent_ms / rounds as f64 + ); + report_work("REMOTE-SPLIT-CONCUR", concurrent_work, concurrent_scans); + + // ---- §mass-op collab rounds (field complaint: whole-doc restyles/deletes/ + // replaces and undo/redo "freeze for a little, especially on collab"). + // Each round times the RECEIVER's import of one mass chunk, or the local + // undo AFTER remote chatter (the recorded-inverse staleness case). --------- + let mut import_peer_chunk = |label: &str| { + let update = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .expect("peer gate"); + let update = guard + .doc() + .export(loro::ExportMode::updates(&peer_handle_vv)) + .expect("peer export"); + peer_handle_vv = guard.doc().state_vv(); + update + }; + let work = instrument::snapshot(); + let scans = block_ix_scan_count(); + let t = Instant::now(); + let mut guard = gate.lock(GateHolder::ImportChunk).expect("gate"); + let lock_ms = t.elapsed().as_secs_f64() * 1000.0; + let t_call = Instant::now(); + let events = guard + .import_remote_update(&update) + .expect("import peer mass chunk"); + let call_ms = t_call.elapsed().as_secs_f64() * 1000.0; + let t_drop = Instant::now(); + drop(events); + drop(guard); + let drop_ms = t_drop.elapsed().as_secs_f64() * 1000.0; + eprintln!( + "{label} {:8.1} ms (one mass import: lock {lock_ms:.1} + call {call_ms:.1} + drop {drop_ms:.1})", + t.elapsed().as_secs_f64() * 1000.0 + ); + report_work(label, work, scans); + }; + + // REMOTE-MASS-RESTYLE: B restyles EVERY paragraph; A imports one chunk. + let peer_paragraphs = peer_handle + .projection() + .expect("peer projection") + .ids + .paragraph_ids + .clone(); + peer_handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: peer_paragraphs.to_vec(), + style: ParagraphStyle::Custom(3), + }) + .expect("peer mass restyle"); + import_peer_chunk("REMOTE-MASS-RESTYLE"); + + // REMOTE-UNDO-MASS: B undoes the mass restyle; A imports the inverse chunk. + peer_handle.apply_undo().expect("peer undo of mass restyle"); + import_peer_chunk("REMOTE-UNDO-MASS "); + + // REMOTE-REPLACE-ALL: B replaces the first char of every non-empty paragraph + // (a same-shape proxy for find&replace-all); A imports one chunk. + { + let peer_projection = peer_handle.projection().expect("peer projection"); + let mut matches = Vec::new(); + let mut paragraph_ix = 0usize; + for block in peer_projection.blocks.iter() { + if let flowstate_document::Block::Paragraph(paragraph) = block { + let id = peer_projection.ids.paragraph_ids[paragraph_ix]; + if paragraph.runs.iter().map(|run| run.len).sum::() >= 3 { + matches.push(ReplaceMatch { + start: TextAnchor::new(id, 0), + end: TextAnchor::new(id, 1), + styles: None, + }); + } + paragraph_ix += 1; + } + } + let match_count = matches.len(); + peer_handle + .replace_matches(ReplaceMatchesIntent { + matches, + replacement: "Q".to_string(), + }) + .expect("peer replace-all"); + eprintln!(" (replace-all touched {match_count} paragraphs)"); + }; + import_peer_chunk("REMOTE-REPLACE-ALL"); + + // REMOTE-MASS-DELETE: B deletes a quarter of the document in one intent; A + // imports one chunk. + { + let peer_projection = peer_handle.projection().expect("peer projection"); + let ids = &peer_projection.ids.paragraph_ids; + let start = TextAnchor::new(ids[ids.len() / 4], 0); + let end = TextAnchor::new(ids[ids.len() / 2], 0); + peer_handle + .delete_range(DeleteRangeIntent { start, end }) + .expect("peer mass delete") + }; + import_peer_chunk("REMOTE-MASS-DELETE"); + + // UNDO-AFTER-REMOTE: the receiver-side staleness case — A commits a mass + // restyle, ONE remote char arrives, then A hits Ctrl-Z. The remote frontier + // advance invalidates the recorded-inverse fast path today, so this measures + // the slow-path cliff the rebase fix must close (gate: compare to the tail + // UNDO, which stays fast-path). + { + let live = handle.projection().expect("projection"); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: live.ids.paragraph_ids.to_vec(), + style: ParagraphStyle::Custom(4), + }) + .expect("mass restyle before remote chatter"); + let peer_first = peer_handle + .projection() + .expect("peer projection") + .ids + .paragraph_ids[0]; + peer_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_first, 0), + text: "r".to_string(), + style_override: None, + }) + .expect("peer chatter keystroke"); + import_peer_chunk("REMOTE-CHATTER "); + let work = instrument::snapshot(); + let scans = block_ix_scan_count(); + let t = Instant::now(); + handle.apply_undo().expect("undo after remote chatter"); + eprintln!( + "UNDO-AFTER-REMOTE {:8.1} ms (select-all restyle: v2 per-coordinate rebase keeps the fast path)", + t.elapsed().as_secs_f64() * 1000.0 + ); + report_work("UNDO-AFTER-REMOTE", work, scans); + }; + + // UNDO-AFTER-REMOTE-PARTIAL (§oom-leads #9): A restyles the BOTTOM half, B's + // chatter lands at the TOP — strictly before the recorded hull, so the + // recorded-inverse REBASE keeps the fast path armed. Gate: this undo must be + // O(change) fast (same order as a no-chatter undo), NOT the slow path. + { + let live = handle.projection().expect("projection"); + let bottom_half: Vec<_> = live + .ids + .paragraph_ids + .iter() + .skip(live.ids.paragraph_ids.len() / 2) + .copied() + .collect(); + handle + .set_paragraph_styles(SetParagraphStylesIntent { + paragraphs: bottom_half, + style: ParagraphStyle::Custom(5), + }) + .expect("partial mass restyle"); + let peer_first = peer_handle + .projection() + .expect("peer projection") + .ids + .paragraph_ids[1]; + peer_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_first, 0), + text: "t".to_string(), + style_override: None, + }) + .expect("peer chatter above hull"); + import_peer_chunk("REMOTE-CHATTER-TOP"); + let work = instrument::snapshot(); + let scans = block_ix_scan_count(); + let t = Instant::now(); + handle.apply_undo().expect("undo after top chatter"); + eprintln!( + "UNDO-AFTER-REMOTE-PARTIAL {:8.1} ms (rebase keeps the fast path)", + t.elapsed().as_secs_f64() * 1000.0 + ); + report_work("UNDO-AFTER-REMOTE-PARTIAL", work, scans); + }; + + // KEYSTROKE-UNDO-AFTER-CHATTER (§oom-leads #9 v2): the single most common + // field freeze — type a char, partner types ABOVE you, Ctrl-Z. The keystroke + // capture + hull-disjoint rebase must keep this O(change); before the + // capture landed this was the uncaptured slow path (~2.6s on this doc with + // dirty history). Gate: same order as a no-chatter undo, zero fallbacks. + { + let live = handle.projection().expect("projection"); + let deep_ix = live.ids.paragraph_ids.len() / 2; + let deep = live.ids.paragraph_ids[deep_ix]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(deep, 0), + text: "k".to_string(), + style_override: None, + }) + .expect("deep keystroke"); + let peer_first = peer_handle + .projection() + .expect("peer projection") + .ids + .paragraph_ids[1]; + peer_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_first, 0), + text: "t".to_string(), + style_override: None, + }) + .expect("peer chatter above hull"); + import_peer_chunk("REMOTE-CHATTER-TOP2"); + let work = instrument::snapshot(); + let scans = block_ix_scan_count(); + let t = Instant::now(); + handle.apply_undo().expect("keystroke undo after chatter"); + eprintln!( + "KEYSTROKE-UNDO-AFTER-CHATTER {:8.1} ms (captured keystroke + rebase keep the fast path)", + t.elapsed().as_secs_f64() * 1000.0 + ); + report_work("KEYSTROKE-UNDO-AFTER-CHATTER", work, scans); + }; + + // REMOTE-STRUCT-BARE: naked '\n' with no record/mark — the docx `w:br` / + // adversarial shape; measures the repair/heal story. Skippable so profiling + // runs can isolate the REAL-split rounds (its per-round cost COMPOUNDS while + // record-less boundaries accumulate un-healed). + if std::env::var_os("FLOWSTATE_BENCH_SKIP_BARE").is_none() { + let bare_work = instrument::snapshot(); + let bare_scans = block_ix_scan_count(); + let mut bare_ms = 0.0; + for _ in 0..rounds { + let len = body_b.len_unicode(); + body_b + .insert(len / 2, "\n") + .expect("bare structural insert"); + doc_b.commit(); + let update = doc_b + .export(loro::ExportMode::updates(&peer_vv)) + .expect("peer export"); + peer_vv = doc_b.state_vv(); + let t = Instant::now(); + gate + .lock(GateHolder::ImportChunk) + .expect("gate") + .import_remote_update(&update) + .expect("import bare structural"); + bare_ms += t.elapsed().as_secs_f64() * 1000.0; + } + eprintln!( + "REMOTE-STRUCT-BARE {:8.1} ms ({rounds} rounds, {:.2} ms/round)", + bare_ms, + bare_ms / rounds as f64 + ); + report_work("REMOTE-STRUCT-BARE", bare_work, bare_scans); + } + } + + // UNDO / REDO the mass restyle. + let undo = Instant::now(); + handle.apply_undo().expect("undo"); + eprintln!("UNDO {:8.1} ms", undo.elapsed().as_secs_f64() * 1000.0); + let redo = Instant::now(); + handle.apply_redo().expect("redo"); + eprintln!("REDO {:8.1} ms", redo.elapsed().as_secs_f64() * 1000.0); + + eprintln!("collab_bench: done — tables below."); +} diff --git a/crates/flowstate-corpus/src/bin/contention_bench.rs b/crates/flowstate-corpus/src/bin/contention_bench.rs new file mode 100644 index 00000000..fd8b4e22 --- /dev/null +++ b/crates/flowstate-corpus/src/bin/contention_bench.rs @@ -0,0 +1,196 @@ +//! Gate-contention bench (G23): measure local keystroke latency SOLO vs while a second +//! thread hammers remote imports through the SAME `WriteGate` — the real "another peer is +//! typing while I type" contention the field logs blamed for ~36 ms keystrokes. +//! +//! Run: `cargo run --release --bin contention_bench -- [KEYSTROKES]`. +//! (No profiler feature — plain wall-clock; contention is a scheduling effect.) + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use flowstate_collab::crdt_runtime::CrdtRuntime; +use flowstate_collab::local_write::{GateHolder, LocalDocHandle, LocalWriteConfig}; +use gpui_flowtext::{InsertTextIntent, TextAnchor}; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(path) = args + .next() + .or_else(|| std::env::var("FLOWSTATE_BENCH_FIXTURE").ok()) + else { + eprintln!("usage: contention_bench [KEYSTROKES]"); + std::process::exit(2); + }; + let keystrokes: usize = args.next().and_then(|v| v.parse().ok()).unwrap_or(200); + + let (projection, report) = flowstate_docx::convert_docx_to_document(&path).expect("convert docx"); + eprintln!("file: {path} ({} paragraphs) keystrokes={keystrokes}", report.paragraphs_imported); + let core = CrdtRuntime::from_document_projection(&projection, "contention").expect("runtime"); + + // Peer B (fork) will stream imports at A during the contended run. + let doc_b = core.doc().fork(); + let doc_b2 = core.doc().fork(); + doc_b.set_peer_id(0x00B0_B0B0).expect("distinct peer id"); + + let (handle, gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let live = handle.projection().expect("projection"); + let first = *live.ids.paragraph_ids.first().expect("a paragraph"); + + let keystroke = |handle: &LocalDocHandle| { + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(first, 0), + text: "z".to_string(), + style_override: None, + }) + .expect("insert_text"); + }; + + // SOLO: keystroke latency with no competing thread. + let solo = measure_latencies(keystrokes, || keystroke(&handle)); + report_lat("SOLO keystroke", &solo); + + // CONTENDED: a background thread hammers remote imports through the same gate while we + // measure keystroke latency on this thread. Both contend on the single write gate. + let stop = Arc::new(AtomicBool::new(false)); + let importer = { + let gate = Arc::clone(&gate); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + let body = flowstate_document::loro_schema::body_text(&doc_b); + let mut vv = doc_b.state_vv(); + let mut imports = 0u64; + while !stop.load(Ordering::Relaxed) { + body.insert(1, "r").expect("peer insert"); + doc_b.commit(); + let update = doc_b + .export(loro::ExportMode::updates(&vv)) + .expect("export"); + vv = doc_b.state_vv(); + if let Ok(mut guard) = gate.lock(GateHolder::ImportChunk) { + let _ = guard.import_remote_update(&update); + imports += 1; + } + } + imports + }) + }; + + let contended = measure_latencies(keystrokes, || keystroke(&handle)); + report_lat("CONTENDED keystroke", &contended); + + // PACED: realistic typing cadence (5ms between keystrokes) so the importer + // actually streams between them — this is the field shape ("keystroke + // waits behind an import hold"), which back-to-back keystrokes mask by + // starving the importer. + let paced = measure_paced_latencies(keystrokes, std::time::Duration::from_millis(5), || keystroke(&handle)); + stop.store(true, Ordering::Relaxed); + let imports = importer.join().expect("importer thread"); + report_lat("PACED keystroke+5ms", &paced); + + // §A14.1 gate: paced typing WHILE a mass op streams in as 2048-atom + // slices (the sender-side chunking shape). Pre-build the slice queue, + // then type paced while a thread feeds them through the gate. + let mass_slices: Vec> = { + let guard = gate.lock(GateHolder::ExportUpdates).expect("gate"); + let from_vv = guard.doc().state_vv(); + drop(guard); + let body = flowstate_document::loro_schema::body_text(&doc_b2); + body + .insert(1, &"mass restyle payload ".repeat(3_000)) + .expect("mass insert"); + doc_b2.commit(); + let peer = doc_b2.peer_id(); + let start = from_vv.get(&peer).copied().unwrap_or(0); + let end = doc_b2.oplog_vv().get(&peer).copied().unwrap_or(start); + (start..end) + .step_by(4096) + .map(|w| { + doc_b2 + .export(loro::ExportMode::updates_in_range(&[loro::IdSpan::new(peer, w, (w + 4096).min(end))])) + .expect("slice export") + }) + .collect() + }; + let slice_count = mass_slices.len(); + let mass_importer = { + let gate = Arc::clone(&gate); + std::thread::spawn(move || { + for slice in mass_slices { + if let Ok(mut guard) = gate.lock(GateHolder::ImportChunk) { + let _ = guard.import_remote_update(&slice); + } + } + }) + }; + let mass_paced = measure_paced_latencies(60, std::time::Duration::from_millis(5), || keystroke(&handle)); + mass_importer.join().expect("mass importer"); + report_lat("PACED during MASS op", &mass_paced); + let metrics = gate.metrics(); + eprintln!( + "mass: slices={slice_count} max_hold(import)={}us max_wait(local)={}us", + metrics.max_hold_micros_import_chunk.load(Ordering::Relaxed), + metrics.max_wait_micros_local_intent.load(Ordering::Relaxed), + ); + let metrics = gate.metrics(); + eprintln!( + "gate: acquisitions={} contended={} max_hold(import)={}us max_hold(local)={}us max_wait(local)={}us", + metrics.acquisitions.load(Ordering::Relaxed), + metrics.contended_acquisitions.load(Ordering::Relaxed), + metrics.max_hold_micros_import_chunk.load(Ordering::Relaxed), + metrics.max_hold_micros_local_intent.load(Ordering::Relaxed), + metrics.max_wait_micros_local_intent.load(Ordering::Relaxed), + ); + + eprintln!( + "\nimports landed during contended run: {imports}\ncontention factor (median contended / median solo): {:.1}x", + if solo.median > 0.0 { contended.median / solo.median } else { 0.0 }, + ); +} + +struct Lat { + median: f64, + p95: f64, + max: f64, +} + +/// Like `measure_latencies`, but sleeps `gap` between ops OUTSIDE the timed +/// region — the importer streams during the gaps (the field shape). +fn measure_paced_latencies(count: usize, gap: std::time::Duration, mut op: impl FnMut()) -> Lat { + let mut samples = Vec::with_capacity(count); + for _ in 0..count { + let t = Instant::now(); + op(); + samples.push(t.elapsed().as_secs_f64() * 1000.0); + std::thread::sleep(gap); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + Lat { + median: samples[samples.len() / 2], + p95: samples[samples.len() * 95 / 100], + max: samples[samples.len() - 1], + } +} + +fn measure_latencies(count: usize, mut op: impl FnMut()) -> Lat { + let mut samples = Vec::with_capacity(count); + for _ in 0..count { + let t = Instant::now(); + op(); + samples.push(t.elapsed().as_secs_f64() * 1000.0); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + Lat { + median: samples[samples.len() / 2], + p95: samples[samples.len() * 95 / 100], + max: samples[samples.len() - 1], + } +} + +fn report_lat(name: &str, lat: &Lat) { + println!( + "{name:<24} median {:6.2} ms p95 {:6.2} ms max {:6.2} ms", + lat.median, lat.p95, lat.max + ); +} diff --git a/crates/flowstate-corpus/src/bin/history_growth_probe.rs b/crates/flowstate-corpus/src/bin/history_growth_probe.rs new file mode 100644 index 00000000..bd152ecb --- /dev/null +++ b/crates/flowstate-corpus/src/bin/history_growth_probe.rs @@ -0,0 +1,191 @@ +//! §oom-leads #1: does document history growth degrade snapshot size and +//! open time without bound? Builds the SAME final content four ways and +//! measures each provenance's snapshot bytes, export time, decode time, and +//! cold first projection: +//! +//! * `fresh` — one bulk transaction (the docx-import shape); +//! * `typed` — one commit per inserted char (a long editing session); +//! * `typed-styled` — as `typed`, plus a run-style mark every 64th op +//! (realistic highlight cadence); +//! * `shallow` — the `typed-styled` doc exported with +//! `ExportMode::shallow_snapshot` at its final frontier (the candidate fix: +//! history GC'd to the frontier). +//! +//! Every leg prints one `HISTORY-GROWTH {json}` line — stable keys, no +//! timestamps — so an agent can diff runs mechanically. Convergence guard: +//! every decoded doc must project to the same body text as `fresh`. +//! +//! Run: `cargo run --release -p flowstate-corpus --bin history_growth_probe [ops]` +//! (default `100_000` ops; pass a smaller count for a smoke run). + +use std::time::Instant; + +use flowstate_document::{MARK_HIGHLIGHT_STYLE, document_from_loro, loro_schema}; +use loro::{ExportMode, LoroDoc, LoroValue}; + +struct Leg { + name: &'static str, + snapshot: Vec, + export_ms: f64, + build_ms: f64, +} + +fn measure_decode(name: &str, snapshot: &[u8], build_ms: f64, export_ms: f64, expected_text: &str) { + let decode_start = Instant::now(); + let doc = LoroDoc::new(); + doc.import(snapshot).expect("snapshot import"); + let decode_ms = decode_start.elapsed().as_secs_f64() * 1e3; + + let project_start = Instant::now(); + let projection = document_from_loro(&doc).expect("cold projection"); + let project_ms = project_start.elapsed().as_secs_f64() * 1e3; + + let text = loro_schema::body_text(&doc).to_string(); + assert_eq!(text, expected_text, "{name}: decoded doc diverged from the fresh content"); + + println!( + "HISTORY-GROWTH {{\"leg\":\"{name}\",\"snapshot_bytes\":{},\"build_ms\":{build_ms:.1},\"export_ms\":{export_ms:.1},\"decode_ms\":{decode_ms:.1},\"cold_project_ms\":{project_ms:.1},\"paragraphs\":{}}}", + snapshot.len(), + projection.paragraphs.len(), + ); +} + +fn main() { + let ops: usize = std::env::args() + .nth(1) + .and_then(|value| value.parse().ok()) + .unwrap_or(100_000); + + // The content everything converges to: `ops` chars, a paragraph break every + // 80 chars (realistic paragraph density). + let char_at = |op_ix: usize| -> char { if op_ix % 80 == 79 { '\n' } else { (b'a' + (op_ix % 26) as u8) as char } }; + + // --- fresh: one bulk insert ------------------------------------------------ + let fresh_start = Instant::now(); + let fresh = loro_schema::new_loro_document("history-growth-fresh").expect("fresh doc"); + let body = loro_schema::body_text(&fresh); + let content: String = (0..ops).map(char_at).collect(); + let base_len = body.len_unicode(); + body.insert(base_len, &content).expect("bulk insert"); + fresh.commit(); + let fresh_build = fresh_start.elapsed().as_secs_f64() * 1e3; + let expected_text = loro_schema::body_text(&fresh).to_string(); + + let export_start = Instant::now(); + let fresh_snapshot = fresh.export(ExportMode::Snapshot).expect("fresh export"); + let fresh_export = export_start.elapsed().as_secs_f64() * 1e3; + + // --- typed / typed-styled: one commit per char ----------------------------- + let mut legs: Vec = vec![Leg { + name: "fresh", + snapshot: fresh_snapshot, + export_ms: fresh_export, + build_ms: fresh_build, + }]; + let mut styled_final_frontiers = None; + for styled in [false, true] { + let build_start = Instant::now(); + let doc = loro_schema::new_loro_document(if styled { "history-growth-styled" } else { "history-growth-typed" }).expect("typed doc"); + let text = loro_schema::body_text(&doc); + let base = text.len_unicode(); + for op_ix in 0..ops { + text + .insert(base + op_ix, &char_at(op_ix).to_string()) + .expect("typed insert"); + if styled && op_ix % 64 == 63 { + let start = base + op_ix.saturating_sub(40); + text + .mark(start..base + op_ix, MARK_HIGHLIGHT_STYLE, LoroValue::I64(1)) + .expect("mark"); + } + doc.commit(); + } + let build_ms = build_start.elapsed().as_secs_f64() * 1e3; + let export_start = Instant::now(); + let snapshot = doc.export(ExportMode::Snapshot).expect("typed export"); + let export_ms = export_start.elapsed().as_secs_f64() * 1e3; + if styled { + styled_final_frontiers = Some((doc.state_frontiers(), doc)); + legs.push(Leg { + name: "typed-styled", + snapshot, + export_ms, + build_ms, + }); + } else { + legs.push(Leg { + name: "typed", + snapshot, + export_ms, + build_ms, + }); + } + } + + // --- churn: steady-state length, content REWRITTEN 5x (tombstone mass) ---- + // The realistic long-lived-doc shape: blocks get replaced, not appended. + // Total inserted chars = 5 * ops, 80% deleted along the way; final visible + // content differs from `expected_text` so it gets its own convergence check. + churn_legs(ops, &char_at); + // --- shallow: the typed-styled doc, history GC'd to the final frontier ----- + let (frontiers, styled_doc) = styled_final_frontiers.expect("styled leg ran"); + let export_start = Instant::now(); + let shallow_snapshot = styled_doc + .export(ExportMode::shallow_snapshot(&frontiers)) + .expect("shallow export"); + let shallow_export = export_start.elapsed().as_secs_f64() * 1e3; + legs.push(Leg { + name: "shallow", + snapshot: shallow_snapshot, + export_ms: shallow_export, + build_ms: 0.0, + }); + + // NOTE: `typed`/`typed-styled` body text includes whatever sentinel the + // schema seeds — identical across legs by construction. The `fresh` text is + // the reference; styled legs carry marks (same characters). + for leg in &legs { + measure_decode(leg.name, &leg.snapshot, leg.build_ms, leg.export_ms, &expected_text); + } +} + +fn churn_legs(ops: usize, char_at: &dyn Fn(usize) -> char) { + let build_start = Instant::now(); + let doc = loro_schema::new_loro_document("history-growth-churn").expect("churn doc"); + let text = loro_schema::body_text(&doc); + let base = text.len_unicode(); + let mut churn_ops = 0usize; + while churn_ops < ops * 5 { + // Type an 80-char paragraph... + let cursor = text.len_unicode(); + for op_ix in 0..80 { + text + .insert(cursor + op_ix, &char_at(churn_ops + op_ix).to_string()) + .expect("churn insert"); + doc.commit(); + } + churn_ops += 80; + // ...then delete 64 of its chars 4 paragraphs back (rewrite churn), + // keeping the doc length roughly steady at ~5x80 visible chars. + let len = text.len_unicode(); + if len > base + 400 { + let del_start = base + ((churn_ops / 80) * 13) % (len - base - 80); + text.delete(del_start, 64).expect("churn delete"); + doc.commit(); + churn_ops += 1; + } + } + let build_ms = build_start.elapsed().as_secs_f64() * 1e3; + let export_start = Instant::now(); + let snapshot = doc.export(ExportMode::Snapshot).expect("churn export"); + let export_ms = export_start.elapsed().as_secs_f64() * 1e3; + let churn_text = loro_schema::body_text(&doc).to_string(); + let frontiers = doc.state_frontiers(); + let shallow_start = Instant::now(); + let churn_shallow = doc + .export(ExportMode::shallow_snapshot(&frontiers)) + .expect("churn shallow export"); + let churn_shallow_export = shallow_start.elapsed().as_secs_f64() * 1e3; + measure_decode("churn", &snapshot, build_ms, export_ms, &churn_text); + measure_decode("churn-shallow", &churn_shallow, 0.0, churn_shallow_export, &churn_text); +} diff --git a/crates/flowstate-corpus/src/bin/hotpath_bench.rs b/crates/flowstate-corpus/src/bin/hotpath_bench.rs new file mode 100644 index 00000000..baf59d1d --- /dev/null +++ b/crates/flowstate-corpus/src/bin/hotpath_bench.rs @@ -0,0 +1,102 @@ +//! Headless profiler bench — the same hotpath timing/alloc/cpu tables `cargo survey` +//! dumps for the app, but driven deterministically over a real `.docx` with no GUI. +//! +//! It reproduces the field "reopen / receiving-peer" condition (the body is UNDECODED +//! at projection time) via the snapshot round-trip trick, then cold-projects N times. +//! The `#[hotpath::measure]` annotations already in `flowstate-document` / +//! `flowstate-docx` (`object_blocks_for_flow`, `projector_body_to_delta`, +//! `import_docx_to_loro`, `paragraph_ids_by_boundary`, …) populate the tables; the +//! `hotpath::main` macro dumps them on exit. +//! +//! Run (release, pick one profiler axis) — e.g. +//! `cargo run --release --features hotpath-alloc --bin hotpath_bench -- [ITERS]`: +//! - `hotpath` → timing table, +//! - `hotpath-cpu` → + cpu attribution, +//! - `hotpath-alloc` → + alloc-bytes per fn. +//! +//! Without any feature it is a plain timed run (no tables) — still useful. + +use std::alloc::{GlobalAlloc, Layout}; + +struct FlowstateAllocator; + +impl Default for FlowstateAllocator { + fn default() -> Self { + Self + } +} + +// SAFETY: every call forwards its arguments unchanged to mimalloc, which upholds the +// `GlobalAlloc` contract; this wrapper adds no behavior of its own. +unsafe impl GlobalAlloc for FlowstateAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: the caller upholds `GlobalAlloc::alloc`'s layout contract; the layout + // is forwarded to mimalloc unchanged. + unsafe { mimalloc::MiMalloc.alloc(layout) } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: the caller guarantees `ptr`/`layout` came from this allocator; both + // are forwarded to mimalloc unchanged. + unsafe { mimalloc::MiMalloc.dealloc(ptr, layout) } + } +} + +// When alloc-profiling is on, hotpath installs its own tracking allocator (wrapping +// this one); otherwise this is the process global allocator, matching the app. +#[cfg(not(feature = "hotpath-alloc"))] +#[global_allocator] +static GLOBAL: FlowstateAllocator = FlowstateAllocator; + +#[hotpath::main(allocator = FlowstateAllocator)] +fn main() { + let mut args = std::env::args().skip(1); + let Some(path) = args + .next() + .or_else(|| std::env::var("FLOWSTATE_BENCH_FIXTURE").ok()) + else { + eprintln!( + "usage: hotpath_bench :> [ITERS] (or set FLOWSTATE_BENCH_FIXTURE)\n \ + synthetic kinds: equations (default), mixed, text — e.g. synthetic:equations:9000" + ); + std::process::exit(2); + }; + let iters: usize = args.next().and_then(|v| v.parse().ok()).unwrap_or(3); + + eprintln!("hotpath_bench: {path} (cold-project x{iters})"); + + // IMPORT: source -> Loro doc. A `synthetic:*` selector builds an equation/object + // fixture (§perf-heaven T7 — no equation docs in the corpus); otherwise docx bytes + // through the interpreter (whose #[hotpath::measure] fns fire). + let doc = if let Some((projection, count)) = flowstate_corpus::fixtures::from_selector(&path) { + eprintln!(" synthetic fixture: {count} paragraphs, {} blocks", projection.blocks.len()); + flowstate_collab::crdt_runtime::CrdtRuntime::from_document_projection(&projection, "hotpath-bench") + .expect("runtime") + .doc() + .clone() + } else { + let (imported, report) = flowstate_docx::import_docx_to_loro(&path, "hotpath-bench").expect("import docx"); + eprintln!(" imported {} paragraphs", report.paragraphs_imported); + imported.doc + }; + let snapshot = doc.export(loro::ExportMode::Snapshot).expect("snapshot"); + + // WARM baseline: project the already-decoded doc once (cheap; for contrast). + let _ = flowstate_document::document_from_loro(&doc).expect("warm project"); + + // COLD: each iteration re-imports the snapshot into a FRESH (lazy/undecoded) doc + // and projects it — the field reopen condition where the body decode + the + // object_blocks / body-to-delta scan family dominate. This is what the tables sample. + for iteration in 0..iters.max(1) { + let doc = loro::LoroDoc::new(); + doc.import(&snapshot).expect("reimport"); + let projection = flowstate_document::document_from_loro(&doc).expect("cold project"); + if iteration == 0 { + eprintln!( + " projection: {} paragraphs, {} blocks", + projection.paragraphs.len(), + projection.blocks.len() + ); + } + } + eprintln!("hotpath_bench: done — tables below."); +} diff --git a/crates/flowstate-corpus/src/bin/open_probe.rs b/crates/flowstate-corpus/src/bin/open_probe.rs new file mode 100644 index 00000000..3334c988 --- /dev/null +++ b/crates/flowstate-corpus/src/bin/open_probe.rs @@ -0,0 +1,67 @@ +//! §act-twelve A12.1.0: the `.db8` cold-open stage probe. Converts a docx +//! once, checkpoints it as a package, then times a fresh `open_package` +//! (set `FLOWSTATE_OPEN_PROBE=1` for the in-runtime stage prints). +//! +//! Usage: `FLOWSTATE_OPEN_PROBE=1 open_probe [N_OPENS]` + +use flowstate_collab::crdt_runtime::CrdtRuntime; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(path) = args + .next() + .or_else(|| std::env::var("FLOWSTATE_BENCH_FIXTURE").ok()) + else { + eprintln!("usage: open_probe [N_OPENS] (or set FLOWSTATE_BENCH_FIXTURE)"); + std::process::exit(2); + }; + let opens: usize = args.next().and_then(|v| v.parse().ok()).unwrap_or(3); + let dir = tempfile::tempdir().expect("tempdir"); + let db8_path = match std::env::var_os("FLOWSTATE_KEEP_DB8") { + Some(p) => std::path::PathBuf::from(p), + None => dir.path().join("open-probe.db8"), + }; + + let t0 = std::time::Instant::now(); + let (projection, _report) = flowstate_docx::convert_docx_to_document(&path).expect("convert docx"); + eprintln!("[open-probe] docx convert: {:?}", t0.elapsed()); + let t1 = std::time::Instant::now(); + let mut runtime = CrdtRuntime::from_document_projection(&projection, "open-probe").expect("runtime"); + eprintln!("[open-probe] CRDT import: {:?}", t1.elapsed()); + let t2 = std::time::Instant::now(); + runtime + .checkpoint_package("open-probe", Some(db8_path.clone())) + .expect("checkpoint"); + eprintln!("[open-probe] checkpoint (.db8 write): {:?}", t2.elapsed()); + drop(runtime); + + let mut last = None; + for i in 0..opens { + let t = std::time::Instant::now(); + let reopened = CrdtRuntime::open_package(&db8_path).expect("open package"); + let open_time = t.elapsed(); + let paragraphs = reopened + .projection_snapshot() + .expect("projection") + .paragraphs + .len(); + eprintln!("[open-probe] OPEN #{i}: {open_time:?} ({paragraphs} paragraphs)"); + last = Some(reopened); + } + + // §A13.4.0: time a ROUTINE checkpoint (the split off-thread job path) on + // the reopened runtime — the shape autosave/explicit-save actually run. + if let Some(mut runtime) = last { + let t = std::time::Instant::now(); + let (job, _events) = runtime + .begin_checkpoint("open-probe-routine", Some(db8_path.clone())) + .expect("begin checkpoint") + .expect("package present"); + let begin_time = t.elapsed(); + let t = std::time::Instant::now(); + let (package, wrote) = job.run(); + wrote.expect("checkpoint job"); + runtime.finish_checkpoint(package, true); + eprintln!("[open-probe] routine checkpoint: begin(gate)={begin_time:?} job={:?}", t.elapsed()); + } +} diff --git a/crates/flowstate-corpus/src/bin/perf_probe.rs b/crates/flowstate-corpus/src/bin/perf_probe.rs new file mode 100644 index 00000000..284cdf5c --- /dev/null +++ b/crates/flowstate-corpus/src/bin/perf_probe.rs @@ -0,0 +1,121 @@ +//! Perf probe: isolate WHERE the time goes when opening a large real `.docx`. +//! +//! Usage: `flowstate-perf-probe [ITERS]` (default 5 iters). +//! +//! Phases, each timed: +//! IMPORT — `.docx` bytes -> Loro doc (the interpreter). +//! PROJECT-warm — `document_from_loro` on the just-imported (state already decoded) doc. +//! SNAPSHOT — export a Loro snapshot (what a `.db8`/reopen carries). +//! PROJECT-cold — import the snapshot into a FRESH doc (lazy/undecoded richtext, the +//! real reopen/receiving-peer condition) and project it. This is the +//! hypothesis-A path where `query_text_id_positions` decodes the whole +//! body per projection. +//! COLD->WARM — on ONE fresh import, project twice: does the decode PERSIST (2nd +//! cheap) or re-decode every projection (2nd still expensive)? + +use std::time::Instant; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(path) = args + .next() + .or_else(|| std::env::var("FLOWSTATE_BENCH_FIXTURE").ok()) + else { + eprintln!("usage: flowstate-perf-probe [ITERS] (or set FLOWSTATE_BENCH_FIXTURE)"); + std::process::exit(2); + }; + let iters: usize = args.next().and_then(|v| v.parse().ok()).unwrap_or(5); + + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) => { + eprintln!("read {path}: {error}"); + std::process::exit(1); + }, + }; + println!("file: {path}\nsize: {:.2} MB iters: {iters}\n", bytes.len() as f64 / 1_048_576.0); + + // IMPORT (once; keep the decoded doc for the warm + snapshot phases). + let t = Instant::now(); + let (imported, report) = match flowstate_docx::import_docx_to_loro(&path, "perf-probe") { + Ok(pair) => pair, + Err(error) => { + eprintln!("import failed: {error}"); + std::process::exit(1); + }, + }; + let import_ms = t.elapsed().as_secs_f64() * 1000.0; + println!( + "IMPORT {import_ms:8.1} ms (paragraphs_imported={})", + report.paragraphs_imported + ); + + // PROJECT-warm: the imported doc's richtext state is already materialized. + let warm = time_median(iters, || { + let _ = flowstate_document::document_from_loro(&imported.doc).expect("project"); + }); + report_phase("PROJECT-warm", &warm); + + // SNAPSHOT + let t = Instant::now(); + let snapshot = imported + .doc + .export(loro::ExportMode::Snapshot) + .expect("snapshot"); + println!( + "SNAPSHOT {:8.1} ms ({:.2} MB)", + t.elapsed().as_secs_f64() * 1000.0, + snapshot.len() as f64 / 1_048_576.0 + ); + + // PROJECT-cold: fresh (undecoded) import each iter — the reopen / receiving-peer path. + let cold = time_median(iters, || { + let doc = loro::LoroDoc::new(); + doc.import(&snapshot).expect("reimport"); + let _ = flowstate_document::document_from_loro(&doc).expect("project cold"); + }); + report_phase("PROJECT-cold", &cold); + + // COLD->WARM on a single fresh instance: does the first projection's decode persist? + let cold_warm_doc = loro::LoroDoc::new(); + cold_warm_doc.import(&snapshot).expect("reimport"); + let t = Instant::now(); + let _ = flowstate_document::document_from_loro(&cold_warm_doc).expect("project"); + let first = t.elapsed().as_secs_f64() * 1000.0; + let t = Instant::now(); + let _ = flowstate_document::document_from_loro(&cold_warm_doc).expect("project"); + let second = t.elapsed().as_secs_f64() * 1000.0; + println!("COLD->WARM first={first:.1} ms second={second:.1} ms (second< decode persists; else re-decodes every projection)"); + + println!( + "\nsummary: import {import_ms:.0}ms | cold-project {:.0}ms | warm-project {:.0}ms | cold/warm ratio {:.1}x", + cold.median, + warm.median, + if warm.median > 0.0 { cold.median / warm.median } else { 0.0 }, + ); +} + +struct Stats { + min: f64, + median: f64, + max: f64, +} + +fn time_median(iters: usize, mut run: impl FnMut()) -> Stats { + let mut samples: Vec = Vec::with_capacity(iters.max(1)); + for _ in 0..iters.max(1) { + let t = Instant::now(); + run(); + samples.push(t.elapsed().as_secs_f64() * 1000.0); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + Stats { + min: samples[0], + median: samples[samples.len() / 2], + max: samples[samples.len() - 1], + } +} + +fn report_phase(name: &str, stats: &Stats) { + println!("{name:<15} {:8.1} ms (min {:.1}, max {:.1})", stats.median, stats.min, stats.max); +} diff --git a/crates/flowstate-corpus/src/bin/screenshot_compare.rs b/crates/flowstate-corpus/src/bin/screenshot_compare.rs new file mode 100644 index 00000000..8a756e34 --- /dev/null +++ b/crates/flowstate-corpus/src/bin/screenshot_compare.rs @@ -0,0 +1,105 @@ +//! §act-eleven / A11.6 net: tolerance comparison of two raster screenshots. +//! `screenshot_compare ` exits 0 when the current +//! raster matches the golden within tolerance, 1 on divergence (with a report +//! and a diff heatmap written next to the current image), 2 on usage/shape +//! errors. +//! +//! Tolerances (env-tunable): antialiasing and subpixel text render slightly +//! differently across driver/font updates, so the gate is two-tier — +//! `FLOWSTATE_SCREENSHOT_MEAN_TOLERANCE` (mean absolute channel delta, +//! default 1.5/255) catches broad drift; `FLOWSTATE_SCREENSHOT_HOT_FRACTION` +//! (fraction of pixels whose max channel delta exceeds 32, default 0.5%) +//! catches localized breakage (a missing glyph run, a wrong quad) that a +//! whole-screen mean would dilute. + +use std::process::ExitCode; + +fn main() -> ExitCode { + let mut args = std::env::args().skip(1); + let (Some(golden_path), Some(current_path)) = (args.next(), args.next()) else { + eprintln!("usage: screenshot_compare "); + return ExitCode::from(2); + }; + let golden = match image::open(&golden_path) { + Ok(image) => image.into_rgba8(), + Err(error) => { + eprintln!("screenshot_compare: cannot open golden {golden_path}: {error}"); + return ExitCode::from(2); + }, + }; + let current = match image::open(¤t_path) { + Ok(image) => image.into_rgba8(), + Err(error) => { + eprintln!("screenshot_compare: cannot open current {current_path}: {error}"); + return ExitCode::from(2); + }, + }; + if golden.dimensions() != current.dimensions() { + eprintln!( + "screenshot_compare: dimensions differ — golden {:?} vs current {:?} (display mode changed? re-baseline with `heaven.sh screenshot-baseline`)", + golden.dimensions(), + current.dimensions() + ); + return ExitCode::from(2); + } + + let mean_tolerance: f64 = std::env::var("FLOWSTATE_SCREENSHOT_MEAN_TOLERANCE") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(1.5); + let hot_fraction_tolerance: f64 = std::env::var("FLOWSTATE_SCREENSHOT_HOT_FRACTION") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(0.005); + const HOT_DELTA: u8 = 32; + + let (width, height) = golden.dimensions(); + let mut total_delta: u64 = 0; + let mut hot_pixels: u64 = 0; + let mut heatmap = image::GrayImage::new(width, height); + for (golden_pixel, current_pixel, heat) in itertools_free_zip(&golden, ¤t, &mut heatmap) { + let mut max_channel_delta = 0u8; + let mut pixel_delta = 0u64; + for channel in 0..3 { + let delta = golden_pixel.0[channel].abs_diff(current_pixel.0[channel]); + pixel_delta += u64::from(delta); + max_channel_delta = max_channel_delta.max(delta); + } + total_delta += pixel_delta; + if max_channel_delta > HOT_DELTA { + hot_pixels += 1; + } + heat.0[0] = max_channel_delta.saturating_mul(4); + } + let pixel_count = u64::from(width) * u64::from(height); + let mean = total_delta as f64 / (pixel_count as f64 * 3.0); + let hot_fraction = hot_pixels as f64 / pixel_count as f64; + eprintln!( + "screenshot_compare: {width}x{height} mean_delta={mean:.3} (tolerance {mean_tolerance}) hot_pixels={hot_pixels} ({:.4}%, tolerance {:.4}%)", + hot_fraction * 100.0, + hot_fraction_tolerance * 100.0 + ); + if mean <= mean_tolerance && hot_fraction <= hot_fraction_tolerance { + return ExitCode::SUCCESS; + } + let heat_path = format!("{current_path}.diff.png"); + if heatmap.save(&heat_path).is_ok() { + eprintln!("screenshot_compare: FAILED — diff heatmap written to {heat_path}"); + } else { + eprintln!("screenshot_compare: FAILED"); + } + ExitCode::from(1) +} + +/// Zip three same-dimension images pixel-wise without extra deps. +fn itertools_free_zip<'imgs>( + golden: &'imgs image::RgbaImage, + current: &'imgs image::RgbaImage, + heatmap: &'imgs mut image::GrayImage, +) -> impl Iterator, &'imgs image::Rgba, &'imgs mut image::Luma)> { + golden + .pixels() + .zip(current.pixels()) + .zip(heatmap.pixels_mut()) + .map(|((golden_pixel, current_pixel), heat)| (golden_pixel, current_pixel, heat)) +} diff --git a/crates/flowstate-corpus/src/bin/screenshot_probe.rs b/crates/flowstate-corpus/src/bin/screenshot_probe.rs new file mode 100644 index 00000000..df18dac1 --- /dev/null +++ b/crates/flowstate-corpus/src/bin/screenshot_probe.rs @@ -0,0 +1,115 @@ +//! §act-eleven / A11.6 net: the raster screenshot probe. Opens a REAL gpui +//! window rendering a deterministic `RichTextEditor` fixture (headings, +//! cite/highlight runs, soft breaks, a table, an equation), pumps frames for a +//! settle interval, then exits. Run under Xvfb with `-fbdir` so the X server's +//! framebuffer FILE is the captured raster — headless, no compositor chrome, +//! no screenshot tooling: +//! +//! ```sh +//! env -u WAYLAND_DISPLAY xvfb-run -a -s "-screen 0 1200x900x24 -fbdir /tmp/fb" \ +//! ./target/release/screenshot_probe +//! ``` +//! +//! `heaven.sh screenshot` wraps this and compares the framebuffer against the +//! machine-local golden (fonts and scale are machine state; the golden lives in +//! `HEAVEN_DIR`, like the hotpath baselines). + +use gpui::{App, AppContext as _, Application, Bounds, WindowBounds, WindowOptions, px, size}; +use gpui_flowtext::{DocumentTheme, InputParagraph, InputRun, ParagraphStyle, RichTextEditor, RunSemanticStyle, RunStyles, document_from_input}; + +fn run(text: &str, styles: RunStyles) -> InputRun { + InputRun { + text: text.to_string(), + styles, + } +} + +fn cite() -> RunStyles { + RunStyles { + semantic: RunSemanticStyle::Custom(2), + ..RunStyles::default() + } +} + +fn underlined() -> RunStyles { + RunStyles { + direct_underline: true, + ..RunStyles::default() + } +} + +/// A paint-diverse fixture: every glyph path the editor paints on a normal +/// document — heading styles, plain/cite/underline/strikethrough runs, a soft +/// line break, CJK + combining marks + emoji-free wide chars (deterministic +/// shaping), and enough paragraphs to fill the viewport. +fn fixture() -> gpui_flowtext::DocumentProjection { + let mut paragraphs = vec![ + InputParagraph { + style: ParagraphStyle::Custom(1), + runs: vec![run("Screenshot Probe — Pocket", RunStyles::default())], + }, + InputParagraph { + style: ParagraphStyle::Custom(3), + runs: vec![run("Hat: deterministic paint fixture", RunStyles::default())], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![ + run("Plain lead-in, then a ", RunStyles::default()), + run("cited span", cite()), + run(" and an ", RunStyles::default()), + run("underlined tail", underlined()), + run(". Soft\u{2028}break line two.", RunStyles::default()), + ], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![run("宽字符 shaping row — 中文测试 mixed with Latin.", RunStyles::default())], + }, + ]; + for filler in 0..14 { + paragraphs.push(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![ + run( + &format!("Filler paragraph {filler:02} with steady content so the viewport is fully covered "), + RunStyles::default(), + ), + run("and a styled span per row.", if filler % 2 == 0 { cite() } else { underlined() }), + ], + }); + } + document_from_input(DocumentTheme::default(), paragraphs) +} + +fn main() { + let settle_ms: u64 = std::env::var("FLOWSTATE_SCREENSHOT_SETTLE_MS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(2500); + Application::new().run(move |cx: &mut App| { + gpui_component::init(cx); + let bounds = Bounds::centered(None, size(px(1160.0), px(860.0)), cx); + let window = cx + .open_window( + WindowOptions { + // FULLSCREEN: the capture is a whole-screen portal screenshot, so the + // probe must BE the whole screen — no compositor placement variance, + // no decorations, nothing else visible. + window_bounds: Some(WindowBounds::Fullscreen(bounds)), + ..Default::default() + }, + |_window, cx| cx.new(|cx| RichTextEditor::new_with_path(fixture(), None, cx)), + ) + .expect("screenshot probe window"); + // Keep painting for the settle interval (font load + first layout), then quit. + cx.spawn(async move |cx| { + cx.background_executor() + .timer(std::time::Duration::from_millis(settle_ms)) + .await; + let _ = window; + let _ = cx.update(|cx| cx.quit()); + }) + .detach(); + }); +} diff --git a/crates/flowstate-corpus/src/fixtures.rs b/crates/flowstate-corpus/src/fixtures.rs new file mode 100644 index 00000000..5e1043f2 --- /dev/null +++ b/crates/flowstate-corpus/src/fixtures.rs @@ -0,0 +1,243 @@ +//! Synthetic perf fixtures for document shapes the real corpus does not cover. +//! +//! §perf-heaven T7: the 15k-document debate corpus has ZERO equation documents +//! (debate is pure carded text), so the equation projection / keystroke cost +//! (F17–F21) is entirely un-measured. These builders synthesize large equation- +//! and mixed-object-bearing documents so `collab_bench` / `hotpath_bench` can +//! measure them WITHOUT needing a real `.docx` — the doctrine's "build the +//! fixture first, measure, THEN cut; never optimize equations blind." +//! +//! They also give the object-bearing shape the object-free corpus never +//! triggers: a NON-empty `object_blocks_for_flow` candidate set (the §perf-heaven +//! T2/T4 paths) and `blocks.len() != paragraphs.len()`. + +use flowstate_document::{ + AssetId, DocumentProjection, DocumentTheme, InputBlock, InputBlockAlignment, InputEquationBlock, InputEquationDisplay, InputEquationSyntax, + InputImageBlock, InputImageSizing, InputParagraph, InputRun, ParagraphStyle, RunStyles, document_from_input_blocks, +}; + +fn paragraph(ix: usize) -> InputBlock { + InputBlock::Paragraph(InputParagraph { + style: if ix.is_multiple_of(40) { + ParagraphStyle::Custom(2) + } else { + ParagraphStyle::Normal + }, + runs: vec![InputRun { + text: format!("Paragraph {ix}: carded evidence body with enough words to shape and reflow across the column."), + styles: RunStyles::default(), + }], + }) +} + +fn equation(ix: usize) -> InputBlock { + InputBlock::Equation(InputEquationBlock { + source: format!("\\sum_{{i=0}}^{{{ix}}} x_i^2 = \\frac{{{ix}}}{{2}}\\pi + \\alpha_{{{ix}}}"), + syntax: InputEquationSyntax::Latex, + display: InputEquationDisplay::Display, + }) +} + +fn image(ix: usize) -> InputBlock { + InputBlock::Image(InputImageBlock { + asset_id: AssetId(1), + alt_text: format!("figure {ix}"), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Left, + external_url: None, + }) +} + +/// A large document with an equation block after every `equation_every` +/// paragraphs — the equation-heavy shape the corpus lacks. `equation_every == 0` +/// disables equations (degenerates to a plain-text doc). +#[must_use] +pub fn synthetic_equation_document(paragraphs: usize, equation_every: usize) -> DocumentProjection { + let extra = if equation_every == 0 { 0 } else { paragraphs / equation_every.max(1) }; + let mut blocks = Vec::with_capacity(paragraphs + extra + 1); + for ix in 0..paragraphs { + blocks.push(paragraph(ix)); + if equation_every > 0 && ix % equation_every == equation_every - 1 { + blocks.push(equation(ix)); + } + } + document_from_input_blocks(DocumentTheme::default(), blocks) +} + +/// A large document interleaving equations, images, and text — the mixed-object +/// shape (a non-empty `object_blocks_for_flow` candidate set, the T4 path the +/// object-free corpus never exercises). +#[must_use] +pub fn synthetic_mixed_object_document(paragraphs: usize) -> DocumentProjection { + let mut blocks = Vec::with_capacity(paragraphs + paragraphs / 4 + 1); + for ix in 0..paragraphs { + blocks.push(paragraph(ix)); + match ix % 12 { + 5 => blocks.push(equation(ix)), + 9 => blocks.push(image(ix)), + _ => {}, + } + } + document_from_input_blocks(DocumentTheme::default(), blocks) +} + +/// Parse a `synthetic::` selector into a projection and its +/// paragraph count, or return `None` if `selector` is not a synthetic spec (so +/// the caller falls through to a real `.docx` path). Kinds: `equations` +/// (default), `mixed`, `text`. +#[must_use] +pub fn from_selector(selector: &str) -> Option<(DocumentProjection, usize)> { + let spec = selector.strip_prefix("synthetic:")?; + let mut parts = spec.split(':'); + let kind = parts.next().unwrap_or("equations"); + let paragraphs: usize = parts + .next() + .and_then(|value| value.parse().ok()) + .unwrap_or(4000); + let projection = match kind { + "mixed" => synthetic_mixed_object_document(paragraphs), + "text" => synthetic_equation_document(paragraphs, 0), + _ => synthetic_equation_document(paragraphs, 8), + }; + let count = projection.paragraphs.len(); + Some((projection, count)) +} + +#[cfg(test)] +mod tests { + use super::*; + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_document::{Block, document_from_loro_with_defects}; + + fn equation_block_count(projection: &DocumentProjection) -> usize { + projection + .blocks + .iter() + .filter(|block| matches!(block, Block::Equation(_))) + .count() + } + + /// §perf-heaven T7 fidelity lock: the synthetic equation fixture must survive + /// the COLD reopen condition (snapshot -> reimport into a fresh undecoded doc + /// -> reproject) with NO DATA LOSS — every equation block round-trips. This + /// closes the F17–F21 projection-fidelity measurement gap the object-free + /// corpus leaves, and (because the reprojection lands on a `LazyLoad::Src` + /// state) it also exercises the §perf-heaven T1 richtext fast-path equivalence + /// assert in debug builds. + /// + /// The synthetic `document_from_input_blocks` construction anchors a standalone + /// object block without minting the durable record for the boundary-less + /// paragraph that follows it, so the cold projection reports REPAIRABLE + /// `missing_paragraph_metadata`/`missing_paragraph_block` defects (the runtime + /// repair pass fixes these canonically — this is exactly what the defect system + /// is for). Those are tolerated; any CONTENT-affecting defect (orphan object, + /// colliding anchor, table topology, invalid asset) is a hard failure. + #[test] + fn synthetic_equation_document_cold_projects_preserving_equations() { + let projection = synthetic_equation_document(200, 8); + let expected_equations = equation_block_count(&projection); + assert!(expected_equations > 0, "fixture must contain equation blocks"); + + let core = CrdtRuntime::from_document_projection(&projection, "eqn-fixture").expect("runtime"); + let snapshot = core + .doc() + .export(loro::ExportMode::Snapshot) + .expect("snapshot"); + let cold = loro::LoroDoc::new(); + cold.import(&snapshot).expect("reimport"); + let (reprojected, defects) = document_from_loro_with_defects(&cold).expect("cold project"); + + let non_repairable: Vec<_> = defects + .iter() + .filter(|defect| !matches!(defect.class(), "missing_paragraph_metadata" | "missing_paragraph_block")) + .collect(); + assert!( + non_repairable.is_empty(), + "equation cold projection lost/corrupted content: {non_repairable:?}" + ); + assert_eq!( + equation_block_count(&reprojected), + expected_equations, + "equation blocks lost across cold projection", + ); + } + + /// §perf-heaven T1: a large OBJECT-FREE doc is the case the fast path actually + /// engages — `object_blocks_for_flow` early-returns (no `to_string`/`char_at` + /// forcing `Dst`), and the O(1) `Src` `len_unicode` keeps the state lazy, so the + /// cold projection's `to_delta` resolves the value from `Src`. In a debug build + /// the equivalence assert fires on that path; this test cold-projects a pure-text + /// doc and asserts text is preserved (and, in debug, that fast == full state). + #[test] + fn object_free_document_cold_projects_via_src_fastpath() { + let projection = synthetic_equation_document(300, 0); // equation_every=0 → pure text + assert_eq!(equation_block_count(&projection), 0, "must be object-free"); + let expected_text = projection.text.to_string(); + + let core = CrdtRuntime::from_document_projection(&projection, "text-fixture").expect("runtime"); + let snapshot = core + .doc() + .export(loro::ExportMode::Snapshot) + .expect("snapshot"); + let cold = loro::LoroDoc::new(); + cold.import(&snapshot).expect("reimport"); + let (reprojection, defects) = document_from_loro_with_defects(&cold).expect("cold project"); + + assert!(defects.is_empty(), "object-free cold projection reported defects: {defects:?}"); + assert_eq!( + reprojection.text.to_string(), + expected_text, + "cold projection text differs from the source" + ); + } + + /// §perf-heaven T7.21 NET: an equation exports as its OWN `` (`m:oMath`), + /// so before the bare-object-wrapper collapse covered equations, reimporting + /// that `` minted a spurious empty paragraph (+1 `\n` boundary) PER + /// equation. This round-trips the fixture through the REAL docx exporter + + /// importer and asserts the equation count survives AND the paragraph count + /// does not grow. Proven-trippable: reverting the equation half of the collapse + /// (`has_objects` → images only) makes the paragraph count grow by the equation + /// count and this fails. + #[test] + fn synthetic_equation_document_survives_docx_roundtrip() { + let mut projection = synthetic_equation_document(60, 8); + // The fixture builds with `DocumentTheme::default()`, which has no custom + // paragraph-style slots; the exporter requires them (the fixture uses + // `Custom(2)`). Use the real theme so export exercises the actual path. + projection.theme = flowstate_document::flowstate_document_theme(); + let expected_equations = equation_block_count(&projection); + let expected_paragraphs = projection.paragraphs.len(); + assert!(expected_equations > 0, "fixture must contain equations"); + + let tmp = std::env::temp_dir().join(format!("fs-eqn-rt-{}.docx", std::process::id())); + flowstate_docx::write_docx(&tmp, &projection).expect("write_docx"); + let bytes = std::fs::read(&tmp).expect("reread docx"); + let _ = std::fs::remove_file(&tmp); + let (imported, _report) = flowstate_docx::import_docx_bytes_to_loro(&bytes, "eqn-rt").expect("reimport"); + let (reprojection, _defects) = document_from_loro_with_defects(&imported.doc).expect("reproject"); + + assert_eq!( + equation_block_count(&reprojection), + expected_equations, + "equation count changed across the docx round-trip", + ); + assert_eq!( + reprojection.paragraphs.len(), + expected_paragraphs, + "docx round-trip grew the paragraph count — a bare-equation-wrapper `` minted a spurious boundary (T7.21 regression)", + ); + } + + #[test] + fn synthetic_mixed_object_document_carries_objects() { + let projection = synthetic_mixed_object_document(120); + assert!( + projection.blocks.len() > projection.paragraphs.len(), + "mixed fixture must carry non-paragraph objects (blocks={}, paragraphs={})", + projection.blocks.len(), + projection.paragraphs.len(), + ); + } +} diff --git a/crates/flowstate-corpus/src/lib.rs b/crates/flowstate-corpus/src/lib.rs new file mode 100644 index 00000000..98235c38 --- /dev/null +++ b/crates/flowstate-corpus/src/lib.rs @@ -0,0 +1,8 @@ +//! Shared library surface for the `flowstate-corpus` perf/fidelity harness bins. +//! +//! The default binary (`main.rs`, the corpus sweep) and the perf-bench bins +//! (`collab_bench`, `hotpath_bench`, `perf_probe`, `contention_bench`) share the +//! synthetic-fixture builders here so equation/object shapes the real corpus +//! lacks can be driven without a `.docx` (§perf-heaven T7). + +pub mod fixtures; diff --git a/crates/flowstate-corpus/src/main.rs b/crates/flowstate-corpus/src/main.rs new file mode 100644 index 00000000..7c4fde82 --- /dev/null +++ b/crates/flowstate-corpus/src/main.rs @@ -0,0 +1,690 @@ +//! Corpus import-fidelity sweep (workplace policy: validate imports over the whole +//! real debate `.docx` corpus, not just synthetic fixtures). +//! +//! Walks a corpus directory (default `$FLOWSTATE_CORPUS_DIR` or the repo-local +//! `helpers/corpus/dropbox-office-flat`), +//! DYNAMICALLY discovering every `.docx` at run time, and checks each through the real +//! import path: +//! 1. `import_docx_bytes_to_loro` — the file opens without error or panic, +//! 2. `document_from_loro_with_defects` — the projection derives with ZERO defects +//! (a defect means the importer left canonical state a repair pass must fix), +//! 3. optional `--roundtrip` — export the projection back to `.docx`, re-import, and +//! assert the text survives (catches export/reimport data loss). +//! +//! An incremental LEDGER (keyed by path + size + mtime) means re-runs — and `--watch` +//! (periodic rescan) — only process files that are NEW or CHANGED since last time, so +//! as files are added to the corpus, the new ones get checked +//! automatically. Failures are reported LOUDLY with their full path. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Condvar, Mutex}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +const DEFAULT_CORPUS_RELATIVE_TO_WORKSPACE: &str = "helpers/corpus/dropbox-office-flat"; + +/// One ledger entry: the file fingerprint plus the last checked outcome. +#[derive(Clone, Serialize, Deserialize)] +struct Record { + size: u64, + mtime_ns: u128, + ok: bool, + /// Failure detail (empty when `ok`). + detail: String, + checked_at_unix: u64, + /// Wall-clock milliseconds the import+project check took (perf meter). + #[serde(default)] + check_ms: u64, +} + +/// path (as a string key) -> last result. A `BTreeMap` keeps the ledger file stable/diffable. +type Ledger = BTreeMap; + +struct Config { + corpus_dir: PathBuf, + ledger_path: PathBuf, + report_path: PathBuf, + jobs: usize, + roundtrip: bool, + recheck: bool, + watch: bool, + watch_interval: Duration, +} + +fn main() { + // The importer is meant to be robust, but the whole point of this sweep is to find + // the files where it ISN'T — so swallow the default panic output; per-file + // `catch_unwind` turns a panic into a recorded failure instead of aborting the run. + std::panic::set_hook(Box::new(|_| {})); + + let config = match parse_config() { + Ok(config) => config, + Err(message) => { + eprintln!("{message}"); + std::process::exit(2); + }, + }; + + if let Err(error) = run(&config) { + eprintln!("corpus sweep failed: {error:#}"); + std::process::exit(1); + } +} + +fn parse_config() -> Result { + let mut corpus_dir: Option = None; + let mut jobs: Option = None; + let mut roundtrip = false; + let mut recheck = false; + let mut watch = false; + let mut watch_interval = Duration::from_secs(30); + let mut ledger_path: Option = None; + let mut report_path: Option = None; + + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--roundtrip" => roundtrip = true, + "--recheck" => recheck = true, + "--watch" => watch = true, + "--jobs" => { + jobs = Some( + args + .next() + .and_then(|v| v.parse().ok()) + .ok_or("--jobs needs a number")?, + ); + }, + "--interval" => { + let secs: u64 = args + .next() + .and_then(|v| v.parse().ok()) + .ok_or("--interval needs seconds")?; + watch_interval = Duration::from_secs(secs.max(1)); + }, + "--ledger" => ledger_path = Some(PathBuf::from(args.next().ok_or("--ledger needs a path")?)), + "--report" => report_path = Some(PathBuf::from(args.next().ok_or("--report needs a path")?)), + "-h" | "--help" => return Err(usage()), + other if other.starts_with('-') => return Err(format!("unknown flag {other}\n{}", usage())), + other => corpus_dir = Some(PathBuf::from(other)), + } + } + + let corpus_dir = corpus_dir + .or_else(|| std::env::var_os("FLOWSTATE_CORPUS_DIR").map(PathBuf::from)) + .unwrap_or_else(default_corpus_dir); + if !corpus_dir.is_dir() { + return Err(format!("corpus dir does not exist: {}", corpus_dir.display())); + } + + let state_dir = default_state_dir(); + Ok(Config { + corpus_dir, + ledger_path: ledger_path.unwrap_or_else(|| state_dir.join("ledger.json")), + report_path: report_path.unwrap_or_else(|| state_dir.join("failures.txt")), + jobs: jobs.unwrap_or_else(default_jobs), + roundtrip, + recheck, + watch, + watch_interval, + }) +} + +fn usage() -> String { + "usage: flowstate-corpus [CORPUS_DIR] [--watch] [--roundtrip] [--recheck] \ + [--jobs N] [--interval SECS] [--ledger PATH] [--report PATH]\n \ + CORPUS_DIR defaults to $FLOWSTATE_CORPUS_DIR or repo-local helpers/corpus/dropbox-office-flat" + .to_string() +} + +fn default_corpus_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join(DEFAULT_CORPUS_RELATIVE_TO_WORKSPACE) +} + +fn default_state_dir() -> PathBuf { + let base = std::env::var_os("HOME").map_or_else(|| PathBuf::from("."), PathBuf::from); + base.join(".flowstate-corpus") +} + +fn default_jobs() -> usize { + std::thread::available_parallelism().map_or(4, |n| n.get().saturating_sub(1).max(1)) +} + +fn run(config: &Config) -> anyhow::Result<()> { + if let Some(parent) = config.ledger_path.parent() { + std::fs::create_dir_all(parent).ok(); + } + let mut ledger = load_ledger(&config.ledger_path); + + eprintln!( + "corpus sweep: dir={} jobs={} roundtrip={} watch={} ledger={}", + config.corpus_dir.display(), + config.jobs, + config.roundtrip, + config.watch, + config.ledger_path.display(), + ); + + sweep_once(config, &mut ledger)?; + + if config.watch { + eprintln!( + "--watch: rescanning every {}s for new/changed files (Ctrl-C to stop)", + config.watch_interval.as_secs() + ); + loop { + std::thread::sleep(config.watch_interval); + sweep_once(config, &mut ledger)?; + } + } + Ok(()) +} + +/// One incremental pass: discover, check only new/changed files, merge + persist, report. +fn sweep_once(config: &Config, ledger: &mut Ledger) -> anyhow::Result<()> { + let discovered = discover_docx(&config.corpus_dir); + let to_check: Vec = discovered + .iter() + .filter(|file| config.recheck || is_stale(ledger, file)) + .cloned() + .collect(); + + if to_check.is_empty() { + report(config, ledger, &discovered, 0); + return Ok(()); + } + + eprintln!("discovered {} .docx | checking {} new/changed", discovered.len(), to_check.len()); + // Share the ledger with the workers so results are recorded AND periodically + // flushed to disk mid-pass — a full sweep of the large case files takes many + // minutes, so a killed run must not lose everything (and the next run resumes). + let shared: Mutex = Mutex::new(std::mem::take(ledger)); + check_all(&to_check, config, &shared); + *ledger = shared.into_inner().expect("ledger mutex"); + save_ledger(&config.ledger_path, ledger)?; + report(config, ledger, &discovered, to_check.len()); + Ok(()) +} + +#[derive(Clone)] +struct Discovered { + path: PathBuf, + key: String, + size: u64, + mtime_ns: u128, +} + +fn discover_docx(dir: &Path) -> Vec { + let mut out = Vec::new(); + let walker = ignore::WalkBuilder::new(dir) + .standard_filters(false) // do not skip hidden/gitignored — sweep everything... + .hidden(false) + .build(); + for entry in walker.flatten() { + let path = entry.path(); + // ...except Dropbox's own bookkeeping dirs and macOS AppleDouble sidecar dirs. + if path.components().any(|component| { + let name = component.as_os_str().to_string_lossy(); + name == ".dropbox" || name == ".dropbox.cache" || name == "__MACOSX" + }) { + continue; + } + let is_docx = path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("docx")); + if !is_docx { + continue; + } + // Skip non-documents that only *look* like `.docx`: Word lock/temp files + // (`~$name.docx`) and macOS AppleDouble resource-fork sidecars (`._name.docx`, + // which are not zip archives). These are expected import failures, not bugs. + if path + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("~$") || n.starts_with("._")) + { + continue; + } + let Ok(meta) = entry.metadata() else { continue }; + if !meta.is_file() { + continue; + } + // Zero-byte files are empty shells or not-yet-synced placeholders — not real + // documents. Skipping keeps them out of the failure count; once a placeholder + // syncs real bytes its size changes and it is discovered and checked normally. + if meta.len() == 0 { + continue; + } + let mtime_ns = meta + .modified() + .ok() + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |d| d.as_nanos()); + out.push(Discovered { + key: path.to_string_lossy().into_owned(), + path: path.to_path_buf(), + size: meta.len(), + mtime_ns, + }); + } + out +} + +/// A file needs checking if we've never seen it, or its size/mtime changed since. +fn is_stale(ledger: &Ledger, file: &Discovered) -> bool { + ledger + .get(&file.key) + .is_none_or(|record| record.size != file.size || record.mtime_ns != file.mtime_ns) +} + +/// §perf-heaven T8.20: a memory-BUDGET gate so `jobs` workers can't each hold a +/// large-doc import + full Dst state at once and OOM (the systemd cap killed +/// the full `--roundtrip` / `RICHTEXT_VERIFY` sweeps at jobs=8/15). Each file +/// acquires permits ≈ its estimated peak memory (docx bytes × a multiplier), so +/// large files serialize against the budget while small files still flow at full +/// `jobs`. A file bigger than the whole budget takes it all (runs alone) rather +/// than deadlocking. This lets the definitive full-corpus sweep run instead of +/// the `--jobs 3-4` workaround. +struct MemGate { + available: Mutex, + cond: Condvar, + total: u64, +} + +impl MemGate { + fn new(total: u64) -> Self { + Self { + available: Mutex::new(total), + cond: Condvar::new(), + total, + } + } + + /// Block until `want` (clamped to the whole budget) permits are free, take them. + fn acquire(&self, want: u64) -> u64 { + let want = want.min(self.total).max(1); + let mut available = self.available.lock().expect("mem-gate mutex"); + while *available < want { + available = self.cond.wait(available).expect("mem-gate condvar"); + } + *available -= want; + want + } + + fn release(&self, amount: u64) { + *self.available.lock().expect("mem-gate mutex") += amount; + self.cond.notify_all(); + } +} + +/// The in-flight memory budget: `FLOWSTATE_CORPUS_MEM_BUDGET_MB`, else ~55% of +/// system RAM (from /proc/meminfo), else a conservative 6 GiB fallback. +fn mem_budget_bytes() -> u64 { + if let Some(mb) = std::env::var("FLOWSTATE_CORPUS_MEM_BUDGET_MB") + .ok() + .and_then(|value| value.parse::().ok()) + { + return mb.saturating_mul(1024 * 1024).max(256 * 1024 * 1024); + } + let system = std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|meminfo| { + meminfo.lines().find_map(|line| { + line + .strip_prefix("MemTotal:") + .and_then(|rest| rest.trim().strip_suffix(" kB")) + .and_then(|kb| kb.trim().parse::().ok()) + }) + }) + .map(|kb| kb.saturating_mul(1024)); + system.map_or(6 * 1024 * 1024 * 1024, |bytes| (bytes / 100).saturating_mul(55)) +} + +/// Estimated peak resident memory for importing a docx of `size` bytes. The +/// import + full-state build expands the compressed docx ~50–100×; use a +/// conservative multiplier so the gate errs toward under-subscription. +fn estimated_peak_bytes(size: u64) -> u64 { + size.saturating_mul(90).max(64 * 1024 * 1024) +} + +/// Check every file across `jobs` workers, recording each result into `shared` +/// (the ledger) and flushing to disk every `FLUSH_EVERY` files so progress survives. +fn check_all(files: &[Discovered], config: &Config, shared: &Mutex) { + const FLUSH_EVERY: usize = 500; + let next = AtomicUsize::new(0); + let done = AtomicUsize::new(0); + let total = files.len(); + let started = Instant::now(); + let gate = MemGate::new(mem_budget_bytes()); + + std::thread::scope(|scope| { + for _ in 0..config.jobs.max(1) { + scope.spawn(|| { + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + if index >= total { + break; + } + let file = &files[index]; + let file_started = Instant::now(); + // Reserve this file's estimated peak memory before importing; release it + // as soon as the check returns (RAII-free: explicit release below). + let held = gate.acquire(estimated_peak_bytes(file.size)); + let outcome = check_one(&file.path, config.roundtrip); + gate.release(held); + let elapsed = file_started.elapsed(); + // Surface pathologically slow files (near-hangs) so they are diagnosable. + if elapsed > Duration::from_secs(10) { + eprintln!(" SLOW {:.1}s: {}", elapsed.as_secs_f64(), file.path.display()); + } + let record = Record { + size: file.size, + mtime_ns: file.mtime_ns, + ok: outcome.is_ok(), + detail: outcome.err().unwrap_or_default(), + checked_at_unix: unix_now(), + #[allow(clippy::cast_possible_truncation, reason = "a single-file import is far under u64::MAX milliseconds")] + check_ms: elapsed.as_millis() as u64, + }; + // Insert under the lock (guard drops at the `;`), then bump the counter — + // the atomic does not need the ledger lock held. + shared + .lock() + .expect("ledger mutex") + .insert(file.key.clone(), record); + let count = done.fetch_add(1, Ordering::Relaxed) + 1; + if count.is_multiple_of(250) { + let rate = count as f64 / started.elapsed().as_secs_f64().max(0.001); + eprintln!(" ... {count}/{total} ({rate:.0}/s)"); + } + if count.is_multiple_of(FLUSH_EVERY) { + // Serialize+write under the lock (brief, ~ms); resumability over speed. + let ledger = shared.lock().expect("ledger mutex"); + save_ledger(&config.ledger_path, &ledger).ok(); + } + } + }); + } + }); +} + +/// Run the fidelity check on one file, catching panics from the importer. +fn check_one(path: &Path, roundtrip: bool) -> Result<(), String> { + let bytes = std::fs::read(path).map_err(|error| format!("read: {error}"))?; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| check_bytes(&bytes, roundtrip))) { + Ok(result) => result, + Err(payload) => Err(format!("PANIC: {}", panic_message(payload.as_ref()))), + } +} + +fn check_bytes(bytes: &[u8], roundtrip: bool) -> Result<(), String> { + let (imported, report) = flowstate_docx::import_docx_bytes_to_loro(bytes, "corpus-sweep").map_err(|error| format!("import: {error}"))?; + let (projection, defects) = flowstate_document::document_from_loro_with_defects(&imported.doc).map_err(|error| format!("project: {error}"))?; + if !defects.is_empty() { + return Err(format!( + "{} projection defect(s) (paragraphs_imported={}); sample: {:?}", + defects.len(), + report.paragraphs_imported, + defects.iter().take(3).collect::>(), + )); + } + // Total content loss: the importer reported paragraphs but the projection is + // ENTIRELY blank — no text AND no blocks. (An image/table-only doc has empty text + // but non-empty blocks, so it is NOT flagged — that was the earlier false positive.) + if report.paragraphs_imported > 0 && projection.text.to_string().trim().is_empty() && projection.blocks.is_empty() { + return Err(format!( + "content loss: imported {} paragraph(s) but the projection has no text and no blocks", + report.paragraphs_imported, + )); + } + // §perf-heaven T1 corpus validation: project a SNAPSHOT-REIMPORTED (cold, + // `LazyLoad::Src`) state so `get_richtext_value` takes the T1 Src fast path + // (the normal import above leaves the state warm/`Dst`, which never exercises + // it). With `FLOWSTATE_RICHTEXT_VERIFY` on, that path also builds the full + // state and asserts bit-identical — so a T1 divergence panics HERE and is + // recorded as this document's failure (per-file catch_unwind). + if std::env::var_os("FLOWSTATE_SNAPSHOT_VERIFY").is_some() { + let snapshot = imported + .doc + .export(loro::ExportMode::Snapshot) + .map_err(|error| format!("snapshot export: {error}"))?; + let cold = loro::LoroDoc::new(); + cold + .import(&snapshot) + .map_err(|error| format!("snapshot reimport: {error}"))?; + let (reprojection, snap_defects) = + flowstate_document::document_from_loro_with_defects(&cold).map_err(|error| format!("cold project: {error}"))?; + // §perf-heaven T7.26: the cold (`LazyLoad::Src`) reprojection must equal the + // warm one. The warm `projection` already passed the zero-defect gate above, + // so the Src path — `get_richtext_value`, and now the Src-safe `char_at` + // object-anchor validation — MUST also produce zero defects and BYTE-identical + // text. A `char_at` Src bug that misplaces/quarantines an object shows up as a + // non-empty `snap_defects` or a text divergence RIGHT HERE, giving the + // object-anchor Src path the equivalence net `get_richtext_value` already has + // via `FLOWSTATE_RICHTEXT_VERIFY`. (Earlier this was ignored as "the doc's own + // repair concern"; that is wrong once the warm side is known defect-free.) + if !snap_defects.is_empty() { + return Err(format!( + "T7.26 Src-path projection produced {} defect(s) the warm path did not; sample: {:?}", + snap_defects.len(), + snap_defects.iter().take(3).collect::>(), + )); + } + if reprojection.text != projection.text { + return Err("T7.26 Src-path projection text diverged from the warm projection".to_string()); + } + } + + if roundtrip { + // Export from the ASSET-BEARING projection (built from the docx during + // import), not the Loro-only re-projection above: Loro stores asset metadata + // (hash/length) but NOT the bytes (they live out-of-band in the package), so + // `document_from_loro`'s projection has no image bytes and the exporter falls + // back to `[alt text]` — a harness artifact the real app never hits (it loads + // asset bytes from the package). `imported.projection` matches the live app. + // + // §act-eleven A11.8: the COMPARISON baseline is the Loro MATERIALIZATION of + // the first import (`projection`), matching the reimport side like-for-like + // — the live app materializes both sides, so the old docx-direct baseline + // asserted a state the user never sees and mis-flagged the materializer's + // object-adjacency folding (the last standing corpus "residual") as loss. + return check_roundtrip(&imported.projection, &projection); + } + Ok(()) +} + +/// Export the projection back to `.docx`, re-import it, and assert the text +/// survives. `export_source` is the asset-bearing docx-built projection (what +/// the app persists/exports); `baseline` is the Loro materialization of the +/// SAME import (what the app displays) — the reimport is compared against +/// `baseline`, Loro-vs-Loro. +fn check_roundtrip( + export_source: &flowstate_document::DocumentProjection, + baseline: &flowstate_document::DocumentProjection, +) -> Result<(), String> { + let unique = format!("fscorpus-{}-{:?}.docx", std::process::id(), std::thread::current().id()); + let tmp = std::env::temp_dir().join(unique); + flowstate_docx::write_docx(&tmp, export_source).map_err(|error| format!("write_docx: {error}"))?; + let bytes = std::fs::read(&tmp).map_err(|error| format!("reread: {error}")); + // Diagnostic: keep the exported bytes for inspection instead of deleting. + if let Some(dir) = std::env::var_os("FLOWSTATE_ROUNDTRIP_DUMP") { + let _ = std::fs::create_dir_all(&dir); + let name = tmp + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| "export.docx".into()); + if let Ok(bytes) = &bytes { + let _ = std::fs::write(std::path::Path::new(&dir).join(name), bytes); + } + } + let _ = std::fs::remove_file(&tmp); + let bytes = bytes?; + let (imported, _report) = + flowstate_docx::import_docx_bytes_to_loro(&bytes, "corpus-sweep-rt").map_err(|error| format!("reimport: {error}"))?; + let (reprojection, defects) = + flowstate_document::document_from_loro_with_defects(&imported.doc).map_err(|error| format!("reproject: {error}"))?; + if !defects.is_empty() { + return Err(format!("round-trip produced {} projection defect(s)", defects.len())); + } + // §act-eleven C5: STYLE census (report-only). The roundtrip gate asserts + // TEXT equality; run/paragraph styles and block kinds can silently degrade + // across the corpus with no signal. Report divergences without failing so + // the pre-existing count can be measured before this is promoted to a gate. + if std::env::var_os("FLOWSTATE_ROUNDTRIP_STYLE_CENSUS").is_some() { + let mut style_divergences = 0usize; + let mut run_divergences = 0usize; + let paragraphs = baseline.paragraphs.len().min(reprojection.paragraphs.len()); + for ix in 0..paragraphs { + if baseline.paragraphs[ix].style != reprojection.paragraphs[ix].style { + style_divergences += 1; + } + if baseline.paragraphs[ix].runs != reprojection.paragraphs[ix].runs { + run_divergences += 1; + } + } + let block_kind = |block: &flowstate_document::Block| match block { + flowstate_document::Block::Paragraph(_) => 0u8, + flowstate_document::Block::Image(_) => 1, + flowstate_document::Block::Equation(_) => 2, + flowstate_document::Block::Table(_) => 3, + }; + let kinds_diverge = baseline.blocks.len() != reprojection.blocks.len() + || baseline + .blocks + .iter() + .zip(reprojection.blocks.iter()) + .any(|(before, after)| block_kind(before) != block_kind(after)); + if style_divergences > 0 || run_divergences > 0 || kinds_diverge { + eprintln!( + "STYLE-CENSUS paragraphs={paragraphs} style_divergences={style_divergences} run_divergences={run_divergences} block_kinds_diverge={kinds_diverge}" + ); + } + } + if baseline.text != reprojection.text { + if std::env::var_os("FLOWSTATE_ROUNDTRIP_DIFF").is_some() { + let before = baseline.text.to_string(); + let after = reprojection.text.to_string(); + if let Some(dir) = std::env::var_os("FLOWSTATE_ROUNDTRIP_TEXTDUMP") { + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::write(std::path::Path::new(&dir).join("before.txt"), &before); + let _ = std::fs::write(std::path::Path::new(&dir).join("after.txt"), &after); + } + let first = before + .char_indices() + .zip(after.chars()) + .find(|((_, b), a)| b != a) + .map(|((i, _), _)| i) + .unwrap_or(before.len().min(after.len())); + let ctx = |s: &str, at: usize| { + let start = at.saturating_sub(160); + let end = (at + 160).min(s.len()); + s.get(start..end) + .unwrap_or("") + .replace('\n', "\\n") + .replace('\u{2028}', "\\u2028") + }; + eprintln!( + "ROUNDTRIP-DIFF len {} -> {} (first diff @ char {}):\n BEFORE: …{}…\n AFTER: …{}…", + before.chars().count(), + after.chars().count(), + first, + ctx(&before, first), + ctx(&after, first), + ); + } + // §perf-heaven T7.27: the mismatch is frequently data ADDED (a spurious + // U+2028 / `\n` / `[alt]`), not lost — report the actual direction from the + // char-count delta instead of always blaming loss. + let before_len = baseline.text.to_string().chars().count(); + let after_len = reprojection.text.to_string().chars().count(); + let direction = match after_len.cmp(&before_len) { + std::cmp::Ordering::Greater => format!("{} char(s) ADDED", after_len - before_len), + std::cmp::Ordering::Less => format!("{} char(s) LOST", before_len - after_len), + std::cmp::Ordering::Equal => "same length, content changed".to_string(), + }; + return Err(format!( + "round-trip text mismatch on export->reimport ({direction}); set FLOWSTATE_ROUNDTRIP_DIFF for the diff" + )); + } + Ok(()) +} + +fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { + if let Some(message) = payload.downcast_ref::<&str>() { + (*message).to_string() + } else if let Some(message) = payload.downcast_ref::() { + message.clone() + } else { + "unknown panic".to_string() + } +} + +fn report(config: &Config, ledger: &Ledger, discovered: &[Discovered], checked_now: usize) { + // Report over CURRENTLY-discovered files only (a file deleted from the corpus is not + // a failure), reading each one's latest ledger outcome. + let mut failures: Vec<(&str, &str)> = Vec::new(); + let mut ok = 0usize; + for file in discovered { + match ledger.get(&file.key) { + Some(record) if record.ok => ok += 1, + Some(record) => failures.push((file.key.as_str(), record.detail.as_str())), + None => {}, + } + } + failures.sort_unstable(); + + eprintln!( + "---\ncorpus: {} files | ok {} | FAILED {} | checked this pass {}", + discovered.len(), + ok, + failures.len(), + checked_now, + ); + for (path, detail) in failures.iter().take(40) { + eprintln!(" FAIL {path}\n {detail}"); + } + if failures.len() > 40 { + eprintln!(" ... and {} more (see {})", failures.len() - 40, config.report_path.display()); + } + + // Persist the full failure list for offline triage. + if let Some(parent) = config.report_path.parent() { + std::fs::create_dir_all(parent).ok(); + } + let mut body = String::new(); + for (path, detail) in &failures { + use std::fmt::Write as _; + let _ = writeln!(body, "{path}\t{detail}"); + } + if let Err(error) = std::fs::write(&config.report_path, body) { + eprintln!(" (could not write report {}: {error})", config.report_path.display()); + } +} + +fn load_ledger(path: &Path) -> Ledger { + std::fs::read(path) + .ok() + .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .unwrap_or_default() +} + +fn save_ledger(path: &Path, ledger: &Ledger) -> anyhow::Result<()> { + let bytes = serde_json::to_vec_pretty(ledger)?; + // Write-then-rename so a crash mid-write never corrupts the ledger. + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, path)?; + Ok(()) +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) +} diff --git a/crates/flowstate-document/Cargo.toml b/crates/flowstate-document/Cargo.toml index 7bc6bee1..6ea81680 100644 --- a/crates/flowstate-document/Cargo.toml +++ b/crates/flowstate-document/Cargo.toml @@ -10,9 +10,31 @@ workspace = true [dependencies] gpui-flowtext = { workspace = true } gpui = { version = "0.2.2", default-features = false } +imagesize = { workspace = true } rustc-hash = { workspace = true } +# §A13.1.2: "rayon" enables update_rayon — multithreaded hashing for the +# multi-MB chunk payloads still verified on the open path. +blake3 = { version = "1.8.5", features = ["rayon"] } +loro.workspace = true +postcard.workspace = true +serde.workspace = true +# §A13.1: postcard's Vec wire format is IDENTICAL to serde_bytes' +# (varint len + raw bytes) — this only swaps the element-wise decode loop +# for a memcpy-scale one. Old packages decode unchanged. +serde_bytes = "0.11" +uuid.workspace = true +tempfile.workspace = true +atomicwrites = "0.4.4" +flowstate-fidelity.workspace = true +hotpath.workspace = true +loro-internal = "1.13.1" + +[package.metadata.cargo-machete] +# serde_bytes is used via `#[serde(with = "serde_bytes")]` attributes in +# package.rs (§A13.1), which cargo-machete's source scan cannot see. +ignored = ["serde_bytes"] [features] -hotpath = ["gpui-flowtext/hotpath"] -hotpath-cpu = ["hotpath", "gpui-flowtext/hotpath-cpu"] -hotpath-alloc = ["hotpath", "gpui-flowtext/hotpath-alloc"] +hotpath = ["hotpath/hotpath", "gpui-flowtext/hotpath"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "gpui-flowtext/hotpath-cpu"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "gpui-flowtext/hotpath-alloc"] diff --git a/crates/flowstate-document/src/instrument.rs b/crates/flowstate-document/src/instrument.rs new file mode 100644 index 00000000..d70b23b0 --- /dev/null +++ b/crates/flowstate-document/src/instrument.rs @@ -0,0 +1,91 @@ +//! Always-on algorithmic-work counters for complexity-regression tests. +//! +//! These count the two operations whose super-linear growth caused the large-document +//! actor hangs: full projection rebuilds (a repair storm fires many per edit) and +//! per-cursor `get_cursor_pos` resolutions (the O(records) scan the batched resolver +//! replaced). Tests [`snapshot`] before/after an operation and assert the delta is BOUNDED +//! independent of document size — a signal that does not depend on wall-clock time or on +//! fidelity tracing being on, so it stays valid in CI and catches a regression the moment +//! it lands. +//! +//! The counters are THREAD-LOCAL: a test measures work done synchronously on its own +//! thread, so parallel tests do not contaminate each other's deltas. (Work a runtime does +//! on a background thread — e.g. behind the CRDT actor — is not visible to a test measuring +//! on the test thread; harnesses that need to measure that drive the runtime directly.) + +use std::cell::Cell; + +thread_local! { + static FULL_PROJECTIONS: Cell = const { Cell::new(0) }; + static CURSOR_POS_RESOLVES: Cell = const { Cell::new(0) }; + static BODY_TO_DELTA_BUILDS: Cell = const { Cell::new(0) }; +} + +/// One full `document_from_loro` rebuild of the whole document. +pub fn record_full_projection() { + FULL_PROJECTIONS.with(|count| count.set(count.get().wrapping_add(1))); +} + +/// One per-cursor `get_cursor_pos` resolution (history-traced; the op the batched +/// `query_text_id_positions` resolver exists to avoid calling O(records) times). +pub fn record_cursor_pos_resolve() { + CURSOR_POS_RESOLVES.with(|count| count.set(count.get().wrapping_add(1))); +} + +/// One WHOLE-BODY `text.to_delta()` materialization (§perf-heaven T3): a full +/// projection allocates the entire body as a `Vec` of run chunks. +/// This is a COLD-path event (reopen / receiving-peer full rebuild). A +/// per-keystroke or single-op edit must go through the REGIONAL rematerializer +/// (§6-R) and trigger ZERO of these — a complexity test asserts the delta stays +/// bounded independent of document size, so a regression back to whole-body +/// rebuild-per-edit trips loudly. +pub fn record_body_to_delta() { + record_body_to_delta_tagged("untagged"); +} + +/// §oom-leads #4: per-call-site whole-body materialization census, printed per +/// call when `FLOWSTATE_BODY_DELTA_PROBE` is set (env-gated, zero cost off). +pub fn record_body_to_delta_tagged(site: &'static str) { + static PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + if *PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_BODY_DELTA_PROBE").is_some()) { + eprintln!("[flowstate-body-delta] site={site}"); + } + record_body_to_delta_inner(); +} + +fn record_body_to_delta_inner() { + BODY_TO_DELTA_BUILDS.with(|count| count.set(count.get().wrapping_add(1))); +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WorkCounts { + pub full_projections: u64, + pub cursor_pos_resolves: u64, + pub body_to_delta_builds: u64, +} + +pub fn snapshot() -> WorkCounts { + WorkCounts { + full_projections: FULL_PROJECTIONS.with(Cell::get), + cursor_pos_resolves: CURSOR_POS_RESOLVES.with(Cell::get), + body_to_delta_builds: BODY_TO_DELTA_BUILDS.with(Cell::get), + } +} + +impl WorkCounts { + /// Work done between `earlier` and `self`. + #[must_use] + pub fn since(self, earlier: WorkCounts) -> WorkCounts { + WorkCounts { + full_projections: self + .full_projections + .saturating_sub(earlier.full_projections), + cursor_pos_resolves: self + .cursor_pos_resolves + .saturating_sub(earlier.cursor_pos_resolves), + body_to_delta_builds: self + .body_to_delta_builds + .saturating_sub(earlier.body_to_delta_builds), + } + } +} diff --git a/crates/flowstate-document/src/lib.rs b/crates/flowstate-document/src/lib.rs index 8fdc1edc..76708a15 100644 --- a/crates/flowstate-document/src/lib.rs +++ b/crates/flowstate-document/src/lib.rs @@ -1,6 +1,42 @@ +pub mod instrument; +pub mod loro_import; +pub mod loro_projection; +pub mod loro_schema; +pub mod package; +mod package_search; +pub mod projection_defects; +pub mod streaming_delta; +pub mod table_topology; + pub use gpui_flowtext::*; +pub use loro_import::{ + ImportedLoroDocument, document_to_loro, import_document_projection, import_paragraphs_as_loro, write_imported_document_as_loro_db8, +}; +pub use loro_projection::{ + ChangedContainerOwner, RegionRows, document_from_loro, document_from_loro_with_defects, materialize_body_region, materialize_table_block, + object_input_blocks_for_ids, object_input_blocks_from_loro, owner_of_changed_container, section_page_attrs, +}; +pub use loro_schema::{ + BLOCKS_BY_ID, BODY_FLOW_ID, COMMENTS_BY_ID, FLOW_ATTRS_KEY, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, MAIN_BODY_BLOCK_ID, + MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, MARK_VERT_ALIGN, META, + OBJECT_REPLACEMENT, PARAGRAPHS_BY_ID, PageNumberFormat, REPLICAS_BY_ID, ROOT, ROOT_BODY_FLOW_ID, ROOT_FIRST_PARAGRAPH_ID, + SECTION_ATTR_COLUMNS, SECTION_ATTR_FOOTER_FLOW_ID, SECTION_ATTR_HEADER_FLOW_ID, SECTION_ATTR_MARGIN_BOTTOM, SECTION_ATTR_MARGIN_LEFT, + SECTION_ATTR_MARGIN_RIGHT, SECTION_ATTR_MARGIN_TOP, SECTION_ATTR_ORIENTATION, SECTION_ATTR_PAGE_HEIGHT, SECTION_ATTR_PAGE_NUMBERING_FORMAT, + SECTION_ATTR_PAGE_NUMBERING_START, SECTION_ATTR_PAGE_WIDTH, SECTIONS_BY_ID, SENTINEL_NEWLINE, SectionMargins, SectionOrientation, + SectionPageAttrs, SectionPageNumbering, SectionPageSize, TABLE_CELLS_BY_ID, TABLE_COLUMN_ORDER, TABLE_COLUMNS_BY_ID, TABLE_KEY, + TABLE_ROW_ORDER, TABLE_ROWS_BY_ID, USERS_BY_ID, cell_flow_loro_id, cell_loro_id, cell_loro_id_for, column_loro_id, document_id, + document_schema_version, ensure_section, fork_document_lineage, init_loro_document, new_loro_document, parse_cell_loro_id, + parse_column_loro_id, parse_row_loro_id, read_section_page_attrs, record_revision, register_replica, register_user, row_loro_id, + set_section_page_attrs, touch_document_metadata, +}; +pub use package::{ + AssetChunk, ChunkRef, DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD, DocumentPackage, DocumentPackageManifest, IntegrityIndexEntry, + LORO_PACKAGE_FORMAT_VERSION, LORO_SCHEMA_VERSION, PackageRevision, ProjectionCacheChunk, SchemaMigrationRecord, SearchUnitChunk, + ThumbnailChunk, loro_db8_bytes, read_loro_db8, write_loro_db8, +}; +pub use projection_defects::{ProjectionDefect, TableTopologyKind}; -use std::{io, path::Path}; +use std::{io, path::Path, sync::Arc}; use gpui::{Pixels, black, px, rgb}; use rustc_hash::{FxHashMap, FxHashSet}; @@ -33,20 +69,66 @@ fn border_eighth_points(value: f32) -> Pixels { pt(value / 8.0) } -pub fn read_db8(path: impl AsRef) -> io::Result { - read_document(path) +pub fn read_db8(path: impl AsRef) -> io::Result { + document_from_package(DocumentPackage::read(path)?) } -pub fn read_db8_bytes(bytes: &[u8]) -> io::Result { - read_document_bytes(bytes) +pub fn read_db8_bytes(bytes: &[u8]) -> io::Result { + document_from_package(DocumentPackage::from_bytes(bytes)?) } -pub fn write_db8(path: impl AsRef, document: &Document) -> io::Result<()> { - write_document(path, document) +fn document_from_package(package: DocumentPackage) -> io::Result { + let mut document = if let Some(document) = package.current_projection_document()? { + document + } else { + document_from_loro(&package.load_loro_doc()?)? + }; + attach_package_assets(&mut document, &package.assets); + Ok(document) } -pub fn db8_bytes(document: &Document) -> io::Result> { - document_bytes(document) +pub fn attach_package_assets(document: &mut DocumentProjection, assets: &[AssetChunk]) { + let referenced = referenced_asset_ids(document); + for asset in assets + .iter() + .filter(|asset| referenced.contains(&AssetId(asset.asset_id))) + { + let bytes = asset.bytes.clone(); + document.assets.assets.insert( + AssetId(asset.asset_id), + AssetRecord { + id: AssetId(asset.asset_id), + mime_type: asset.mime_type.clone().into(), + original_name: None, + content_hash: AssetRecord::stable_content_hash(&bytes), + bytes: Arc::new(bytes), + }, + ); + } + for id in referenced { + document + .assets + .assets + .entry(id) + .or_insert_with(|| AssetRecord { + id, + mime_type: "application/octet-stream".into(), + original_name: None, + content_hash: AssetRecord::stable_content_hash(&[]), + bytes: Arc::new(Vec::new()), + }); + } +} + +fn referenced_asset_ids(document: &DocumentProjection) -> FxHashSet { + document + .blocks + .iter() + .filter_map(|block| match block { + Block::Image(image) => Some(image.asset_id), + Block::Paragraph(_) | Block::Equation(_) | Block::Table(_) => None, + }) + .collect() } pub fn flowstate_document_theme() -> DocumentTheme { diff --git a/crates/flowstate-document/src/loro_import.rs b/crates/flowstate-document/src/loro_import.rs new file mode 100644 index 00000000..3ee42219 --- /dev/null +++ b/crates/flowstate-document/src/loro_import.rs @@ -0,0 +1,1266 @@ +use std::{io, path::Path}; + +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use gpui_flowtext::{ + Block, BlockAlignment, DocumentParagraphInput, DocumentProjection, DocumentTheme, EquationDisplay, EquationSyntax, HighlightStyle, ImageBlock, + ImageSizing, Paragraph, ParagraphStyle, RunSemanticStyle, RunStyles, TableBlock, TableCellBlock, TableColumnWidth, document_from_paragraphs, + paragraph_text, +}; +use loro::{ContainerTrait as _, LoroDoc, LoroMap, LoroMovableList, LoroResult, LoroText, LoroValue, TextDelta, ValueOrContainer, cursor::Side}; +use rustc_hash::FxHashMap; +use uuid::Uuid; + +use crate::{ + AssetChunk, BLOCKS_BY_ID, BODY_FLOW_ID, FLOW_ATTRS_KEY, FLOW_ID_KEY, FLOW_KIND_KEY, FLOW_TEXT_KEY, FLOWS_BY_ID, MARK_DIRECT_UNDERLINE, + MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, MARK_VERT_ALIGN, OBJECT_REPLACEMENT, + PARAGRAPHS_BY_ID, ROOT, ROOT_BODY_FLOW_ID, SECTIONS_BY_ID, + loro_schema::{ + ASSETS_BY_ID, REVISIONS, SectionPageAttrs, TABLE_CELLS_BY_ID, TABLE_COLUMN_ORDER, TABLE_COLUMNS_BY_ID, TABLE_KEY, TABLE_ROW_ORDER, + TABLE_ROWS_BY_ID, cell_flow_loro_id, cell_loro_id, column_loro_id, row_loro_id, write_section_page_attrs, + }, +}; + +/// Canonical result of an external import. The Loro document is authoritative; +/// the projection is a frontier-matched initial view built from the same semantic +/// import plan so the UI does not need to project the document a second time. +pub struct ImportedLoroDocument { + pub doc: LoroDoc, + pub projection: DocumentProjection, +} + +pub fn import_document_projection(mut document: DocumentProjection, title: &str) -> io::Result { + let doc = crate::loro_schema::new_loro_import_document(title).map_err(loro_io_error)?; + if document.ids.document_id != 0 { + crate::loro_schema::set_document_id(&doc, Uuid::from_u128(document.ids.document_id)).map_err(loro_io_error)?; + } + replace_body_from_document(&doc, &document).map_err(loro_io_error)?; + import_assets(&doc, &document).map_err(loro_io_error)?; + doc.commit(); + document.frontier = doc.state_frontiers().encode(); + fidelity_report_import(&doc, &document); + Ok(ImportedLoroDocument { doc, projection: document }) +} + +/// §fidelity import completion event + seed invariant. The caller path always +/// reaches here after `replace_body_from_document`, so the checks read the freshly +/// imported canonical state. Gated on [`fidelity::enabled`]; strictly read-only +/// (no `ensure_*`, so a failing invariant never fabricates the structure it is +/// asserting). Emits the imported block-shape counts and asserts the document +/// seeds a leading sentinel newline plus at least one durable first-paragraph +/// metadata record. +fn fidelity_report_import(doc: &LoroDoc, document: &DocumentProjection) { + if !fidelity::enabled() { + return; + } + let paragraphs = document.paragraphs.len(); + let (mut blocks, mut tables, mut images, mut equations) = (0_usize, 0_usize, 0_usize, 0_usize); + for block in document.blocks.iter() { + blocks += 1; + match block { + Block::Table(_) => tables += 1, + Block::Image(_) => images += 1, + Block::Equation(_) => equations += 1, + Block::Paragraph(_) => {}, + } + } + fidelity::event(FidelityClass::ImportExport, "import-complete", || { + format!("paragraphs={paragraphs} blocks={blocks} tables={tables} images={images} equations={equations}") + }); + let sentinel_ok = read_body_text(doc).is_some_and(|text| text.to_string().starts_with(crate::SENTINEL_NEWLINE)); + fidelity::check(sentinel_ok, FidelityClass::ImportExport, "missing-sentinel", || { + "imported document body does not start with the sentinel newline".to_string() + }); + let first_paragraph_ok = read_root_child_map(doc, PARAGRAPHS_BY_ID).is_some_and(|map| map.keys().next().is_some()); + fidelity::check(first_paragraph_ok, FidelityClass::ImportExport, "missing-first-paragraph", || { + "imported document has no durable first-paragraph metadata record".to_string() + }); +} + +/// Read-only resolution of a top-level `flowstate.root` child map. Unlike +/// `ensure_mergeable_map`, this never creates the container, so a fidelity +/// invariant cannot fabricate the very structure it is meant to verify. +fn read_root_child_map(doc: &LoroDoc, key: &str) -> Option { + match doc.get_map(ROOT).get(key)? { + ValueOrContainer::Container(container) => container.into_map().ok(), + ValueOrContainer::Value(_) => None, + } +} + +/// Read-only resolution of the body flow's text container (no `ensure_*`). +fn read_body_text(doc: &LoroDoc) -> Option { + let flows = read_root_child_map(doc, FLOWS_BY_ID)?; + let ValueOrContainer::Container(body) = flows.get(ROOT_BODY_FLOW_ID)? else { + return None; + }; + let body = body.into_map().ok()?; + let ValueOrContainer::Container(text) = body.get(FLOW_TEXT_KEY)? else { + return None; + }; + text.into_text().ok() +} + +pub fn import_paragraphs_as_loro( + theme: DocumentTheme, + paragraphs: Vec, + title: &str, +) -> io::Result { + import_document_projection(document_from_paragraphs(theme, paragraphs), title) +} + +pub fn document_to_loro(document: &DocumentProjection, title: &str) -> io::Result { + Ok(import_document_projection(document.clone(), title)?.doc) +} + +pub fn write_imported_document_as_loro_db8(path: impl AsRef, document: &DocumentProjection, title: &str) -> io::Result<()> { + let imported = import_document_projection(document.clone(), title)?; + crate::DocumentPackage::from_loro_snapshot_with_assets(&imported.doc, title, assets_from_document(&imported.projection))?.write(path) +} + +pub(crate) fn replace_body_from_document(doc: &LoroDoc, document: &DocumentProjection) -> LoroResult<()> { + let root = doc.get_map(ROOT); + let flows = root.ensure_mergeable_map(FLOWS_BY_ID)?; + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; + let sections = root.ensure_mergeable_map(SECTIONS_BY_ID)?; + root.ensure_mergeable_list(REVISIONS)?; + + let body_flow = ensure_flow(&flows, ROOT_BODY_FLOW_ID, "body")?; + let body_text = body_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + clear_map(&blocks)?; + clear_map(¶graphs)?; + clear_map(§ions)?; + + let plan = hotpath::measure_block!("import_plan_build", FlowTextImportPlan::for_document(document)); + let body_text_op_base = hotpath::measure_block!("import_plan_write_body", plan.write_to(Some(doc), &body_text)?); + + let mut paragraph_ix = 0_usize; + for (block_ix, (block, position)) in document + .blocks + .iter() + .zip(&plan.block_positions) + .enumerate() + { + match (block, position) { + (Block::Paragraph(_), FlowBlockPosition::Paragraph { boundary_pos, .. }) => { + import_paragraph_record( + ¶graphs, + &blocks, + BODY_FLOW_ID, + &body_text, + body_text_op_base, + *boundary_pos, + projection_block_id(document, block_ix, "paragraph_block"), + projection_paragraph_id(document, paragraph_ix), + )?; + paragraph_ix += 1; + }, + (Block::Image(image), FlowBlockPosition::Object { anchor_pos }) => { + import_image_block( + &flows, + &blocks, + document, + image, + projection_block_id(document, block_ix, "image"), + &body_text, + body_text_op_base, + *anchor_pos, + )?; + }, + (Block::Equation(equation), FlowBlockPosition::Object { anchor_pos }) => { + let durable_block_id = projection_block_id(document, block_ix, "equation"); + let block = ensure_block( + &blocks, + durable_block_id.clone(), + "equation", + BODY_FLOW_ID, + &body_text, + body_text_op_base, + *anchor_pos, + )?; + let source_flow_id = nested_flow_id("equation_source", &durable_block_id); + block.insert("source_flow_id", source_flow_id.as_str())?; + let source_flow = ensure_flow(&flows, &source_flow_id, "equation_source")?; + replace_text(&source_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, equation.source.as_ref())?; + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("syntax", equation_syntax_name(equation.syntax))?; + attrs.insert("display", equation_display_name(equation.display))?; + }, + (Block::Table(table), FlowBlockPosition::Object { anchor_pos }) => { + let durable_block_id = projection_block_id(document, block_ix, "table"); + let block = ensure_block( + &blocks, + durable_block_id.clone(), + "table", + BODY_FLOW_ID, + &body_text, + body_text_op_base, + *anchor_pos, + )?; + import_table(&flows, &block, table)?; + }, + _ => unreachable!("flow import plan must preserve document block shape"), + } + } + + import_sections(document, §ions, &flows, &body_text, &plan.paragraphs)?; + // §P2a: an import that carried no blocks still needs the canonical first + // paragraph seed so the projector never has to fabricate an identity for the + // lone sentinel boundary. A non-empty import already wrote its own + // paragraph/block records, so only the empty case routes through the shared + // seed (which is idempotent and converges with the runtime repair path). + if document.blocks.is_empty() { + crate::loro_schema::seed_document_body(doc)?; + } + Ok(()) +} + +fn import_image_block( + flows: &LoroMap, + blocks: &LoroMap, + document: &DocumentProjection, + image: &ImageBlock, + durable_block_id: String, + body_text: &LoroText, + text_op_base: Option<(u64, i32)>, + anchor_pos: usize, +) -> LoroResult<()> { + let block = ensure_block( + blocks, + durable_block_id.clone(), + "image", + BODY_FLOW_ID, + body_text, + text_op_base, + anchor_pos, + )?; + block.insert("asset_id", image.asset_id.0.to_string())?; + // §A11.9: a genuinely-LINKED image persists its external URL; the key is only + // written when a non-empty URL exists (embedded images carry no key at all — + // the presence-guarded delete keeps a re-imported block from resurrecting a + // stale URL without minting tombstone ops on the common embedded path). + match image + .external_url + .as_ref() + .map(|url| -> &str { url.as_ref() }) + .filter(|url| !url.is_empty()) + { + Some(url) => block.insert("external_url", url)?, + None => { + if block.get("external_url").is_some() { + block.delete("external_url")?; + } + }, + } + if let Some(asset) = document.assets.assets.get(&image.asset_id) { + block.insert("content_hash", blake3::hash(&asset.bytes).to_hex().as_str())?; + block.insert("mime_type", asset.mime_type.as_ref())?; + block.insert("byte_length", i64::try_from(asset.bytes.len()).unwrap_or(i64::MAX))?; + } + let alt_text_flow_id = nested_flow_id("image_alt", &durable_block_id); + block.insert("alt_text_flow_id", alt_text_flow_id.as_str())?; + let alt_flow = ensure_flow(flows, &alt_text_flow_id, "alt_text")?; + replace_text(&alt_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?, image.alt_text.as_ref())?; + if let Some(caption) = &image.caption { + let caption_flow_id = nested_flow_id("image_caption", &durable_block_id); + block.insert("caption_flow_id", caption_flow_id.as_str())?; + let caption_flow = ensure_flow(flows, &caption_flow_id, "caption")?; + let caption_text = caption_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + let mut caption_plan = FlowTextImportPlan::new(1, caption.runs.len().saturating_add(1)); + caption_plan.push_paragraph(caption, ""); + caption_plan.write_to(None, &caption_text)?; + } + let attrs = block.ensure_mergeable_map("attrs")?; + attrs.insert("alignment", alignment_name(image.alignment))?; + match image.sizing { + ImageSizing::Intrinsic => attrs.insert("sizing", "intrinsic")?, + ImageSizing::FitWidth => attrs.insert("sizing", "fit_width")?, + ImageSizing::Fixed { width_px, height_px } => { + attrs.insert("sizing", "fixed")?; + attrs.insert("width_px", i64::from(width_px))?; + if let Some(height_px) = height_px { + attrs.insert("height_px", i64::from(height_px))?; + } + }, + }; + Ok(()) +} + +fn import_sections( + document: &DocumentProjection, + sections: &LoroMap, + flows: &LoroMap, + body_text: &LoroText, + paragraph_plans: &[ParagraphTextImportPlan], +) -> LoroResult<()> { + let paragraph_indexes = document + .ids + .paragraph_ids + .iter() + .enumerate() + .map(|(ix, id)| (*id, ix)) + .collect::>(); + for section in document.sections.iter() { + let section_id = section.id.0.to_string(); + let section_map = sections.ensure_mergeable_map(§ion_id)?; + section_map.insert("id", section_id.as_str())?; + section_map.insert("start_paragraph_id", section.start_paragraph.0.to_string())?; + if let Some(parent_id) = section.parent_id { + section_map.insert("parent_section_id", parent_id.0.to_string())?; + } + if let Some(heading_id) = section.heading_paragraph { + section_map.insert("heading_paragraph_id", heading_id.0.to_string())?; + } + if let Some(end_id) = section.end_paragraph_exclusive { + section_map.insert("end_paragraph_exclusive_id", end_id.0.to_string())?; + } + let gpui_flowtext::SectionKind::Custom(kind_slot) = section.kind; + section_map.insert("kind_slot", i64::from(kind_slot))?; + if let Some(paragraph_ix) = paragraph_indexes.get(§ion.start_paragraph).copied() + && let Some(boundary_pos) = paragraph_plans + .get(paragraph_ix) + .map(|paragraph| paragraph.boundary_pos) + && let Some(cursor) = body_text.get_cursor(boundary_pos, Side::Left) + { + section_map.insert("start_cursor", cursor.encode())?; + } + let attrs = section_map.ensure_mergeable_map("attrs")?; + attrs.insert("source", "paragraph_style_outline")?; + // §11/§31: persist this section's page-structure attrs so they round-trip + // losslessly through Loro. When the projection carries them on + // `DocumentSection::page`, map them field-for-field onto the canonical Loro + // mirror and write them (the writer also creates any referenced + // header/footer flows via the `flows` map). Page-less sections fall back to + // the documented defaults (US Letter, 1-inch margins, 1 column, portrait, + // no numbering, no header/footer), matching the read path which always + // projects `Some(..)`. + let page_attrs = section + .page + .as_ref() + .map_or_else(SectionPageAttrs::default, section_page_attrs_to_loro); + write_section_page_attrs(&attrs, flows, &page_attrs)?; + } + Ok(()) +} + +/// §11: map a section's gpui-flowtext page-structure attrs onto the canonical +/// Loro mirror (`crate::loro_schema::SectionPageAttrs`) for the import write +/// path. This is the exact inverse of `loro_projection`'s +/// `project_section_page_attrs` read mapping; fully-qualified paths disambiguate +/// the structurally-identical `gpui_flowtext` and `crate::loro_schema` type +/// names. The owned header/footer flow id strings are cloned because the source +/// section is borrowed from the projection. +fn section_page_attrs_to_loro(page: &gpui_flowtext::SectionPageAttrs) -> SectionPageAttrs { + SectionPageAttrs { + page_size: crate::loro_schema::SectionPageSize { + width_twips: page.page_size.width_twips, + height_twips: page.page_size.height_twips, + }, + margins: crate::loro_schema::SectionMargins { + top_twips: page.margins.top_twips, + right_twips: page.margins.right_twips, + bottom_twips: page.margins.bottom_twips, + left_twips: page.margins.left_twips, + }, + columns: page.columns, + orientation: match page.orientation { + gpui_flowtext::SectionOrientation::Portrait => crate::loro_schema::SectionOrientation::Portrait, + gpui_flowtext::SectionOrientation::Landscape => crate::loro_schema::SectionOrientation::Landscape, + }, + page_numbering: crate::loro_schema::SectionPageNumbering { + format: match page.page_numbering.format { + gpui_flowtext::PageNumberFormat::None => crate::loro_schema::PageNumberFormat::None, + gpui_flowtext::PageNumberFormat::Decimal => crate::loro_schema::PageNumberFormat::Decimal, + gpui_flowtext::PageNumberFormat::LowerRoman => crate::loro_schema::PageNumberFormat::LowerRoman, + gpui_flowtext::PageNumberFormat::UpperRoman => crate::loro_schema::PageNumberFormat::UpperRoman, + gpui_flowtext::PageNumberFormat::LowerAlpha => crate::loro_schema::PageNumberFormat::LowerAlpha, + gpui_flowtext::PageNumberFormat::UpperAlpha => crate::loro_schema::PageNumberFormat::UpperAlpha, + }, + start: page.page_numbering.start, + }, + header_flow_id: page.header_flow_id.clone(), + footer_flow_id: page.footer_flow_id.clone(), + } +} + +#[hotpath::measure] +/// §act-twelve A12.3.2b: a body-text cursor by ARITHMETIC. The whole body is +/// ONE contiguous insert op, so the char at unicode `pos` has id +/// `(peer, base_counter + pos)` — no per-boundary chunk walk (`get_cursor` +/// was 2-3 walks per paragraph over a growing tree, the dominant CRDT-import +/// cost after the batched body write). `origin_pos` is not part of the +/// encoded form. Oracle: `FLOWSTATE_IMPORT_CURSOR_VERIFY=1` cross-checks +/// every constructed cursor against `get_cursor` (armed in the corpus +/// sweep). +fn body_cursor_at(text: &LoroText, base: Option<(u64, i32)>, pos: usize, side: Side) -> Option { + let constructed = base.map(|(peer, counter)| loro::cursor::Cursor::new(Some(loro::ID::new(peer, counter + pos as i32)), text.id(), side, pos)); + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + let verify = *VERIFY.get_or_init(|| std::env::var_os("FLOWSTATE_IMPORT_CURSOR_VERIFY").is_some()); + if constructed.is_none() || verify { + let walked = text.get_cursor(pos, side); + if let (Some(constructed), Some(walked)) = (constructed.as_ref(), walked.as_ref()) { + assert_eq!( + constructed.encode(), + walked.encode(), + "import cursor arithmetic diverged from get_cursor at pos {pos} (side {side:?}): constructed={constructed:?} walked={walked:?}" + ); + } + if constructed.is_none() { + return walked; + } + } + constructed +} + +fn import_paragraph_record( + paragraphs: &LoroMap, + blocks: &LoroMap, + flow_id: &str, + body_text: &LoroText, + text_op_base: Option<(u64, i32)>, + boundary_pos: usize, + block_id: String, + paragraph_id: String, +) -> LoroResult<()> { + let paragraph_map = hotpath::measure_block!("import_record_maps", { + let paragraph_map = paragraphs.ensure_mergeable_map(¶graph_id)?; + paragraph_map.insert("id", paragraph_id.as_str())?; + paragraph_map.insert("flow_id", flow_id)?; + paragraph_map + }); + hotpath::measure_block!("import_record_cursors", { + if let Some(cursor) = body_cursor_at(body_text, text_op_base, boundary_pos, Side::Left) { + paragraph_map.insert("start_cursor", cursor.encode())?; + } + if let Some(cursor) = body_cursor_at(body_text, text_op_base, boundary_pos, Side::Right) { + paragraph_map.insert("boundary_cursor", cursor.encode())?; + } + }); + // §perf-heaven T8.2: the `attrs` container is kept (the projection reads it and + // the invalidation whitelist expects it), but the `*_container_id` MIRROR + // values are dropped. They duplicated `map.id()` — write-only (no reader in the + // projection or collab), yet each was a long flattened-cid `String` stored as a + // map value AND minted as an op, ×5 per paragraph ×N paragraphs (a large slice + // of the 38.7 KB/record). Derivable from the container itself if ever needed. + paragraph_map.ensure_mergeable_map("attrs")?; + ensure_block(blocks, block_id, "paragraph", flow_id, body_text, text_op_base, boundary_pos)?; + Ok(()) +} + +#[derive(Clone, Copy, Debug)] +enum FlowBlockPosition { + Paragraph { boundary_pos: usize }, + Object { anchor_pos: usize }, +} + +#[derive(Clone, Debug)] +struct ParagraphTextImportPlan { + boundary_pos: usize, +} + +#[derive(Clone, Debug)] +struct FlowTextImportPlan { + delta: Vec, + unicode_len: usize, + block_positions: Vec, + paragraphs: Vec, +} + +impl FlowTextImportPlan { + fn new(block_capacity: usize, delta_capacity: usize) -> Self { + let mut delta = Vec::with_capacity(delta_capacity.max(block_capacity.saturating_add(1))); + delta.push(TextDelta::Insert { + insert: "\n".to_string(), + attributes: Some(paragraph_style_attributes(ParagraphStyle::Normal)), + }); + Self { + delta, + unicode_len: 1, + block_positions: Vec::with_capacity(block_capacity), + paragraphs: Vec::new(), + } + } + + fn for_document(document: &DocumentProjection) -> Self { + let run_count = document + .paragraphs + .iter() + .map(|paragraph| paragraph.runs.len()) + .sum::(); + let mut plan = Self::new( + document.blocks.len(), + run_count + .saturating_add(document.blocks.len()) + .saturating_add(1), + ); + let mut paragraph_ix = 0_usize; + for block in document.blocks.iter() { + match block { + Block::Paragraph(paragraph) => { + let paragraph_body = paragraph_text(document, paragraph_ix); + plan.push_paragraph(paragraph, ¶graph_body); + paragraph_ix += 1; + }, + Block::Image(_) | Block::Equation(_) | Block::Table(_) => plan.push_object(), + } + } + plan + } + + fn push_paragraph(&mut self, paragraph: &Paragraph, paragraph_body: &str) { + let boundary_pos = if self.block_positions.is_empty() { + self.set_initial_paragraph_style(paragraph.style); + 0 + } else { + let boundary_pos = self.unicode_len; + push_rich_text_insert(&mut self.delta, "\n", Some(paragraph_style_attributes(paragraph.style))); + self.unicode_len += 1; + boundary_pos + }; + + self.push_paragraph_body(paragraph_body, ¶graph.runs); + self + .paragraphs + .push(ParagraphTextImportPlan { boundary_pos }); + self + .block_positions + .push(FlowBlockPosition::Paragraph { boundary_pos }); + } + + fn set_initial_paragraph_style(&mut self, style: ParagraphStyle) { + let Some(TextDelta::Insert { insert, attributes }) = self.delta.first_mut() else { + unreachable!("flow import plan always starts with a sentinel newline"); + }; + debug_assert_eq!(insert, "\n"); + *attributes = Some(paragraph_style_attributes(style)); + } + + fn push_paragraph_body(&mut self, paragraph_body: &str, runs: &[gpui_flowtext::TextRun]) { + let mut byte_offset = 0_usize; + for run in runs { + let byte_end = byte_offset.saturating_add(run.len); + if byte_end > paragraph_body.len() || !paragraph_body.is_char_boundary(byte_offset) || !paragraph_body.is_char_boundary(byte_end) { + break; + } + push_rich_text_insert(&mut self.delta, ¶graph_body[byte_offset..byte_end], run_style_attributes(run.styles)); + byte_offset = byte_end; + } + if byte_offset < paragraph_body.len() && paragraph_body.is_char_boundary(byte_offset) { + push_rich_text_insert(&mut self.delta, ¶graph_body[byte_offset..], None); + } + self.unicode_len += paragraph_body.chars().count(); + } + + fn push_object(&mut self) { + let anchor_pos = self.unicode_len; + let object = OBJECT_REPLACEMENT.to_string(); + push_rich_text_insert(&mut self.delta, &object, None); + self.unicode_len += 1; + self + .block_positions + .push(FlowBlockPosition::Object { anchor_pos }); + } + + /// `doc` enables the batched mark path (needs the inner-doc handle); the + /// tiny nested flows (captions, table cells) pass `None` and mark per run. + /// Returns the `(peer, start_counter)` of the single contiguous body text + /// op when a doc handle is supplied — every in-body position's cursor is + /// then pure arithmetic off it (§act-twelve A12.3.2b). + fn write_to(&self, doc: Option<&LoroDoc>, text: &LoroText) -> LoroResult> { + let len = text.len_unicode(); + if len > 0 { + text.delete(0, len)?; + } + // §perf (act three D.1/D.2): ONE contiguous insert op for the whole flow + // text, then explicit merged style-mark ranges. The former `apply_delta` + // minted one insert op (one op-id span) PER STYLE RUN — ~128k spans on + // the reference doc — which fragmented every later whole-range operation + // (a select-all delete became 128k delete ops, ~640 ms) and paid per-span + // tree insertion at build time. Marks are anchor ops over the contiguous + // text and do not fragment text ids; the end state (read back via + // `to_delta`/the style tree) is identical because `apply_delta` itself + // resolves attributes into `mark` calls after inserting. + let mut full_text = String::with_capacity( + self + .delta + .iter() + .map(|span| span_text(span).map_or(0, str::len)) + .sum(), + ); + for span in &self.delta { + if let Some(insert) = span_text(span) { + full_text.push_str(insert); + } + } + // §act-twelve A12.3.2b: capture the (peer, counter) the contiguous insert + // starts at — pending-txn ops extend the peer's oplog counter linearly. + let text_op_base = doc.map(|doc| { + let peer = doc.peer_id(); + // `oplog_vv` already reflects ops applied inside the open transaction, + // so the next op's counter is exactly the vv entry (verified against + // `get_cursor` by the FLOWSTATE_IMPORT_CURSOR_VERIFY oracle). + (peer, doc.oplog_vv().get(&peer).copied().unwrap_or(0)) + }); + static POP_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + let pop_probe = *POP_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_POPULATE_PROBE").is_some()); + let probe_t = std::time::Instant::now(); + hotpath::measure_block!("import_body_single_insert", text.insert(0, &full_text)?); + if pop_probe { + eprintln!( + "[flowstate-populate-probe] insert={:?} chars={} spans={}", + probe_t.elapsed(), + full_text.chars().count(), + self.delta.len() + ); + } + let probe_t = std::time::Instant::now(); + + // Per-key run merging: adjacent spans that share one (key, value) pair + // become a single mark op even when their full attribute maps differ. + let mut runs_by_key: FxHashMap<&str, Vec> = FxHashMap::default(); + let mut unicode_pos = 0usize; + for span in &self.delta { + let TextDelta::Insert { insert, attributes } = span else { + continue; + }; + let span_len = insert.chars().count(); + if let Some(attributes) = attributes { + for (key, value) in attributes { + let runs = runs_by_key.entry(key.as_str()).or_default(); + match runs.last_mut() { + Some(last) if last.end == unicode_pos && last.value == *value => last.end = unicode_pos + span_len, + _ => runs.push(MarkRun { + start: unicode_pos, + end: unicode_pos + span_len, + value: value.clone(), + }), + } + } + } + unicode_pos += span_len; + } + hotpath::measure_block!("import_body_marks", { + match doc { + // Batched mark application (vendored `mark_batch`): one transaction + // ceremony for the whole style set instead of two per mark — the mark + // pass dominated the body write (~730 ms) once the text became a + // single insert. + Some(doc) => { + let mut marks: Vec<(usize, usize, loro::InternalString, LoroValue)> = Vec::new(); + for (key, runs) in runs_by_key { + for run in runs { + marks.push((run.start, run.end, key.into(), run.value)); + } + } + if pop_probe { + eprintln!("[flowstate-populate-probe] mark_count={}", marks.len()); + } + doc + .inner() + .get_text(text.id()) + .mark_batch(marks, loro::cursor::PosType::Unicode)?; + }, + None => { + for (key, runs) in runs_by_key { + for run in runs { + text.mark(run.start..run.end, key, run.value)?; + } + } + }, + } + }); + if pop_probe { + eprintln!("[flowstate-populate-probe] marks={:?}", probe_t.elapsed()); + } + Ok(text_op_base) + } +} + +struct MarkRun { + start: usize, + end: usize, + value: LoroValue, +} + +fn span_text(span: &TextDelta) -> Option<&str> { + match span { + TextDelta::Insert { insert, .. } => Some(insert.as_str()), + _ => None, + } +} + +fn paragraph_style_attributes(style: ParagraphStyle) -> FxHashMap { + let mut attributes = FxHashMap::default(); + attributes.insert(MARK_PARAGRAPH_STYLE.to_string(), paragraph_style_value(style).into()); + attributes +} + +fn run_style_attributes(styles: RunStyles) -> Option> { + let mut attributes = FxHashMap::default(); + if let RunSemanticStyle::Custom(slot) = styles.semantic { + attributes.insert(MARK_RUN_SEMANTIC_STYLE.to_string(), i64::from(slot).into()); + } + if let Some(HighlightStyle::Custom(slot)) = styles.highlight { + attributes.insert(MARK_HIGHLIGHT_STYLE.to_string(), i64::from(slot).into()); + } + if styles.direct_underline { + attributes.insert(MARK_DIRECT_UNDERLINE.to_string(), true.into()); + } + if styles.strikethrough { + attributes.insert(MARK_STRIKETHROUGH.to_string(), true.into()); + } + if let Some(value) = styles.vert_align.mark_value() { + attributes.insert(MARK_VERT_ALIGN.to_string(), value.into()); + } + (!attributes.is_empty()).then_some(attributes) +} + +fn push_rich_text_insert(delta: &mut Vec, value: &str, attributes: Option>) { + if value.is_empty() { + return; + } + if let Some(TextDelta::Insert { + insert, + attributes: previous_attributes, + }) = delta.last_mut() + && previous_attributes.as_ref() == attributes.as_ref() + { + insert.push_str(value); + return; + } + delta.push(TextDelta::Insert { + insert: value.to_string(), + attributes, + }); +} + +fn import_table(flows: &LoroMap, block: &LoroMap, table: &TableBlock) -> LoroResult<()> { + let table_map = block.ensure_mergeable_map(TABLE_KEY)?; + let row_order = table_map.ensure_mergeable_movable_list(TABLE_ROW_ORDER)?; + let column_order = table_map.ensure_mergeable_movable_list(TABLE_COLUMN_ORDER)?; + let rows_by_id = table_map.ensure_mergeable_map(TABLE_ROWS_BY_ID)?; + let columns_by_id = table_map.ensure_mergeable_map(TABLE_COLUMNS_BY_ID)?; + let cells_by_id = table_map.ensure_mergeable_map(TABLE_CELLS_BY_ID)?; + table_map.insert("columns_container_id", columns_by_id.id().to_string())?; + table_map.insert("cells_container_id", cells_by_id.id().to_string())?; + table_map.insert("header_row", table.style.header_row)?; + + // §P2b create-only: import builds a brand-new table, but it uses the SAME + // durable-id `ensure` scheme as the incremental edit path (never clear/rekey), + // so row/column/cell identity is stable and concurrent creation of the same id + // merges (LWW) instead of duplicating. + for column in &table.columns { + let column_id = column_loro_id(column.id); + column_order.push(column_id.as_str())?; + let column_map = columns_by_id.ensure_mergeable_map(&column_id)?; + column_map.insert("id", column_id.as_str())?; + let _attrs = column_map.ensure_mergeable_map("attrs")?; + match column.width { + TableColumnWidth::Auto => column_map.insert("width_kind", "auto")?, + TableColumnWidth::FixedPx(px) => { + column_map.insert("width_kind", "fixed_px")?; + column_map.insert("width_px", i64::from(px))?; + }, + TableColumnWidth::Fraction(fraction) => { + column_map.insert("width_kind", "fraction")?; + column_map.insert("fraction", i64::from(fraction))?; + }, + }; + } + + for row in &table.rows { + let row_id = row_loro_id(row.id); + row_order.push(row_id.as_str())?; + let row_map = rows_by_id.ensure_mergeable_map(&row_id)?; + row_map.insert("id", row_id.as_str())?; + let _attrs = row_map.ensure_mergeable_map("attrs")?; + for cell in &row.cells { + let cell_id = cell_loro_id(cell.id); + let column_id = column_loro_id(cell.column_id); + let cell_row_id = row_loro_id(cell.row_id); + let cell_map = cells_by_id.ensure_mergeable_map(&cell_id)?; + cell_map.insert("id", cell_id.as_str())?; + cell_map.insert("row_id", cell_row_id.as_str())?; + cell_map.insert("column_id", column_id.as_str())?; + cell_map.insert("row_span", i64::from(cell.row_span))?; + cell_map.insert("column_span", i64::from(cell.col_span))?; + let _attrs = cell_map.ensure_mergeable_map("attrs")?; + let flow_id = cell_flow_loro_id(&cell_id); + cell_map.insert("flow_id", flow_id.as_str())?; + let nested_table_ids = cell_map.ensure_mergeable_movable_list("nested_table_ids")?; + let nested_tables_by_id = cell_map.ensure_mergeable_map("nested_tables_by_id")?; + clear_movable_list(&nested_table_ids)?; + clear_map(&nested_tables_by_id)?; + let flow = ensure_flow(flows, &flow_id, "table_cell")?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + cell_map.insert("text_container_id", text.id().to_string())?; + let cell_delta_capacity = cell + .blocks + .iter() + .map(|block| match block { + TableCellBlock::Paragraph(paragraph) => paragraph.paragraph.runs.len().saturating_add(1), + TableCellBlock::Table(_) => 1, + }) + .sum(); + let mut cell_plan = FlowTextImportPlan::new(cell.blocks.len(), cell_delta_capacity); + for cell_block in &cell.blocks { + match cell_block { + TableCellBlock::Paragraph(paragraph) => cell_plan.push_paragraph(¶graph.paragraph, ¶graph.text), + TableCellBlock::Table(_) => cell_plan.push_object(), + } + } + cell_plan.write_to(None, &text)?; + for (block_ix, (cell_block, position)) in cell + .blocks + .iter() + .zip(&cell_plan.block_positions) + .enumerate() + { + let (TableCellBlock::Table(nested), FlowBlockPosition::Object { anchor_pos }) = (cell_block, position) else { + continue; + }; + let nested_table_id = format!("{cell_id}.nested_table.{block_ix}"); + nested_table_ids.push(nested_table_id.as_str())?; + let nested_map = nested_tables_by_id.ensure_mergeable_map(&nested_table_id)?; + nested_map.insert("id", nested_table_id.as_str())?; + nested_map.insert("kind", "table")?; + if let Some(cursor) = text.get_cursor(*anchor_pos, Side::Left) { + nested_map.insert("anchor_cursor", cursor.encode())?; + } + nested_map.ensure_mergeable_map("attrs")?; + import_table(flows, &nested_map, nested)?; + } + } + } + Ok(()) +} + +pub fn import_assets(doc: &LoroDoc, document: &DocumentProjection) -> LoroResult<()> { + let root = doc.get_map(ROOT); + let assets = root.ensure_mergeable_map(ASSETS_BY_ID)?; + clear_map(&assets)?; + for asset in document.assets.assets.values() { + let asset_id = asset.asset_id_string(); + let asset_map = assets.ensure_mergeable_map(&asset_id)?; + let hash = blake3::hash(&asset.bytes); + asset_map.insert("asset_id", asset_id.as_str())?; + asset_map.insert("content_hash", hash.to_hex().as_str())?; + asset_map.insert("mime_type", asset.mime_type.as_ref())?; + asset_map.insert("byte_length", i64::try_from(asset.bytes.len()).unwrap_or(i64::MAX))?; + if let Some((width, height)) = image_dimensions(asset.mime_type.as_ref(), &asset.bytes) { + // §31/§14: canonical AssetMap dimensions, stored as two i64 keys when derivable. + asset_map.insert("dimension_width", width)?; + asset_map.insert("dimension_height", height)?; + } + if let Some(original_name) = &asset.original_name { + asset_map.insert("original_name", original_name.as_ref())?; + } + } + Ok(()) +} + +/// Derive an image asset's intrinsic pixel dimensions from its bytes. +/// +/// Returns `None` for non-image assets or when the size cannot be determined, +/// in which case the asset map simply omits its `dimension_*` keys (§14). +fn image_dimensions(mime_type: &str, bytes: &[u8]) -> Option<(i64, i64)> { + if !mime_type.starts_with("image/") { + return None; + } + let size = imagesize::blob_size(bytes).ok()?; + Some((i64::try_from(size.width).ok()?, i64::try_from(size.height).ok()?)) +} + +pub fn assets_from_document(document: &DocumentProjection) -> Vec { + document + .assets + .assets + .values() + .map(|asset| AssetChunk { + asset_id: asset.id.0, + content_hash: *blake3::hash(&asset.bytes).as_bytes(), + mime_type: asset.mime_type.to_string(), + byte_length: asset.bytes.len() as u64, + bytes: Vec::clone(&asset.bytes), + metadata: Vec::new(), + }) + .collect() +} + +trait AssetRecordExt { + fn asset_id_string(&self) -> String; +} + +impl AssetRecordExt for gpui_flowtext::AssetRecord { + fn asset_id_string(&self) -> String { + self.id.0.to_string() + } +} + +fn ensure_flow(flows: &LoroMap, flow_id: &str, kind: &str) -> LoroResult { + let flow = flows.ensure_mergeable_map(flow_id)?; + flow.insert(FLOW_ID_KEY, flow_id)?; + flow.insert(FLOW_KIND_KEY, kind)?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + let _attrs = flow.ensure_mergeable_map(FLOW_ATTRS_KEY)?; + flow.insert("text_container_id", text.id().to_string())?; + Ok(flow) +} + +fn ensure_block( + blocks: &LoroMap, + block_id: String, + kind: &str, + flow_id: &str, + text: &LoroText, + text_op_base: Option<(u64, i32)>, + pos: usize, +) -> LoroResult { + let block = blocks.ensure_mergeable_map(&block_id)?; + block.insert("id", block_id.as_str())?; + block.insert("kind", kind)?; + block.insert("flow_id", flow_id)?; + if let Some(cursor) = body_cursor_at(text, text_op_base, pos, Side::Left) { + block.insert("anchor_cursor", cursor.encode())?; + } + // §perf-heaven T8.2: keep the `attrs`/`nested_refs` containers (read + whitelisted), + // drop the write-only `*_container_id` mirror strings (duplicated `map.id()`). + block.ensure_mergeable_map("attrs")?; + block.ensure_mergeable_map("nested_refs")?; + Ok(block) +} + +fn replace_text(text: &LoroText, value: &str) -> LoroResult<()> { + let len = text.len_unicode(); + if len > 0 { + text.delete(0, len)?; + } + if !value.is_empty() { + text.insert(0, value)?; + } + Ok(()) +} + +fn clear_map(map: &LoroMap) -> LoroResult<()> { + // §29: use Loro's native container clear instead of deleting keys one-by-one. + map.clear() +} + +fn clear_movable_list(list: &LoroMovableList) -> LoroResult<()> { + list.clear() +} + +fn paragraph_style_value(style: ParagraphStyle) -> i64 { + match style { + ParagraphStyle::Normal => 0, + ParagraphStyle::Custom(slot) => i64::from(slot) + 1, + } +} + +fn projection_block_id(document: &DocumentProjection, block_ix: usize, kind: &str) -> String { + document + .ids + .block_ids + .get(block_ix) + .map_or_else(|| fallback_id(kind, block_ix), |id| format!("{kind}.{}", id.0)) +} + +fn projection_paragraph_id(document: &DocumentProjection, paragraph_ix: usize) -> String { + document + .ids + .paragraph_ids + .get(paragraph_ix) + .map_or_else(|| fallback_id("paragraph", paragraph_ix), |id| format!("paragraph.{}", id.0)) +} + +fn fallback_id(kind: &str, ix: usize) -> String { + format!( + "{kind}.{ix}.{}", + Uuid::new_v5(&Uuid::NAMESPACE_OID, format!("{kind}.{ix}").as_bytes()).as_u128() + ) +} + +fn nested_flow_id(kind: &str, block_id: &str) -> String { + format!("{block_id}.{kind}") +} + +fn alignment_name(alignment: BlockAlignment) -> &'static str { + match alignment { + BlockAlignment::Left => "left", + BlockAlignment::Center => "center", + BlockAlignment::Right => "right", + } +} + +fn equation_syntax_name(syntax: EquationSyntax) -> &'static str { + match syntax { + EquationSyntax::Latex => "latex", + } +} + +fn equation_display_name(display: EquationDisplay) -> &'static str { + match display { + EquationDisplay::Display => "display", + EquationDisplay::InlineLikeParagraph => "inline_like_paragraph", + } +} + +fn loro_io_error(error: impl std::error::Error + Send + Sync + 'static) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn projection_identity_round_trips_into_loro() -> io::Result<()> { + let mut source = gpui_flowtext::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![gpui_flowtext::InputBlock::Paragraph(gpui_flowtext::InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![gpui_flowtext::InputRun { + text: "identity".to_string(), + styles: RunStyles::default(), + }], + })], + ); + source.ids.document_id = 0x0123; + std::sync::Arc::make_mut(&mut source.ids.paragraph_ids)[0] = gpui_flowtext::ParagraphId(0x0456); + std::sync::Arc::make_mut(&mut source.ids.block_ids)[0] = gpui_flowtext::BlockId(0x0789); + + let doc = document_to_loro(&source, "Identity")?; + let projected = crate::document_from_loro(&doc)?; + + assert_eq!(projected.ids.document_id, source.ids.document_id); + assert_eq!(projected.ids.paragraph_ids, source.ids.paragraph_ids); + assert_eq!(projected.ids.block_ids, source.ids.block_ids); + Ok(()) + } + + #[test] + fn custom_paragraph_style_slot_zero_round_trips() -> io::Result<()> { + let source = gpui_flowtext::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![gpui_flowtext::InputBlock::Paragraph(gpui_flowtext::InputParagraph { + style: ParagraphStyle::Custom(0), + runs: vec![gpui_flowtext::InputRun { + text: "pocket".to_string(), + styles: RunStyles::default(), + }], + })], + ); + + let doc = document_to_loro(&source, "Pocket")?; + let projected = crate::document_from_loro(&doc)?; + assert_eq!(projected.paragraphs[0].style, ParagraphStyle::Custom(0)); + Ok(()) + } + + #[test] + fn bulk_import_preserves_empty_paragraphs_unicode_and_frontier() -> io::Result<()> { + let paragraphs = vec![ + DocumentParagraphInput { + style: ParagraphStyle::Custom(0), + runs: Vec::new(), + }, + DocumentParagraphInput { + style: ParagraphStyle::Normal, + runs: Vec::new(), + }, + DocumentParagraphInput { + style: ParagraphStyle::Custom(2), + runs: vec![gpui_flowtext::DocumentRunInput { + text: "héllo 世界".to_string(), + styles: RunStyles { + semantic: RunSemanticStyle::Custom(3), + direct_underline: true, + ..RunStyles::default() + }, + }], + }, + ]; + + let imported = import_paragraphs_as_loro(DocumentTheme::default(), paragraphs, "Bulk import")?; + assert_eq!(imported.projection.frontier, imported.doc.state_frontiers().encode()); + assert_eq!(crate::loro_schema::body_text(&imported.doc).to_string(), "\n\n\nhéllo 世界"); + + let projected = crate::document_from_loro(&imported.doc)?; + assert_eq!(projected.paragraphs.len(), 3); + assert_eq!(paragraph_text(&projected, 0), ""); + assert_eq!(paragraph_text(&projected, 1), ""); + assert_eq!(paragraph_text(&projected, 2), "héllo 世界"); + assert_eq!(projected.paragraphs[0].style, ParagraphStyle::Custom(0)); + assert_eq!(projected.paragraphs[2].style, ParagraphStyle::Custom(2)); + assert_eq!(projected.paragraphs[2].runs[0].styles.semantic, RunSemanticStyle::Custom(3)); + assert!(projected.paragraphs[2].runs[0].styles.direct_underline); + Ok(()) + } + + #[test] + fn bulk_delta_import_preserves_all_semantic_run_attributes() -> io::Result<()> { + let expected = RunStyles { + semantic: RunSemanticStyle::Custom(3), + direct_underline: true, + strikethrough: true, + highlight: Some(HighlightStyle::Custom(4)), + // Superscript rides the same import→mark→projection round-trip as the other + // orthogonal run attributes; the `== expected` assertion below verifies it. + vert_align: gpui_flowtext::VertAlign::Superscript, + }; + let imported = import_paragraphs_as_loro( + crate::flowstate_document_theme(), + vec![DocumentParagraphInput { + style: ParagraphStyle::Custom(2), + runs: vec![ + gpui_flowtext::DocumentRunInput { + text: "styled".to_string(), + styles: expected, + }, + gpui_flowtext::DocumentRunInput { + text: " plain".to_string(), + styles: RunStyles::default(), + }, + ], + }], + "Rich delta import", + )?; + + let projected = crate::document_from_loro(&imported.doc)?; + assert_eq!(paragraph_text(&projected, 0), "styled plain"); + assert_eq!(projected.paragraphs[0].style, ParagraphStyle::Custom(2)); + assert!( + projected.paragraphs[0] + .runs + .iter() + .any(|run| run.styles == expected) + ); + Ok(()) + } + + #[test] + fn bulk_import_handles_large_paragraph_sets_without_reprojection() -> io::Result<()> { + let paragraphs = (0..2_000) + .map(|ix| DocumentParagraphInput { + style: if ix % 11 == 0 { + ParagraphStyle::Custom(1) + } else { + ParagraphStyle::Normal + }, + runs: vec![gpui_flowtext::DocumentRunInput { + text: format!("paragraph {ix}"), + styles: RunStyles::default(), + }], + }) + .collect(); + + let imported = import_paragraphs_as_loro(DocumentTheme::default(), paragraphs, "Large import")?; + assert_eq!(imported.projection.paragraphs.len(), 2_000); + assert_eq!(imported.projection.frontier, imported.doc.state_frontiers().encode()); + assert_eq!(paragraph_text(&imported.projection, 1_999), "paragraph 1999"); + Ok(()) + } + + #[test] + fn large_document_materializes_without_quadratic_boundary_scan() -> io::Result<()> { + // Regression for the large-document collab/editor hang. Unlike the import path + // above (which reuses its freshly-built projection), `document_from_loro` + // RE-derives the projection from the canonical doc, resolving a durable + // paragraph + paragraph-block id for every boundary. That resolution formerly + // rescanned every paragraph/block record per boundary and materialized the + // whole body string per record — O(paragraphs²·chars) — which pegged the CRDT + // actor thread at 100% CPU and never returned. With the one-pass boundary index + // it is ~linear; this test projecting a 2,000-paragraph doc to completion is the + // guard (reintroducing the quadratic would hang the suite), and matching the + // import projection's ids proves the index selects the same id the per-boundary + // scan did, at scale. + let paragraphs = (0..2_000) + .map(|ix| DocumentParagraphInput { + style: if ix % 11 == 0 { + ParagraphStyle::Custom(1) + } else { + ParagraphStyle::Normal + }, + runs: vec![gpui_flowtext::DocumentRunInput { + text: format!("paragraph {ix}"), + styles: RunStyles::default(), + }], + }) + .collect(); + let imported = import_paragraphs_as_loro(DocumentTheme::default(), paragraphs, "Large projection")?; + + let projected = crate::loro_projection::document_from_loro(&imported.doc)?; + + assert_eq!(projected.paragraphs.len(), 2_000); + assert_eq!(paragraph_text(&projected, 1_999), "paragraph 1999"); + // Re-derived ids match the import projection's ids (the one-pass index resolves + // every boundary to the identical durable id the old per-boundary scan did)... + assert_eq!(projected.ids.paragraph_ids, imported.projection.ids.paragraph_ids); + assert_eq!(projected.ids.block_ids, imported.projection.ids.block_ids); + // ...and every boundary resolved to a distinct durable id (no collisions, no + // fabricated fallbacks from a mis-keyed index). + let distinct_paragraph_ids = projected + .ids + .paragraph_ids + .iter() + .map(|id| id.0) + .collect::>(); + assert_eq!( + distinct_paragraph_ids.len(), + 2_000, + "every boundary must resolve to a unique paragraph id" + ); + Ok(()) + } + + #[test] + fn section_page_attrs_round_trip_through_import_write_path() -> io::Result<()> { + let mut source = gpui_flowtext::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![gpui_flowtext::InputBlock::Paragraph(gpui_flowtext::InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![gpui_flowtext::InputRun { + text: "section body".to_string(), + styles: RunStyles::default(), + }], + })], + ); + // §11: non-default page structure on the source section must survive the + // import write path and be projected back identically. + let page = gpui_flowtext::SectionPageAttrs { + page_size: gpui_flowtext::SectionPageSize { + width_twips: 15_840, + height_twips: 12_240, + }, + margins: gpui_flowtext::SectionMargins { + top_twips: 720, + right_twips: 720, + bottom_twips: 1_440, + left_twips: 1_440, + }, + columns: 2, + orientation: gpui_flowtext::SectionOrientation::Landscape, + page_numbering: gpui_flowtext::SectionPageNumbering { + format: gpui_flowtext::PageNumberFormat::LowerRoman, + start: 3, + }, + header_flow_id: Some("section.s1.header".to_string()), + footer_flow_id: Some("section.s1.footer".to_string()), + }; + source.sections = std::sync::Arc::new(vec![gpui_flowtext::DocumentSection { + id: gpui_flowtext::SectionId(0x5ec1), + parent_id: None, + kind: gpui_flowtext::SectionKind::Custom(0), + heading_paragraph: None, + start_paragraph: source.ids.paragraph_ids[0], + end_paragraph_exclusive: None, + page: Some(page.clone()), + }]); + + let doc = document_to_loro(&source, "Section page attrs")?; + let projected = crate::document_from_loro(&doc)?; + + assert_eq!(projected.sections.len(), 1); + assert_eq!(projected.sections[0].page, Some(page)); + Ok(()) + } +} diff --git a/crates/flowstate-document/src/loro_projection.rs b/crates/flowstate-document/src/loro_projection.rs new file mode 100644 index 00000000..1219947e --- /dev/null +++ b/crates/flowstate-document/src/loro_projection.rs @@ -0,0 +1,2351 @@ +use std::{collections::BTreeMap, io, sync::Arc}; + +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use gpui_flowtext::{ + AssetId, BlockId, CellId, DocumentProjection, DocumentSection, DocumentTheme, HighlightStyle, InputBlock, InputBlockAlignment, + InputEquationBlock, InputEquationDisplay, InputEquationSyntax, InputImageBlock, InputImageSizing, InputParagraph, InputRun, InputTableBlock, + InputTableCell, InputTableCellBlock, InputTableColumn, InputTableColumnWidth, InputTableRow, InputTableStyle, ParagraphId, RunSemanticStyle, + RunStyles, SectionId, SectionKind, document_from_input_blocks, +}; +use loro::{ + Container, ContainerID, ContainerTrait, ID, LoroDoc, LoroMap, LoroText, LoroValue, ValueOrContainer, + cursor::{Cursor, Side}, +}; +use rustc_hash::FxHashMap; + +use crate::{ + BLOCKS_BY_ID, FLOW_TEXT_KEY, FLOWS_BY_ID, MAIN_BODY_BLOCK_ID, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_PARAGRAPH_STYLE, + MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH, MARK_VERT_ALIGN, OBJECT_REPLACEMENT, PARAGRAPHS_BY_ID, ROOT, ROOT_BODY_FLOW_ID, + ROOT_FIRST_PARAGRAPH_ID, SECTIONS_BY_ID, TABLE_CELLS_BY_ID, TABLE_COLUMN_ORDER, TABLE_COLUMNS_BY_ID, TABLE_KEY, TABLE_ROW_ORDER, + flowstate_document_theme, parse_cell_loro_id, parse_column_loro_id, parse_row_loro_id, + projection_defects::{ProjectionDefect, TableTopologyKind}, + table_topology, +}; + +// §perf-heaven T7.24: object-anchor validation resolves positions from +// `query_text_id_positions(use_event_index = true)` and then checks them with +// `char_at`, which treats the index as UNICODE. That identity (event index == +// unicode index) holds only on non-wasm Loro builds; on wasm the event index is +// UTF-16 and the object anchors would silently misresolve. Flowstate is a native +// desktop app — fail loudly at compile time if that ever changes, rather than +// shipping a subtly wrong projection. +#[cfg(target_arch = "wasm32")] +compile_error!("loro_projection object-anchor validation assumes non-wasm event indexing (event index == unicode index)"); + +pub fn document_from_loro(doc: &LoroDoc) -> io::Result { + // Callers that cannot repair still get the deterministic projection; the + // defects are simply discarded here (the runtime uses the sibling below). + Ok(document_from_loro_with_defects(doc)?.0) +} + +/// §5: project the document and report every malformed-canonical-state defect +/// encountered on the way. The projection is deterministic even when defective: +/// unresolvable blocks are quarantined (appended in stable order) instead of +/// dropped, and fabricated identities are deterministic per projection. +#[hotpath::measure] +pub fn document_from_loro_with_defects(doc: &LoroDoc) -> io::Result<(DocumentProjection, Vec)> { + crate::instrument::record_full_projection(); + let mut defects = Vec::new(); + let projection = projection_from_loro_with_defects(doc, &mut defects)?; + let mut document = document_from_projection_blocks(projection); + document.frontier = doc.state_frontiers().encode(); + Ok((document, defects)) +} + +/// Rows produced by a REGIONAL rematerialization (spec §6-R): the same +/// materializer law as [`document_from_loro`], applied to one body region. +pub struct RegionRows { + pub blocks: Vec, + pub paragraph_ids: Vec, + pub block_ids: Vec, + pub defects: Vec, +} + +/// Spec §6-R: materialize the body rows covering `[sentinel_unicode, end_unicode)` +/// — a region that STARTS at a row's leading boundary sentinel (`\n`, or the seed +/// sentinel at 0) and ENDS exclusively at the next retained row's leading sentinel +/// (or the body end). Runs the SAME flow walk as the full materialization +/// ([`document_from_loro`]) over a `slice_delta` of the region, so coalescing, +/// style defaults, fabrication, and defect reporting are one law, not a copy. +/// +/// Callers supply the identity context, all keyed by ABSOLUTE flow positions: +/// boundary→record-key maps for the candidate paragraph/paragraph-block records +/// and the region's resolved object blocks. Quarantine append and the empty-doc +/// placeholder are full-rebuild concerns and intentionally do not apply here. +#[allow( + clippy::implicit_hasher, + reason = "the maps are shared with the internal flow walker, whose boundary indexes are FxHashMap by construction" +)] +pub fn materialize_body_region( + doc: &LoroDoc, + sentinel_unicode: usize, + end_unicode: usize, + paragraph_ids_by_boundary: &FxHashMap, + paragraph_block_ids_by_boundary: &FxHashMap, + object_blocks_by_pos: &BTreeMap, +) -> io::Result { + let projector = Projector::new(doc)?; + let body = projector.flow_text(ROOT_BODY_FLOW_ID)?; + let end = end_unicode.min(body.len_unicode()); + if sentinel_unicode >= end { + return Err(invalid("regional rematerialization given an empty region")); + } + let delta = body + .slice_delta(sentinel_unicode, end, loro::cursor::PosType::Unicode) + .map_err(|error| invalid(format!("regional slice_delta failed: {error}")))?; + let mut blocks = Vec::new(); + let mut paragraph_ids = Vec::new(); + let mut block_ids = Vec::new(); + let mut defects = Vec::new(); + Projector::walk_flow_delta( + &body, + delta, + sentinel_unicode, + object_blocks_by_pos, + ROOT_BODY_FLOW_ID, + Some(paragraph_ids_by_boundary), + Some(paragraph_block_ids_by_boundary), + Some(&mut paragraph_ids), + Some(&mut block_ids), + &mut blocks, + &mut defects, + false, + |block, defects| projector.object_block(block, defects), + )?; + Ok(RegionRows { + blocks, + paragraph_ids, + block_ids, + defects, + }) +} + +/// §act-four M4 (cold viewport load): materialize just the body rows covering +/// `[start_unicode, end_unicode)` from a cold-loaded document, WITHOUT building +/// the whole projection. Content materialization is `O(viewport)` (the §6-R +/// `slice_delta` region walk); the boundary→id maps reuse the SAME assembly the +/// full [`document_from_loro`] uses, so the output is byte-identical to the +/// corresponding slice of the full rebuild. `start_unicode` is snapped DOWN to +/// the nearest row-leading boundary (the region walk must start at a sentinel); +/// pass `[0, body_len)` for the whole doc, or a viewport for scroll. +pub fn materialize_viewport(doc: &LoroDoc, start_unicode: usize, end_unicode: usize) -> io::Result { + let projector = Projector::new(doc)?; + let body = projector.flow_text(ROOT_BODY_FLOW_ID)?; + // Metadata-only maps (no content materialization) — the same functions the + // full projection resolves once per flow. + let paragraph_map = paragraph_ids_by_boundary(doc, &body); + let mut defects = Vec::new(); + // §perf-heaven T4: the paragraph-block boundary map comes from the SAME scan + // that finds the objects (was a separate `paragraph_block_ids_by_boundary`). + let (object_map, _quarantined, pblock_map) = projector.object_blocks_for_flow(&body, ROOT_BODY_FLOW_ID, &mut defects)?; + // Snap the start to the row-leading boundary at or before it (0 covers the + // seed sentinel), so the region walk begins at a sentinel as required. + let sentinel = paragraph_map + .keys() + .copied() + .filter(|boundary| *boundary <= start_unicode) + .max() + .unwrap_or(0); + let end = end_unicode.max(sentinel + 1).min(body.len_unicode()); + materialize_body_region(doc, sentinel, end, ¶graph_map, &pblock_map, &object_map) +} + +/// §act-four M4 (cold scroll): the body-unicode position of every block's +/// leading boundary, in block order — paragraph leading `\n`s plus object +/// U+FFFC placeholders, sorted + deduped. Persisted in the package manifest so +/// a cold open maps a block-index viewport to a unicode range in `O(1)` (no +/// per-open `O(records)` boundary scan) before calling [`materialize_viewport`]. +/// `boundaries[i]` is block `i`'s leading position; a viewport `[a, b)` decodes +/// `[boundaries[a], boundaries[b])`. `O(records)` — metadata only, no content. +pub fn body_block_boundaries(doc: &LoroDoc) -> io::Result> { + let projector = Projector::new(doc)?; + let body = projector.flow_text(ROOT_BODY_FLOW_ID)?; + let mut boundaries: Vec = paragraph_ids_by_boundary(doc, &body).into_keys().collect(); + let mut defects = Vec::new(); + let (objects, _quarantined, _paragraph_block_index) = projector.object_blocks_for_flow(&body, ROOT_BODY_FLOW_ID, &mut defects)?; + boundaries.extend(objects.keys().copied()); + boundaries.sort_unstable(); + boundaries.dedup(); + Ok( + boundaries + .into_iter() + .map(|position| u32::try_from(position).unwrap_or(u32::MAX)) + .collect(), + ) +} + +pub(crate) fn input_blocks_from_loro(doc: &LoroDoc) -> io::Result> { + Ok(projection_from_loro(doc)?.blocks) +} + +pub fn object_input_blocks_from_loro(doc: &LoroDoc) -> io::Result> { + let projector = Projector::new(doc)?; + let mut blocks = Vec::new(); + let mut defect_sink = Vec::new(); + for key in projector.blocks.keys().map(|key| key.to_string()) { + let Some(block) = child_map(&projector.blocks, &key)? else { + continue; + }; + if map_string_opt(&block, "kind")?.as_deref() == Some("paragraph") { + continue; + } + let id = map_string_opt(&block, "id")?.unwrap_or(key); + blocks.push((BlockId(loro_id_u128(&id)), projector.object_block(&block, &mut defect_sink)?)); + } + blocks.sort_by_key(|(id, _)| id.0); + Ok(blocks) +} + +/// §act-ten A10.4: the object readback, scoped to the given registry block +/// keys — a remote table-cell keystroke re-projects ONE table instead of every +/// object block in the document. Keys not present in the registry (or that are +/// paragraph records) are skipped; the caller matches ids against its own rows +/// and falls back to the full readback on any surprise. +pub fn object_input_blocks_for_ids(doc: &LoroDoc, ids: &[u128]) -> io::Result> { + let projector = Projector::new(doc)?; + let mut blocks = Vec::new(); + let mut defect_sink = Vec::new(); + for key in projector.blocks.keys().map(|key| key.to_string()) { + // Registry keys embed the durable id's trailing numeric segment — match on + // it WITHOUT materializing the block (materialization is the cost this + // scoping exists to avoid; the key iteration is a cheap map-key walk). + if !crate::loro_schema::loro_id_trailing_u128(&key).is_some_and(|id| ids.contains(&id)) { + continue; + } + let Some(block) = child_map(&projector.blocks, &key)? else { + continue; + }; + if map_string_opt(&block, "kind")?.as_deref() == Some("paragraph") { + continue; + } + let id = map_string_opt(&block, "id")?.unwrap_or(key); + blocks.push((BlockId(loro_id_u128(&id)), projector.object_block(&block, &mut defect_sink)?)); + } + blocks.sort_by_key(|(id, _)| id.0); + blocks.dedup_by_key(|(id, _)| id.0); + Ok(blocks) +} + +/// §act-ten A10.4: attribute a changed Loro container to its owning registry +/// entry by walking the container's path to the root. `Block(key)` for a +/// container under `blocks_by_id//...`; `Flow(key)` for one under +/// `flows_by_id//...`; `None` when the container cannot be attributed +/// (unparseable id, deleted container, a registry ROOT itself, or any other +/// shape) — callers must treat `None` as "fall back to the full readback". +pub fn owner_of_changed_container(doc: &LoroDoc, target: &str) -> Option { + let id = loro::ContainerID::try_from(target).ok()?; + let path = doc.inner().get_path_to_container(&id)?; + let mut segments = path.iter(); + loop { + let (_, index) = segments.next()?; + let loro::Index::Key(key) = index else { + continue; + }; + enum Registry { + Blocks, + Flows, + Paragraphs, + Replicas, + } + let registry = match key.as_ref() { + crate::loro_schema::BLOCKS_BY_ID => Registry::Blocks, + crate::loro_schema::FLOWS_BY_ID => Registry::Flows, + crate::loro_schema::PARAGRAPHS_BY_ID => Registry::Paragraphs, + crate::loro_schema::REPLICAS_BY_ID => Registry::Replicas, + _ => continue, + }; + // The registry ROOT map itself (its key set changed — e.g. a paragraph + // split registering new records): the accompanying child-record entries in + // the same invalidation carry the specifics, so the root change alone is + // object-readback-inert. + let Some((_, owner_index)) = segments.next() else { + return Some(ChangedContainerOwner::ObjectReadbackInert); + }; + let loro::Index::Key(owner_key) = owner_index else { + return None; + }; + return Some(match registry { + Registry::Blocks => ChangedContainerOwner::Block(owner_key.to_string()), + Registry::Flows => ChangedContainerOwner::Flow(owner_key.to_string()), + // Paragraph records are owned by the ranged/regional paragraph readback + // paths; replica records are presence state outside the projection. The + // full object readback skips both (it materializes object blocks only), + // so classifying them inert keeps scoped ≡ full for these shapes. + Registry::Paragraphs | Registry::Replicas => ChangedContainerOwner::ObjectReadbackInert, + }); + } +} + +/// §act-ten A10.4: owner classification for [`owner_of_changed_container`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ChangedContainerOwner { + /// A container under `blocks_by_id/`. + Block(String), + /// A container under `flows_by_id/`. + Flow(String), + /// A change the OBJECT readback provably never patches: a registry root map + /// itself, a paragraph record (`paragraphs_by_id/` — the paragraph + /// readback paths own those), or a replica/presence record. Callers scoping + /// an object readback skip these instead of falling back to the full + /// O(objects) walk (which skips them too). + ObjectReadbackInert, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct ProjectionBlocks { + #[serde(default)] + pub document_id: u128, + pub blocks: Vec, + pub paragraph_ids: Vec, + pub block_ids: Vec, + #[serde(default)] + pub sections: Vec, +} + +fn projection_from_loro(doc: &LoroDoc) -> io::Result { + let mut defects = Vec::new(); + projection_from_loro_with_defects(doc, &mut defects) +} + +fn projection_from_loro_with_defects(doc: &LoroDoc, defects: &mut Vec) -> io::Result { + let projector = Projector::new(doc)?; + projector.body_projection(defects) +} + +pub(crate) fn projection_blocks_from_loro(doc: &LoroDoc) -> io::Result { + projection_from_loro(doc) +} + +pub(crate) fn document_from_projection_blocks(projection: ProjectionBlocks) -> DocumentProjection { + let mut document = document_from_input_blocks(DocumentTheme::clone(&flowstate_document_theme()), projection.blocks); + if projection.document_id != 0 { + document.ids.document_id = projection.document_id; + } + if projection.paragraph_ids.len() == document.paragraphs.len() { + document.ids.paragraph_ids = Arc::new(projection.paragraph_ids); + } + if projection.block_ids.len() == document.blocks.len() { + document.ids.block_ids = Arc::new(projection.block_ids); + } + if !projection.sections.is_empty() { + document.sections = Arc::new(projection.sections); + } + document +} + +struct Projector<'a> { + doc: &'a LoroDoc, + flows: LoroMap, + blocks: LoroMap, +} + +impl<'a> Projector<'a> { + fn new(doc: &'a LoroDoc) -> io::Result { + let root = doc.get_map(ROOT); + let flows = child_map(&root, FLOWS_BY_ID)?.ok_or_else(|| invalid("Flowstate Loro package has no flows map"))?; + let blocks = child_map(&root, BLOCKS_BY_ID)?.ok_or_else(|| invalid("Flowstate Loro package has no block registry"))?; + Ok(Self { doc, flows, blocks }) + } + + fn body_projection(&self, defects: &mut Vec) -> io::Result { + let body = self.flow_text(ROOT_BODY_FLOW_ID)?; + let (body_blocks, quarantined, paragraph_block_index) = self.object_blocks_for_flow(&body, ROOT_BODY_FLOW_ID, defects)?; + let mut blocks = Vec::new(); + let mut paragraph_ids = Vec::new(); + let mut block_ids = Vec::new(); + self.push_flow_blocks( + &body, + &body_blocks, + ROOT_BODY_FLOW_ID, + Some(&mut paragraph_ids), + Some(&mut block_ids), + Some(paragraph_block_index), + &mut blocks, + defects, + true, + )?; + // §5 quarantine: blocks whose anchors no longer resolve are appended at the + // end in stable (sorted block key) order instead of vanishing. Their defects + // were already recorded by `object_blocks_for_flow`. + for (key, block) in quarantined { + if let Ok(projected) = self.object_block(&block, defects) { + let id = map_string_opt(&block, "id")?.unwrap_or(key); + blocks.push(projected); + block_ids.push(BlockId(loro_id_u128(&id))); + } + } + if paragraph_ids.is_empty() { + // No paragraph rows at all — either a truly empty projection, or a body + // ending in object rows only (e.g. an object inserted into an empty + // document). The editor's document assembly appends a mandatory trailing + // paragraph in both shapes; emitting it HERE, with a deterministic + // fabricated identity, keeps `document_from_loro` a pure function — the + // silent length-mismatch fallback in `document_from_projection_blocks` + // previously let the assembly mint a RANDOM id for that row, so two + // rebuilds of the same doc disagreed (found by the object-fuzz undo arm). + blocks.push(InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: Vec::new(), + })); + // §5/FS-004: even the deterministic empty-projection placeholder is a + // fabricated identity; report it so the runtime seeds durable state. + defects.push(ProjectionDefect::MissingParagraphMetadata { + flow_id: ROOT_BODY_FLOW_ID.to_string(), + boundary_unicode: None, + fabricated_id: loro_id_u128("paragraph.projection.empty"), + }); + paragraph_ids.push(ParagraphId(loro_id_u128("paragraph.projection.empty"))); + block_ids.push(BlockId(loro_id_u128("block.projection.empty"))); + } + // §5 fidelity: surface every collected projection defect (Structure firehose) + // and assert the canonical body-projection invariants. Fully read-only and + // gated, so it costs a single atomic load when tracing is disabled. + if fidelity::enabled() { + for defect in defects.iter() { + fidelity::event(FidelityClass::Structure, "defect", || { + format!("{} @ {}", defect.class(), defect.stable_key()) + }); + } + check_body_projection_integrity(&body, &body_blocks, defects.as_slice()); + } + let sections = self.sections_for_projection(¶graph_ids)?; + Ok(ProjectionBlocks { + document_id: crate::loro_schema::document_id(self.doc).map_or(0, |id| id.as_u128()), + blocks, + paragraph_ids, + block_ids, + sections, + }) + } + + /// Project a `DocumentSection` for each Loro section, including its §11 page + /// structure (page size, margins, columns, orientation, page numbering, + /// header/footer flow ids). The canonical attrs live in Loro and are read + /// back via [`crate::loro_schema::read_section_page_attrs`] (which substitutes + /// documented defaults for missing keys), then mapped field-for-field onto the + /// gpui-flowtext projection mirror `gpui_flowtext::SectionPageAttrs`. + #[hotpath::measure] + fn sections_for_projection(&self, paragraph_ids: &[ParagraphId]) -> io::Result> { + let root = self.doc.get_map(ROOT); + let Some(sections_by_id) = child_map(&root, SECTIONS_BY_ID)? else { + return Ok(Vec::new()); + }; + // §perf: this index is only ever point-queried (section ordering comes from the + // explicit sort_by_key below), so an FxHashMap avoids the red-black-tree build. + let paragraph_order = paragraph_ids + .iter() + .enumerate() + .map(|(ix, id)| (id.0, ix)) + .collect::>(); + let mut sections = Vec::new(); + for key in map_keys(§ions_by_id) { + let Some(section) = child_map(§ions_by_id, &key)? else { + continue; + }; + let Some(start_paragraph) = section_id_field(§ion, "start_paragraph_id")? else { + continue; + }; + let section_id = map_string_opt(§ion, "id")? + .and_then(|value| parse_u128(&value)) + .unwrap_or_else(|| loro_id_u128(&key)); + let kind_slot = map_i64_opt(§ion, "kind_slot")? + .and_then(i64_to_u8) + .unwrap_or(0); + // §11: read the section's canonical page attrs from its `attrs` child map + // (defaults substituted for missing keys) and project them. The section + // map always exists here, so `page` is always `Some(..)` for determinism. + let canonical_page = match child_map(§ion, "attrs")? { + Some(attrs) => crate::loro_schema::read_section_page_attrs(&attrs), + None => crate::loro_schema::SectionPageAttrs::default(), + }; + sections.push(DocumentSection { + id: SectionId(section_id), + parent_id: section_id_field(§ion, "parent_section_id")?.map(SectionId), + kind: SectionKind::Custom(kind_slot), + heading_paragraph: section_id_field(§ion, "heading_paragraph_id")?.map(ParagraphId), + start_paragraph: ParagraphId(start_paragraph), + end_paragraph_exclusive: section_id_field(§ion, "end_paragraph_exclusive_id")?.map(ParagraphId), + page: Some(project_section_page_attrs(canonical_page)), + }); + } + sections.sort_by_key(|section| { + paragraph_order + .get(§ion.start_paragraph.0) + .copied() + .unwrap_or(usize::MAX) + }); + Ok(sections) + } + + #[allow( + clippy::too_many_arguments, + reason = "projection threading requires flow context, id pools, output and defect sink together" + )] + #[allow( + clippy::too_many_arguments, + reason = "projection threading requires flow context, id pools, output and defect sink together" + )] + fn push_flow_blocks( + &self, + text: &LoroText, + object_blocks: &BTreeMap, + flow_id: &str, + paragraph_ids: Option<&mut Vec>, + block_ids: Option<&mut Vec>, + // §perf-heaven T4: the paragraph-block boundary index, PRE-COMPUTED by the + // caller in the same BLOCKS_BY_ID pass that found the objects (body flow); a + // cell flow that collects no block ids passes `None`. + paragraph_block_index: Option>, + output: &mut Vec, + defects: &mut Vec, + flush_trailing_after_object: bool, + ) -> io::Result<()> { + // §perf: resolve this flow's paragraph boundary→id map ONCE (only for the id + // sinks that are actually collected — cell flows pass `None` and skip the + // build), then do O(1) lookups per boundary below. Replaces a per-boundary + // full rescan — an O(records²·chars) blow-up that pegged the CRDT actor + // thread. The paragraph-BLOCK index arrives pre-built (§perf-heaven T4). + let paragraph_index = paragraph_ids + .as_ref() + .map(|_| paragraph_ids_by_boundary(self.doc, text)); + + // §perf-heaven T3 tripwire: this materializes the WHOLE body as a delta + // Vec (the cold-path allocation). Per-keystroke edits must take the regional + // rematerializer instead and never reach here. + crate::instrument::record_body_to_delta_tagged(if flow_id == crate::BODY_FLOW_ID { "body-flow" } else { "cell-flow" }); + let delta = hotpath::measure_block!("projector_body_to_delta", crate::streaming_delta::streaming_to_delta(text)); + hotpath::measure_block!( + "projector_walk_flow_delta", + Self::walk_flow_delta( + text, + delta, + 0, + object_blocks, + flow_id, + paragraph_index.as_ref(), + paragraph_block_index.as_ref(), + paragraph_ids, + block_ids, + output, + defects, + flush_trailing_after_object, + |block, defects| self.object_block(block, defects), + ) + ) + } + + /// Core flow walk shared by the FULL materialization ([`Self::push_flow_blocks`]) + /// and the REGIONAL rematerialization ([`materialize_body_region`], spec §6-R): + /// ONE implementation of the paragraph/object/coalescing/defect law, applied to + /// either the whole flow delta from position 0 or a `slice_delta` region that + /// starts at a row's leading boundary sentinel. `unicode_pos` runs in ABSOLUTE + /// flow coordinates either way, so boundary-id maps and object positions are + /// always absolute. + #[allow( + clippy::too_many_arguments, + reason = "projection threading requires flow context, id pools, output and defect sink together" + )] + fn walk_flow_delta( + text: &LoroText, + delta: Vec, + start_unicode: usize, + object_blocks: &BTreeMap, + flow_id: &str, + paragraph_index: Option<&FxHashMap>, + paragraph_block_index: Option<&FxHashMap>, + mut paragraph_ids: Option<&mut Vec>, + mut block_ids: Option<&mut Vec>, + output: &mut Vec, + defects: &mut Vec, + flush_trailing_after_object: bool, + mut project_object: impl FnMut(&LoroMap, &mut Vec) -> io::Result, + ) -> io::Result<()> { + let mut current = InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: Vec::new(), + }; + // Durable key of the most recently emitted object row: the identity + // anchor for any boundary-less (interstitial/trailing) paragraph row that + // follows it (see `push_paragraph_projection_metadata`). + let mut last_object_key: Option = None; + let mut pending_style = gpui_flowtext::ParagraphStyle::Normal; + let mut seen_sentinel = false; + let mut unicode_pos = start_unicode; + let mut current_boundary = None; + + for item in delta { + let loro::TextDelta::Insert { insert, attributes } = item else { + continue; + }; + let run_styles = run_styles_from_attrs(attributes.as_ref()); + for ch in insert.chars() { + match ch { + '\n' => { + // §5 (adjustmentplan:224): a paragraph boundary without a + // paragraph-style mark projects as Normal and is reported so the + // runtime repairs the canonical mark, instead of silently + // inheriting the previous paragraph's style. + let style = match paragraph_style_from_attrs(attributes.as_ref()) { + Some(style) => style, + None => { + defects.push(ProjectionDefect::MissingParagraphStyleMark { + flow_id: flow_id.to_string(), + boundary_unicode: unicode_pos, + }); + gpui_flowtext::ParagraphStyle::Normal + }, + }; + if !seen_sentinel { + seen_sentinel = true; + pending_style = style; + current.style = style; + current_boundary = Some(unicode_pos); + } else if current.runs.is_empty() + && current_boundary.is_none() + && output + .last() + .is_some_and(|block| !matches!(block, InputBlock::Paragraph(_))) + { + // Fork B (docs/collab-coalescing-parity.md): coalesce ONLY the phantom + // empty paragraph that an object's own block implies — the first `\n` + // after an object, when `current_boundary` is still None because the + // object reset it. A SUBSEQUENT empty `\n` (current_boundary already Some) + // is a REAL, user/edit-created empty paragraph with a durable record; + // KEEP it so an empty line next to an image survives instead of being + // silently dropped. (The incremental replay's matching half is still in + // progress — see the doc; the structural fuzz stays #[ignore]d until then.) + current.style = style; + pending_style = style; + current_boundary = Some(unicode_pos); + } else { + push_paragraph_projection_metadata( + text, + paragraph_index, + paragraph_block_index, + flow_id, + current_boundary, + output.len(), + last_object_key.as_deref(), + paragraph_ids.as_deref_mut(), + block_ids.as_deref_mut(), + defects, + ); + output.push(InputBlock::Paragraph(current)); + current = InputParagraph { style, runs: Vec::new() }; + pending_style = style; + current_boundary = Some(unicode_pos); + } + }, + OBJECT_REPLACEMENT => { + if let Some(block) = object_blocks.get(&unicode_pos) { + // Fork B symmetry (docs/collab-coalescing-parity.md): flush a REAL empty + // paragraph sitting just BEFORE an object — one with a live boundary + // (`current_boundary.is_some()`) that follows already-emitted content + // (`!output.is_empty()`), i.e. a durably-recorded empty the user created + // or an edit produced (e.g. splitting at the end of the paragraph that + // precedes an image). Excluded: the record-less phantom (current_boundary + // == None, right after another object) AND the leading sentinel empty + // before a doc-first object (output still empty — the object stays block 0). + // Previously an empty `current` was always dropped here, so the full rebuild + // lost an empty line before an image that the incremental replay kept. + if !current.runs.is_empty() || (current_boundary.is_some() && !output.is_empty()) { + push_paragraph_projection_metadata( + text, + paragraph_index, + paragraph_block_index, + flow_id, + current_boundary, + output.len(), + last_object_key.as_deref(), + paragraph_ids.as_deref_mut(), + block_ids.as_deref_mut(), + defects, + ); + output.push(InputBlock::Paragraph(current)); + current = InputParagraph { + style: pending_style, + runs: Vec::new(), + }; + } + output.push(project_object(block, defects)?); + let object_key = map_string(block, "id")?; + if let Some(block_ids) = block_ids.as_deref_mut() { + block_ids.push(BlockId(loro_id_u128(&object_key))); + } + last_object_key = Some(object_key); + current_boundary = None; + } else if flow_id == ROOT_BODY_FLOW_ID { + // §5/FS-036 backstop: a placeholder no block claims would + // silently vanish; report it so the runtime removes the orphan + // character canonically. + defects.push(ProjectionDefect::OrphanObjectPlaceholder { + flow_id: flow_id.to_string(), + unicode_pos, + }); + } + }, + _ => push_char(&mut current, ch, run_styles), + } + unicode_pos += 1; + } + } + + // A WHOLE flow that ends in object rows still carries `current` — the + // style-bearing (pending-style) trailing paragraph the editor's document + // assembly would otherwise mint with a RANDOM id and a default style. + // Emitting it here keeps the materializer a pure function AND keeps the + // trailing row's style equal to what the last boundary's mark says (the + // object-fuzz undo arm caught both: nondeterministic ids, then a stale + // Normal style after SetParagraphStyle on the interstitial row). Regions + // and cell flows keep the old behavior — a region legitimately ends + // mid-geometry and must not gain rows. + let ends_with_object = flush_trailing_after_object && matches!(output.last(), Some(block) if !matches!(block, InputBlock::Paragraph(_))); + if ends_with_object && current.runs.is_empty() && current_boundary.is_none() { + // The synthesized empty trailing row is a FRESH paragraph, not an + // inheritor: pending style belongs to the last real boundary's + // paragraph, and carrying it here would make restyling that paragraph + // silently restyle this row too (maintained-vs-canonical style drift). + current.style = gpui_flowtext::ParagraphStyle::Normal; + } + if !current.runs.is_empty() || current_boundary.is_some() || output.is_empty() && seen_sentinel || ends_with_object { + push_paragraph_projection_metadata( + text, + paragraph_index, + paragraph_block_index, + flow_id, + current_boundary, + output.len(), + last_object_key.as_deref(), + paragraph_ids, + block_ids, + defects, + ); + output.push(InputBlock::Paragraph(current)); + } + Ok(()) + } + + /// Resolve every object block anchored into `flow_id`. The first return value + /// maps live placeholder positions to their blocks; the second collects the + /// quarantined blocks (unresolved or displaced-by-collision anchors) in stable + /// sorted-key order, with one defect recorded per quarantined block. + #[allow( + clippy::type_complexity, + reason = "returns resolved-by-position blocks, quarantined blocks, and the paragraph-block boundary index from ONE registry pass" + )] + #[hotpath::measure] + fn object_blocks_for_flow( + &self, + text: &LoroText, + flow_id: &str, + defects: &mut Vec, + ) -> io::Result<(BTreeMap, Vec<(String, LoroMap)>, FxHashMap)> { + // §act-five P4: ONE registry pass. The old shape scanned the whole block + // registry TWICE — once inside `boundary_cursor_positions` (which iterates + // EVERY block record to harvest anchor cursors) and again in the loop below + // — and materialized the ENTIRE body as `Vec` even for a flow with no + // objects. Now we gather this flow's object candidates in a single sorted + // pass (quarantine order preserved), batch-resolve ONLY their anchors, and + // build the body snapshot only when there is something to place. A DEAD + // anchor is simply absent from the batch (→ quarantine + canonical re-anchor + // repair, identity preserved), so behaviour is bit-identical to the former + // per-record path; the id-less-cursor fallback (`get_cursor_pos`) is kept. + let container = text.id(); + // §perf-heaven T4: ONE pass over BLOCKS_BY_ID builds BOTH this flow's object + // candidates AND the paragraph-block boundary index (formerly a SECOND full + // scan in `paragraph_block_ids_by_boundary`). Every block is decoded once, + // partitioned by kind; both anchor sets resolve through ONE batched + // `query_text_id_positions`. Keys are consumed by value (no per-key clone). + let mut candidates: Vec<(String, LoroMap, Option, Option>)> = Vec::new(); + let mut paragraph_candidates: Vec<(String, Option)> = Vec::new(); + let mut anchor_ids: Vec = Vec::new(); + for key in map_keys(&self.blocks) { + let Some(block) = child_map(&self.blocks, &key)? else { + continue; + }; + if map_string_opt(&block, "flow_id")?.as_deref() != Some(flow_id) { + continue; + } + let cursor_bytes = map_binary_opt(&block, "anchor_cursor")?; + let cursor = cursor_bytes + .as_deref() + .and_then(|bytes| Cursor::decode(bytes).ok()) + .filter(|cursor| cursor.container == container); + if let Some(id) = cursor.as_ref().and_then(|cursor| cursor.id) { + anchor_ids.push(id); + } + if map_string_opt(&block, "kind")?.as_deref() == Some("paragraph") { + paragraph_candidates.push((key, cursor)); + } else { + candidates.push((key, block, cursor, cursor_bytes)); + } + } + // ONE batched resolution feeds both the object placement and the + // paragraph-block boundary index below. + let mut anchor_positions: FxHashMap = FxHashMap::default(); + if !anchor_ids.is_empty() { + for (id, pos) in anchor_ids.iter().copied().zip( + self + .doc + .inner() + .query_text_id_positions(&container, &anchor_ids), + ) { + if let Some(pos) = pos { + anchor_positions.insert(id, pos); + } + } + } + + let mut by_pos = BTreeMap::new(); + let mut keys_by_pos: BTreeMap = BTreeMap::new(); + let mut quarantined = Vec::new(); + // Object placement: skip the body snapshot/resolver entirely when the flow + // has no objects. Validate that each resolved anchor still sits on an + // object-replacement char via a Src-safe `char_at` per object. + // + // §perf-heaven T7.1: the former ≤8192-unicode branch materialized the body as + // a `Vec` via `text.to_string()`. That call drives the container store's + // Lazy-VALUE decode (path 1) — a full bytes-decode that leaves the container + // `Lazy`-with-cached-value — and object validation runs BEFORE the body delta + // (`push_flow_blocks` → `text.to_delta()`), which then forces the Src STATE + // decode (path 2, `decode_snapshot_fast`) from the same bytes. So the small + // body paid a DOUBLE bytes-decode plus a `Vec` allocation. (This is the + // real cause of the reverted materialize-once "4× slower" — `to_string` never + // forced `Dst`; it drove a redundant second decode.) `char_at` instead promotes + // the container to `LazyLoad::Src` ONCE (path 2) and the later `to_delta` + // reuses that state — a single decode, no `Vec`. Bounded per-object + // O(elements) walk; object counts on a real body are tiny. Out-of-range + // positions resolve to `None` → quarantine. Guarded by the corpus Src-path + // equivalence net (T7.26) + the convergence fuzz. + if !candidates.is_empty() { + let is_object_char = |pos: usize| text.char_at(pos).ok() == Some(OBJECT_REPLACEMENT); + for (key, block, cursor, cursor_bytes) in candidates { + let resolved = cursor + .and_then(|cursor| match cursor.id { + Some(id) => anchor_positions.get(&id).copied(), + None => { + crate::instrument::record_cursor_pos_resolve(); + self + .doc + .get_cursor_pos(&cursor) + .ok() + .map(|pos| pos.current.pos) + }, + }) + .filter(|pos| is_object_char(*pos)); + let Some(pos) = resolved else { + // FS-002: never silently drop a block whose anchor is unresolvable. + defects.push(ProjectionDefect::UnresolvedObjectAnchor { + block_key: key.clone(), + flow_id: flow_id.to_string(), + anchor_cursor: cursor_bytes, + }); + quarantined.push((key, block)); + continue; + }; + if let Some(kept_key) = keys_by_pos.get(&pos) { + // FS-003: colliding cursors previously overwrote each other in the map. + defects.push(ProjectionDefect::CollidingObjectAnchors { + flow_id: flow_id.to_string(), + anchor_unicode: pos, + kept_block_key: kept_key.clone(), + displaced_block_key: key.clone(), + }); + quarantined.push((key, block)); + continue; + } + keys_by_pos.insert(pos, key); + by_pos.insert(pos, block); + } + } + + // §perf-heaven T4: the paragraph-block boundary index, built from the SAME + // pass + query (was a whole second BLOCKS_BY_ID scan). Selection rule matches + // `paragraph_block_ids_by_boundary` exactly: first sorted key wins per + // boundary; boundary 0 prefers `MAIN_BODY_BLOCK_ID`. + let mut paragraph_block_index: FxHashMap = FxHashMap::default(); + let mut main_body_at_zero = false; + let text_len = text.len_unicode(); + for (key, cursor) in paragraph_candidates { + let Some(pos) = resolve_decoded_cursor(self.doc, text_len, cursor.as_ref(), &anchor_positions) else { + continue; + }; + if pos == 0 && key.as_str() == MAIN_BODY_BLOCK_ID { + main_body_at_zero = true; + } + paragraph_block_index.entry(pos).or_insert(key); + } + if main_body_at_zero { + paragraph_block_index.insert(0, MAIN_BODY_BLOCK_ID.to_string()); + } + // The specific net: prove the fused index is bit-identical to the standalone + // scan. Runs in debug (tests / fuzz / debug corpus); release trusts the + // corpus sweep (projection == reimport). + debug_assert_eq!( + paragraph_block_index, + paragraph_block_ids_by_boundary(self.doc, text), + "T4 fused paragraph-block index diverged from paragraph_block_ids_by_boundary", + ); + + Ok((by_pos, quarantined, paragraph_block_index)) + } + + fn object_block(&self, block: &LoroMap, defects: &mut Vec) -> io::Result { + match map_string(block, "kind")?.as_str() { + "image" => self.image_block(block, defects).map(InputBlock::Image), + "equation" => self.equation_block(block).map(InputBlock::Equation), + "table" => self.table_block(block, defects).map(InputBlock::Table), + kind => Err(invalid(format!("unsupported Loro block kind `{kind}`"))), + } + } + + fn image_block(&self, block: &LoroMap, defects: &mut Vec) -> io::Result { + let attrs = child_map(block, "attrs")?; + // FS-011: an invalid asset id projects as a deterministic placeholder id + // (never a silent coercion) and is reported for canonical recovery. + let raw_asset_id = map_string_opt(block, "asset_id")?; + let asset_id = raw_asset_id.as_deref().and_then(parse_u128); + if asset_id.is_none() { + defects.push(ProjectionDefect::InvalidAssetId { + block_key: map_string_opt(block, "id")?.unwrap_or_default(), + raw_asset_id, + }); + } + Ok(InputImageBlock { + asset_id: AssetId(asset_id.unwrap_or(0)), + alt_text: map_string_opt(block, "alt_text_flow_id")? + .map(|flow_id| self.plain_flow_text(&flow_id)) + .transpose()? + .unwrap_or_default(), + caption: map_string_opt(block, "caption_flow_id")? + .map(|flow_id| self.caption_paragraph(&flow_id)) + .transpose()?, + sizing: image_sizing(attrs.as_ref())?, + alignment: alignment(attrs.as_ref())?, + // §A11.9: the linked-image URL; absent/empty means an embedded image. + external_url: map_string_opt(block, "external_url")?.filter(|url| !url.is_empty()), + }) + } + + fn equation_block(&self, block: &LoroMap) -> io::Result { + let attrs = child_map(block, "attrs")?; + Ok(InputEquationBlock { + source: map_string_opt(block, "source_flow_id")? + .map(|flow_id| self.plain_flow_text(&flow_id)) + .transpose()? + .unwrap_or_default(), + syntax: equation_syntax(attrs.as_ref())?, + display: equation_display(attrs.as_ref())?, + }) + } + + fn table_block(&self, owner: &LoroMap, defects: &mut Vec) -> io::Result { + let table = child_map(owner, TABLE_KEY)?.ok_or_else(|| invalid("table block has no table map"))?; + let block_key = map_string_opt(owner, "id")?.unwrap_or_default(); + self.table_from_map(&table, &block_key, defects) + } + + fn table_from_map(&self, table: &LoroMap, block_key: &str, defects: &mut Vec) -> io::Result { + // §28: resolve the table's child containers through their stored raw + // container ids, falling back to key traversal when unavailable. + let columns_map = self + .resolve_child_map(table, "columns_container_id", TABLE_COLUMNS_BY_ID)? + .ok_or_else(|| invalid("table has no column map"))?; + let cells_by_id = self + .resolve_child_map(table, "cells_container_id", TABLE_CELLS_BY_ID)? + .ok_or_else(|| invalid("table has no cell map"))?; + + // §P2b: read the durable ordered ids, then build the column list carrying + // each column's durable id + width. Malformed ids (never produced by our + // writers) are skipped so a single bad id can't sink the whole table. + let mut columns = Vec::new(); + let mut column_ids = Vec::new(); + for column_id_str in ordered_ids(table, TABLE_COLUMN_ORDER)? { + let Some(column_id) = parse_column_loro_id(&column_id_str) else { + continue; + }; + // A concurrent DeleteTableColumn removes the column's map from `columns_by_id`, but + // the ordered `column_order` list is a separate CRDT and can still reference it after + // an out-of-order merge (e.g. concurrent delete + move). Skip the stale order entry + // rather than failing the whole projection — deterministic across peers (all read the + // same order + map state), matching the malformed-id skip above and §P2b's "a single + // bad id can't sink the whole table". Cells left referencing it are dropped by the + // topology normalization below. + let Some(column) = child_map(&columns_map, &column_id_str)? else { + continue; + }; + columns.push(InputTableColumn { + id: column_id, + width: table_column_width(&column)?, + }); + column_ids.push(column_id); + } + let mut row_ids = Vec::new(); + for row_id_str in ordered_ids(table, TABLE_ROW_ORDER)? { + if let Some(row_id) = parse_row_loro_id(&row_id_str) { + row_ids.push(row_id); + } + } + + // Read every stored cell into a raw record (for topology normalization) and + // keep its container keyed by coordinate for block projection. + let mut raw = Vec::new(); + let mut cell_maps: FxHashMap<(u128, u128), LoroMap> = FxHashMap::default(); + for cell_key in cells_by_id.keys().map(|key| key.to_string()) { + let Some(cell) = child_map(&cells_by_id, &cell_key)? else { + continue; + }; + let (Some(row_id), Some(column_id)) = ( + map_string_opt(&cell, "row_id")? + .as_deref() + .and_then(parse_row_loro_id), + map_string_opt(&cell, "column_id")? + .as_deref() + .and_then(parse_column_loro_id), + ) else { + continue; + }; + let cell_id = parse_cell_loro_id(&cell_key).unwrap_or_else(|| CellId::from_coordinate(row_id, column_id)); + raw.push(table_topology::RawCellRecord { + row_id, + column_id, + cell_id, + row_span: map_i64_opt(&cell, "row_span")? + .and_then(i64_to_u16) + .unwrap_or(1), + col_span: map_i64_opt(&cell, "column_span")? + .and_then(i64_to_u16) + .unwrap_or(1), + }); + cell_maps.insert((row_id.0, column_id.0), cell); + } + + // §P2b/FS-010: normalize to a full, well-formed grid + a defect list, so + // every peer reads the identical grid and the runtime repairs the canonical + // state. `normalize` returns exactly `row_ids.len() * column_ids.len()` + // cells, row-major. + let normalized = table_topology::normalize(&row_ids, &column_ids, &raw); + for defect in &normalized.defects { + defects.push(map_topology_defect(block_key, defect)); + } + + let column_count = column_ids.len(); + let mut rows = Vec::with_capacity(row_ids.len()); + for (row_index, &row_id) in row_ids.iter().enumerate() { + let mut cells = Vec::with_capacity(column_count); + for (col_index, &column_id) in column_ids.iter().enumerate() { + let normalized_cell = &normalized.cells[row_index * column_count + col_index]; + let blocks = match cell_maps.get(&(row_id.0, column_id.0)) { + Some(cell_map) if !normalized_cell.synthesized => self.table_cell_blocks(cell_map, defects)?, + _ => vec![InputTableCellBlock::Paragraph(empty_input_paragraph())], + }; + cells.push(InputTableCell { + id: normalized_cell.cell_id, + row_id, + column_id, + blocks, + row_span: normalized_cell.row_span, + col_span: normalized_cell.col_span, + }); + } + rows.push(InputTableRow { id: row_id, cells }); + } + + Ok(InputTableBlock { + rows, + columns, + style: InputTableStyle { + header_row: map_bool_opt(table, "header_row")?.unwrap_or(false), + }, + }) + } + + fn table_cell_blocks(&self, cell: &LoroMap, defects: &mut Vec) -> io::Result> { + let flow_id = map_string(cell, "flow_id")?; + let flow = self.flow_text(&flow_id)?; + let object_blocks = self.cell_nested_tables(cell, &flow)?; + let mut projected = Vec::new(); + self.push_flow_blocks(&flow, &object_blocks, &flow_id, None, None, None, &mut projected, defects, false)?; + let mut blocks = projected + .into_iter() + .filter_map(|block| match block { + InputBlock::Paragraph(paragraph) => Some(Ok(InputTableCellBlock::Paragraph(paragraph))), + InputBlock::Table(table) => Some(Ok(InputTableCellBlock::Table(table))), + InputBlock::Image(_) | InputBlock::Equation(_) => None, + }) + .collect::>>()?; + if blocks.is_empty() { + blocks.push(InputTableCellBlock::Paragraph(empty_input_paragraph())); + } + Ok(blocks) + } + + fn cell_nested_tables(&self, cell: &LoroMap, flow: &LoroText) -> io::Result> { + let mut tables = BTreeMap::new(); + let Some(tables_by_id) = child_map(cell, "nested_tables_by_id")? else { + return Ok(tables); + }; + for nested_table_id in ordered_ids(cell, "nested_table_ids")? { + let Some(owner) = child_map(&tables_by_id, &nested_table_id)? else { + continue; + }; + let Some(cursor_bytes) = map_binary_opt(&owner, "anchor_cursor")? else { + continue; + }; + let Ok(cursor) = Cursor::decode(&cursor_bytes) else { + continue; + }; + if cursor.container != flow.id() { + continue; + } + crate::instrument::record_cursor_pos_resolve(); + if let Ok(pos) = self.doc.get_cursor_pos(&cursor) { + tables.insert(pos.current.pos, owner); + } + } + Ok(tables) + } + + fn flow_text(&self, flow_id: &str) -> io::Result { + let flow = child_map(&self.flows, flow_id)?.ok_or_else(|| invalid(format!("missing flow `{flow_id}`")))?; + // §28: prefer direct resolution via the flow's stored raw container id, and + // only fall back to map-key traversal when the id is missing/unresolvable. + if let Some(container_id) = map_string_opt(&flow, "text_container_id")? + && let Some(text) = resolve_text_by_container_id(self.doc, &container_id) + { + return Ok(text); + } + child_text(&flow, FLOW_TEXT_KEY)?.ok_or_else(|| invalid(format!("flow `{flow_id}` has no text"))) + } + + /// §28: resolve a child container map by its stored raw container id, falling + /// back to direct map-key traversal when the id is missing/unresolvable. + fn resolve_child_map(&self, owner: &LoroMap, container_id_key: &str, fallback_key: &str) -> io::Result> { + if let Some(container_id) = map_string_opt(owner, container_id_key)? + && let Some(map) = resolve_map_by_container_id(self.doc, &container_id) + { + return Ok(Some(map)); + } + child_map(owner, fallback_key) + } + + fn plain_flow_text(&self, flow_id: &str) -> io::Result { + // §perf: drop a single leading '\n' in place rather than allocating a second + // String via strip_prefix(..).to_string(). + let mut text = self.flow_text(flow_id)?.to_string(); + if text.starts_with('\n') { + text.remove(0); + } + Ok(text) + } + + fn caption_paragraph(&self, flow_id: &str) -> io::Result { + let paragraphs = paragraphs_from_text(&self.flow_text(flow_id)?); + Ok(paragraphs.into_iter().next().unwrap_or(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: Vec::new(), + })) + } +} + +/// §5 fidelity integrity invariant for the body projection. The caller gates this +/// on [`fidelity::enabled`]; it never mutates the document and only reads the +/// already-resolved body text and object-block map. It asserts the canonical +/// invariants that the projector's defect reporting is designed to guarantee: +/// * `missing-sentinel` (Structure): the body flow starts with the sentinel newline. +/// * `boundary-without-mark` (Structure): every boundary newline carries a +/// paragraph-style mark. +/// * `orphan-object` (Structure): every U+FFFC placeholder resolves to a live +/// block record (a key of `object_blocks`). +/// * `orphan-metadata` (Identity): no non-paragraph block record dangles without a +/// live U+FFFC placeholder (surfaced as an unresolved-anchor defect). +/// +/// (d) A fabricated-id paragraph is only ever emitted alongside a recorded +/// `MissingParagraph*` defect (see [`push_paragraph_projection_metadata`] and the +/// empty-projection placeholder), each of which is surfaced by the per-defect +/// `Structure`/`defect` events, so fabricated identities need no separate scan. +fn check_body_projection_integrity(body: &LoroText, object_blocks: &BTreeMap, defects: &[ProjectionDefect]) { + let snapshot = body.to_string(); + fidelity::check( + snapshot.starts_with(crate::SENTINEL_NEWLINE), + FidelityClass::Structure, + "missing-sentinel", + || { + format!( + "body flow does not start with the sentinel newline (first char {:?})", + snapshot.chars().next() + ) + }, + ); + // (b) Every boundary newline must carry a paragraph-style mark. Walk the rich + // delta so we see each insert's attributes, mirroring the projector's own scan. + let mut unicode_pos = 0_usize; + for item in crate::streaming_delta::streaming_to_delta(body) { + let loro::TextDelta::Insert { insert, attributes } = item else { + continue; + }; + for ch in insert.chars() { + if ch == '\n' { + fidelity::check( + paragraph_style_from_attrs(attributes.as_ref()).is_some(), + FidelityClass::Structure, + "boundary-without-mark", + || format!("paragraph boundary newline at body unicode pos {unicode_pos} carries no paragraph-style mark"), + ); + } + unicode_pos += 1; + } + } + // (c) Every live U+FFFC placeholder must be claimed by a resolved block record. + for (pos, ch) in snapshot.chars().enumerate() { + if ch == OBJECT_REPLACEMENT { + fidelity::check(object_blocks.contains_key(&pos), FidelityClass::Structure, "orphan-object", || { + format!("U+FFFC object placeholder at body unicode pos {pos} has no live block record") + }); + } + } + // (c, vice-versa) A block record whose anchor no longer resolves to a live + // placeholder is dangling metadata; the projector reports it as an + // unresolved-anchor defect, which this invariant escalates loudly. + for defect in defects { + if let ProjectionDefect::UnresolvedObjectAnchor { block_key, flow_id, .. } = defect { + fidelity::check(false, FidelityClass::Identity, "orphan-metadata", || { + format!("block `{block_key}` in flow `{flow_id}` has no live U+FFFC placeholder") + }); + } + } +} + +fn paragraphs_from_text(text: &LoroText) -> Vec { + let mut blocks = Vec::new(); + let projector = ParagraphOnlyProjector; + projector.push_flow_blocks(text, &mut blocks); + blocks +} + +struct ParagraphOnlyProjector; + +impl ParagraphOnlyProjector { + fn push_flow_blocks(&self, text: &LoroText, output: &mut Vec) { + let mut current = InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: Vec::new(), + }; + let mut pending_style = gpui_flowtext::ParagraphStyle::Normal; + let mut seen_sentinel = false; + for item in crate::streaming_delta::streaming_to_delta(text) { + let loro::TextDelta::Insert { insert, attributes } = item else { + continue; + }; + let run_styles = run_styles_from_attrs(attributes.as_ref()); + for ch in insert.chars() { + if ch == '\n' { + let style = paragraph_style_from_attrs(attributes.as_ref()).unwrap_or(pending_style); + if !seen_sentinel { + seen_sentinel = true; + pending_style = style; + current.style = style; + } else { + output.push(current); + current = InputParagraph { style, runs: Vec::new() }; + pending_style = style; + } + } else if ch != OBJECT_REPLACEMENT { + push_char(&mut current, ch, run_styles); + } + } + } + if seen_sentinel || !current.runs.is_empty() { + output.push(current); + } + } +} + +fn push_char(paragraph: &mut InputParagraph, ch: char, styles: RunStyles) { + if let Some(last) = paragraph.runs.last_mut() + && last.styles == styles + { + last.text.push(ch); + return; + } + paragraph.runs.push(InputRun { + text: ch.to_string(), + styles, + }); +} + +#[allow( + clippy::too_many_arguments, + reason = "paragraph metadata projection needs the flow text, prebuilt boundary indexes, flow/boundary context, id pools and defect sink" +)] +fn push_paragraph_projection_metadata( + text: &LoroText, + paragraph_index: Option<&FxHashMap>, + paragraph_block_index: Option<&FxHashMap>, + flow_id: &str, + boundary: Option, + block_ix: usize, + interstitial_anchor: Option<&str>, + paragraph_ids: Option<&mut Vec>, + block_ids: Option<&mut Vec>, + defects: &mut Vec, +) { + let paragraph_resolved = boundary.and_then(|boundary| { + paragraph_index + .and_then(|index| index.get(&boundary)) + .cloned() + }); + let block_resolved = boundary.and_then(|boundary| { + paragraph_block_index + .and_then(|index| index.get(&boundary)) + .cloned() + }); + // §5: fabricate stable, position-independent ids (from the boundary's OpID — + // the SAME keys the repair writer mints) for any boundary without a durable + // record. Resolve the anchor once, and only when we actually must fabricate, + // since `get_cursor` is not free. + let needs_fabrication = (paragraph_ids.is_some() && paragraph_resolved.is_none()) || (block_ids.is_some() && block_resolved.is_none()); + let fabricated_keys = needs_fabrication + .then(|| boundary.and_then(|boundary| stable_boundary_metadata_keys(text, boundary))) + .flatten(); + + if let Some(paragraph_ids) = paragraph_ids { + // §5/FS-004: a boundary with no durable paragraph metadata gets a fabricated, + // projection-only id (reported so the runtime writes the real record). Deriving + // it from the boundary OpID makes it identical to the repaired record's id and + // stable across peers, so incremental/full and both peers converge. + let id = match paragraph_resolved { + Some(id) => id, + None => { + // Boundary-less (interstitial/trailing) rows anchor their fabricated + // identity to the durable key of the OBJECT they follow — a row-index + // key ("paragraph.projection.{ix}") silently re-identified the row on + // every edit above it (the object-fuzz undo arm caught the drift as a + // maintained-vs-canonical id mismatch after a join). + let fabricated_id = fabricated_keys + .as_ref() + .map(|(paragraph_key, _)| paragraph_key.clone()) + .unwrap_or_else(|| { + interstitial_anchor.map_or_else( + || format!("paragraph.projection.{block_ix}"), + |anchor| format!("paragraph.after.{anchor}"), + ) + }); + { + static FAB_DEBUG: std::sync::OnceLock = std::sync::OnceLock::new(); + if *FAB_DEBUG.get_or_init(|| std::env::var_os("FLOWSTATE_DERIVE_DEBUG").is_some()) { + eprintln!( + "project[fabricate]: boundary {boundary:?} block_ix {block_ix} → {fabricated_id} (u128 {})", + loro_id_u128(&fabricated_id) + ); + } + } + defects.push(ProjectionDefect::MissingParagraphMetadata { + flow_id: flow_id.to_string(), + boundary_unicode: boundary, + fabricated_id: loro_id_u128(&fabricated_id), + }); + fabricated_id + }, + }; + paragraph_ids.push(ParagraphId(loro_id_u128(&id))); + } + if let Some(block_ids) = block_ids { + // §5/FS-005: mirror the paragraph-metadata report for the paragraph block registry. + let id = match block_resolved { + Some(id) => id, + None => { + let fabricated_id = fabricated_keys + .as_ref() + .map(|(_, block_key)| block_key.clone()) + .unwrap_or_else(|| { + interstitial_anchor.map_or_else( + || format!("paragraph_block.projection.{block_ix}"), + |anchor| format!("paragraph_block.after.{anchor}"), + ) + }); + defects.push(ProjectionDefect::MissingParagraphBlock { + flow_id: flow_id.to_string(), + boundary_unicode: boundary, + fabricated_id: loro_id_u128(&fabricated_id), + }); + fabricated_id + }, + }; + block_ids.push(BlockId(loro_id_u128(&id))); + } +} + +/// Build a `boundary position → paragraph metadata loro id` index for `text` in a +/// SINGLE pass over the paragraph registry. Projecting a flow calls this once and +/// then does O(1) lookups per boundary — replacing the former +/// `paragraph_loro_id_at_boundary` which rescanned every paragraph record for each +/// boundary, an O(paragraphs²·chars) hot path that hung the CRDT actor thread when +/// materializing a large document. +/// +/// Selection matches the previous scan exactly: keys are visited in sorted order +/// (via [`map_keys`]) so the lexicographically-smallest id wins a shared boundary, +/// except boundary 0 always prefers `ROOT_FIRST_PARAGRAPH_ID` when it anchors there. +#[hotpath::measure] +fn paragraph_ids_by_boundary(doc: &LoroDoc, text: &LoroText) -> FxHashMap { + let mut index: FxHashMap = FxHashMap::default(); + let root = doc.get_map(ROOT); + let Some(paragraphs) = child_map(&root, PARAGRAPHS_BY_ID).ok().flatten() else { + return index; + }; + // §act-five P4: ONE pass over the paragraph registry. The old shape scanned it + // TWICE (once inside `boundary_cursor_positions`, once in the loop) and decoded + // each cursor twice (there, then again in `live_cursor_pos`). Gather the decoded + // boundary/start cursors once, batch-resolve their ids, then build the index + // from the pre-decoded cursors. Selection rule (boundary_cursor then start_cursor + // fallback), sorted-key first-wins ordering, root-first special-case, and the + // dead-anchor / id-less-cursor fallbacks are all preserved bit-identically. + let container = text.id(); + // §perf-heaven T4: consume the sorted keys by value (no per-key clone). + let sorted_keys = map_keys(¶graphs); + let mut cands: Vec<(String, Option, Option)> = Vec::with_capacity(sorted_keys.len()); + let mut ids: Vec = Vec::new(); + for key in sorted_keys { + let Some(record) = child_map(¶graphs, &key).ok().flatten() else { + continue; + }; + let decode = |field: &str| { + map_binary_opt(&record, field) + .ok() + .flatten() + .and_then(|bytes| Cursor::decode(&bytes).ok()) + .filter(|cursor| cursor.container == container) + }; + let boundary = decode("boundary_cursor"); + let start = decode("start_cursor"); + for cursor in [boundary.as_ref(), start.as_ref()].into_iter().flatten() { + if let Some(id) = cursor.id { + ids.push(id); + } + } + cands.push((key, boundary, start)); + } + let mut pos_by_id: FxHashMap = FxHashMap::default(); + if !ids.is_empty() { + for (id, pos) in ids + .iter() + .copied() + .zip(doc.inner().query_text_id_positions(&container, &ids)) + { + if let Some(pos) = pos { + pos_by_id.insert(id, pos); + } + } + } + let mut root_first_at_zero = false; + let text_len = text.len_unicode(); + for (key, boundary, start) in cands { + let Some(pos) = resolve_decoded_cursor(doc, text_len, boundary.as_ref(), &pos_by_id) + .or_else(|| resolve_decoded_cursor(doc, text_len, start.as_ref(), &pos_by_id)) + else { + continue; + }; + if pos == 0 && key.as_str() == ROOT_FIRST_PARAGRAPH_ID { + root_first_at_zero = true; + } + index.entry(pos).or_insert(key); + } + if root_first_at_zero { + index.insert(0, ROOT_FIRST_PARAGRAPH_ID.to_string()); + } + index +} + +/// Resolve a PRE-DECODED boundary cursor to a live unicode position — the +/// pre-decoded companion of [`live_cursor_pos`] (same batch-hit / id-less +/// `get_cursor_pos` fallback / in-range validation), for the fused single-pass +/// boundary indexers that decode each record's cursors exactly once. +fn resolve_decoded_cursor(doc: &LoroDoc, text_len: usize, cursor: Option<&Cursor>, pos_by_id: &FxHashMap) -> Option { + // §perf-heaven T7.8: takes the body's unicode length precomputed once by the + // caller, rather than calling `text.len_unicode()` per boundary. Even O(1), + // hoisting it out of the per-record loop drops N redundant calls. + let cursor = cursor?; + let pos = match cursor.id { + Some(id) => pos_by_id.get(&id).copied()?, + None => { + crate::instrument::record_cursor_pos_resolve(); + doc.get_cursor_pos(cursor).ok()?.current.pos + }, + }; + (pos < text_len).then_some(pos) +} + +/// Build a `boundary position → paragraph *block* loro id` index for `text` in a +/// single pass over the block registry (paragraph-kind blocks only). Companion to +/// [`paragraph_ids_by_boundary`] with the same one-pass rationale and selection +/// rule, except boundary 0 prefers `MAIN_BODY_BLOCK_ID`. +/// +/// §perf-heaven T4: the projection now builds this index inside +/// `object_blocks_for_flow` (same `BLOCKS_BY_ID` pass as the objects). This +/// standalone function is retained as the REFERENCE the fused index is +/// `debug_assert_eq!`'d against (and used by tests); hence `dead_code` in a +/// release non-test build where the debug assert compiles out. +#[allow( + dead_code, + reason = "reference implementation for the T4 fused-index debug_assert_eq + tests; unused only in release non-test builds" +)] +#[hotpath::measure] +fn paragraph_block_ids_by_boundary(doc: &LoroDoc, text: &LoroText) -> FxHashMap { + let mut index: FxHashMap = FxHashMap::default(); + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID).ok().flatten() else { + return index; + }; + // §act-five P4: ONE pass (see `paragraph_ids_by_boundary`). Gather paragraph-kind + // blocks' decoded anchors once, batch-resolve, build the index from the + // pre-decoded cursors — no second scan, no second decode. Kind filter, + // sorted-key first-wins, main-body-at-0 special case, and dead/id-less-cursor + // fallbacks preserved. + let container = text.id(); + // §perf-heaven T4: consume the sorted keys by value (no per-key clone). + let sorted_keys = map_keys(&blocks); + let mut cands: Vec<(String, Option)> = Vec::with_capacity(sorted_keys.len()); + let mut ids: Vec = Vec::new(); + for key in sorted_keys { + let Some(block) = child_map(&blocks, &key).ok().flatten() else { + continue; + }; + if map_string_opt(&block, "kind").ok().flatten().as_deref() != Some("paragraph") { + continue; + } + let cursor = map_binary_opt(&block, "anchor_cursor") + .ok() + .flatten() + .and_then(|bytes| Cursor::decode(&bytes).ok()) + .filter(|cursor| cursor.container == container); + if let Some(id) = cursor.as_ref().and_then(|cursor| cursor.id) { + ids.push(id); + } + cands.push((key, cursor)); + } + let mut pos_by_id: FxHashMap = FxHashMap::default(); + if !ids.is_empty() { + for (id, pos) in ids + .iter() + .copied() + .zip(doc.inner().query_text_id_positions(&container, &ids)) + { + if let Some(pos) = pos { + pos_by_id.insert(id, pos); + } + } + } + let mut main_body_at_zero = false; + let text_len = text.len_unicode(); + for (key, cursor) in cands { + let Some(pos) = resolve_decoded_cursor(doc, text_len, cursor.as_ref(), &pos_by_id) else { + continue; + }; + if pos == 0 && key.as_str() == MAIN_BODY_BLOCK_ID { + main_body_at_zero = true; + } + index.entry(pos).or_insert(key); + } + if main_body_at_zero { + index.insert(0, MAIN_BODY_BLOCK_ID.to_string()); + } + index +} + +fn map_keys(map: &LoroMap) -> Vec { + let mut keys = map.keys().map(|key| key.to_string()).collect::>(); + keys.sort(); + keys +} + +fn loro_id_u128(id: &str) -> u128 { + if let Some(value) = id + .rsplit('.') + .next() + .and_then(|suffix| suffix.parse::().ok()) + { + return value; + } + let hash = blake3::hash(id.as_bytes()); + let mut bytes = [0_u8; 16]; + bytes.copy_from_slice(&hash.as_bytes()[..16]); + u128::from_le_bytes(bytes) +} + +/// Deterministic `(paragraph_key, block_key)` for the durable metadata records that +/// anchor `boundary` in `text` — the SINGLE source of these ids, shared by the +/// projection (which fabricates them when a boundary has no durable record) and the +/// runtime's repair writer (which materializes the records). Because both derive +/// the same key, a fabricated id and a later-repaired record's id are the SAME +/// value on every peer, so they converge instead of clobbering each other. +/// +/// The keys are POSITION-INDEPENDENT: boundary 0 is the canonical first paragraph; +/// every other boundary derives from the boundary newline's stable Loro `OpID` +/// (globally unique and insertion-stable), so unlike a `block_ix` or unicode-offset +/// key it does not change when text shifts and is identical across peers. The +/// non-numeric `op-…` suffix routes both keys through `loro_id_u128`'s hash (rather +/// than its trailing-number rule), keeping the paragraph and block ids distinct. +/// Returns `None` only when `boundary` has no live anchor (e.g. an empty container). +/// Materialize ONE table block from canonical state — the same law +/// `document_from_loro` applies to it, exposed so table-op patch synthesis +/// READS the committed table back instead of simulating the op on the old +/// projection. Simulation is a second doc→projection semantics and diverged +/// from canonical under undo-churned histories (found by the table-fuzz undo +/// arm as stale cell spans). A missing canonical record or any reported +/// defect is the caller's cue to fall back to the full rebuild. +pub fn materialize_table_block(doc: &LoroDoc, block_id: u128) -> io::Result<(InputTableBlock, Vec)> { + let projector = Projector::new(doc)?; + let record = child_map(&projector.blocks, &format!("table.{block_id}"))?.ok_or_else(|| invalid("table block record missing"))?; + let mut defects = Vec::new(); + let table = projector.table_block(&record, &mut defects)?; + Ok((table, defects)) +} + +#[must_use] +pub fn stable_boundary_metadata_keys(text: &LoroText, boundary: usize) -> Option<(String, String)> { + if boundary == 0 { + return Some((ROOT_FIRST_PARAGRAPH_ID.to_string(), MAIN_BODY_BLOCK_ID.to_string())); + } + let anchor = text.get_cursor(boundary, Side::Left)?.id?; + Some(( + format!("paragraph.anchor.op-{}-{}", anchor.peer, anchor.counter), + format!("paragraph_block.anchor.op-{}-{}", anchor.peer, anchor.counter), + )) +} + +fn child_map(parent: &LoroMap, key: &str) -> io::Result> { + Ok(parent.get(key).and_then(|value| match value { + ValueOrContainer::Container(container) => container.into_map().ok(), + ValueOrContainer::Value(_) => None, + })) +} + +fn child_text(parent: &LoroMap, key: &str) -> io::Result> { + Ok(parent.get(key).and_then(|value| match value { + ValueOrContainer::Container(container) => container.into_text().ok(), + ValueOrContainer::Value(_) => None, + })) +} + +/// §28: centralized resolution of a stored raw Loro container id string. +/// +/// Parses the durable `*_container_id` string into a [`ContainerID`] and fetches +/// the live container directly from the document for efficient runtime access. +/// Returns `None` when the id is missing/unparseable or the container is +/// absent/detached/deleted, so callers can fall back to map-key traversal. +fn resolve_container(doc: &LoroDoc, container_id: &str) -> Option { + let container = doc.get_container(ContainerID::try_from(container_id).ok()?)?; + (container.is_attached() && !container.is_deleted()).then_some(container) +} + +fn resolve_map_by_container_id(doc: &LoroDoc, container_id: &str) -> Option { + resolve_container(doc, container_id)?.into_map().ok() +} + +fn resolve_text_by_container_id(doc: &LoroDoc, container_id: &str) -> Option { + resolve_container(doc, container_id)?.into_text().ok() +} + +fn ordered_ids(map: &LoroMap, key: &str) -> io::Result> { + let Some(ValueOrContainer::Container(container)) = map.get(key) else { + return Ok(Vec::new()); + }; + let value = match container { + Container::MovableList(list) => list.get_deep_value(), + _ => return Ok(Vec::new()), + }; + Ok( + value + .into_list() + .unwrap_or_default() + .iter() + .filter_map(|value| match value { + LoroValue::String(value) => Some(value.to_string()), + _ => None, + }) + .collect(), + ) +} + +fn map_string(map: &LoroMap, key: &str) -> io::Result { + map_string_opt(map, key)?.ok_or_else(|| invalid(format!("missing string field `{key}`"))) +} + +fn map_string_opt(map: &LoroMap, key: &str) -> io::Result> { + Ok(map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + })) +} + +fn map_binary_opt(map: &LoroMap, key: &str) -> io::Result>> { + Ok(map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::Binary(value)) => Some(value.to_vec()), + _ => None, + })) +} + +fn map_i64_opt(map: &LoroMap, key: &str) -> io::Result> { + Ok(map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::I64(value)) => Some(value), + _ => None, + })) +} + +fn map_bool_opt(map: &LoroMap, key: &str) -> io::Result> { + Ok(map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::Bool(value)) => Some(value), + _ => None, + })) +} + +fn paragraph_style_from_attrs(attrs: Option<&FxHashMap>) -> Option { + let value = attrs?.get(MARK_PARAGRAPH_STYLE)?; + match value { + LoroValue::I64(0) => Some(gpui_flowtext::ParagraphStyle::Normal), + LoroValue::I64(slot) if *slot > 0 => u8::try_from(*slot - 1) + .ok() + .map(gpui_flowtext::ParagraphStyle::Custom), + _ => None, + } +} + +fn run_styles_from_attrs(attrs: Option<&FxHashMap>) -> RunStyles { + let mut styles = RunStyles::default(); + let Some(attrs) = attrs else { + return styles; + }; + if let Some(LoroValue::I64(slot)) = attrs.get(MARK_RUN_SEMANTIC_STYLE) + && let Ok(slot) = u8::try_from(*slot) + { + styles.semantic = RunSemanticStyle::Custom(slot); + } + if let Some(LoroValue::I64(slot)) = attrs.get(MARK_HIGHLIGHT_STYLE) + && let Ok(slot) = u8::try_from(*slot) + { + styles.highlight = Some(HighlightStyle::Custom(slot)); + } + if matches!(attrs.get(MARK_DIRECT_UNDERLINE), Some(LoroValue::Bool(true))) { + styles.direct_underline = true; + } + if matches!(attrs.get(MARK_STRIKETHROUGH), Some(LoroValue::Bool(true))) { + styles.strikethrough = true; + } + if let Some(LoroValue::I64(value)) = attrs.get(MARK_VERT_ALIGN) { + styles.vert_align = gpui_flowtext::VertAlign::from_mark_value(*value); + } + styles +} + +fn image_sizing(attrs: Option<&LoroMap>) -> io::Result { + let Some(attrs) = attrs else { + return Ok(InputImageSizing::FitWidth); + }; + match map_string_opt(attrs, "sizing")?.as_deref() { + Some("intrinsic") => Ok(InputImageSizing::Intrinsic), + Some("fixed") => Ok(InputImageSizing::Fixed { + width_px: map_i64_opt(attrs, "width_px")? + .and_then(i64_to_u32) + .unwrap_or(640), + height_px: map_i64_opt(attrs, "height_px")?.and_then(i64_to_u32), + }), + Some("fit_width") | None => Ok(InputImageSizing::FitWidth), + Some(_) => Ok(InputImageSizing::FitWidth), + } +} + +fn alignment(attrs: Option<&LoroMap>) -> io::Result { + let Some(attrs) = attrs else { + return Ok(InputBlockAlignment::Left); + }; + Ok(match map_string_opt(attrs, "alignment")?.as_deref() { + Some("center") => InputBlockAlignment::Center, + Some("right") => InputBlockAlignment::Right, + Some("left") | None => InputBlockAlignment::Left, + Some(_) => InputBlockAlignment::Left, + }) +} + +fn equation_syntax(attrs: Option<&LoroMap>) -> io::Result { + let Some(attrs) = attrs else { + return Ok(InputEquationSyntax::Latex); + }; + Ok(match map_string_opt(attrs, "syntax")?.as_deref() { + Some("latex") | None => InputEquationSyntax::Latex, + Some(_) => InputEquationSyntax::Latex, + }) +} + +fn equation_display(attrs: Option<&LoroMap>) -> io::Result { + let Some(attrs) = attrs else { + return Ok(InputEquationDisplay::Display); + }; + Ok(match map_string_opt(attrs, "display")?.as_deref() { + Some("inline_like_paragraph") => InputEquationDisplay::InlineLikeParagraph, + Some("display") | None => InputEquationDisplay::Display, + Some(_) => InputEquationDisplay::Display, + }) +} + +fn table_column_width(column: &LoroMap) -> io::Result { + Ok(match map_string_opt(column, "width_kind")?.as_deref() { + Some("fixed_px") => InputTableColumnWidth::FixedPx( + map_i64_opt(column, "width_px")? + .and_then(i64_to_u32) + .unwrap_or(120), + ), + Some("fraction") => InputTableColumnWidth::Fraction( + map_i64_opt(column, "fraction")? + .and_then(i64_to_u32) + .unwrap_or(1), + ), + Some("auto") | None => InputTableColumnWidth::Auto, + Some(_) => InputTableColumnWidth::Auto, + }) +} + +/// Map a pure [`table_topology`] grid defect onto a [`ProjectionDefect`] the +/// runtime repair pipeline understands (§P2b / FS-010). +fn map_topology_defect(block_key: &str, defect: &table_topology::TableTopologyDefect) -> ProjectionDefect { + use table_topology::TableTopologyDefect as Defect; + let (row_id, column_id, kind) = match defect { + Defect::MissingCell { row_id, column_id } => (row_id.0, column_id.0, TableTopologyKind::MissingCell), + Defect::DuplicateCoordinate { row_id, column_id } => (row_id.0, column_id.0, TableTopologyKind::DuplicateCoordinate), + Defect::InvalidSpan { row_id, column_id } => (row_id.0, column_id.0, TableTopologyKind::InvalidSpan), + Defect::OrphanCell { row_id, column_id } => (row_id.0, column_id.0, TableTopologyKind::OrphanCell), + }; + ProjectionDefect::TableTopology { + table_block_key: block_key.to_string(), + row_id: Some(row_id), + column_id: Some(column_id), + kind, + } +} + +/// The deterministic empty cell/quarantine paragraph the projector emits for a +/// synthesized or empty table cell. +fn empty_input_paragraph() -> InputParagraph { + InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: Vec::new(), + } +} + +fn parse_u128(value: &str) -> Option { + value.parse::().ok() +} + +fn i64_to_u32(value: i64) -> Option { + u32::try_from(value).ok() +} + +fn i64_to_u16(value: i64) -> Option { + u16::try_from(value).ok() +} + +fn i64_to_u8(value: i64) -> Option { + u8::try_from(value).ok() +} + +fn section_id_field(map: &LoroMap, key: &str) -> io::Result> { + Ok(map_string_opt(map, key)?.and_then(|value| parse_u128(&value))) +} + +/// §11: read a section's page-structure attributes back from the canonical Loro +/// document, substituting documented defaults for any missing keys. Returns +/// `None` only when the named section does not exist. +/// +/// `DocumentProjection` now carries these attrs on `DocumentSection::page`, +/// populated from canonical Loro during projection (see +/// [`Projector::sections_for_projection`]). This helper remains the direct, +/// single-section read-back path for callers that only need one section's attrs +/// without projecting the whole document. The canonical values always live in +/// Loro and round-trip losslessly. +#[must_use] +pub fn section_page_attrs(doc: &LoroDoc, section_id: &str) -> Option { + let root = doc.get_map(ROOT); + let sections = child_map(&root, SECTIONS_BY_ID).ok().flatten()?; + let section = child_map(§ions, section_id).ok().flatten()?; + let attrs = child_map(§ion, "attrs").ok().flatten()?; + Some(crate::loro_schema::read_section_page_attrs(&attrs)) +} + +/// §11: map canonical Loro page-structure attrs +/// (`crate::loro_schema::SectionPageAttrs`) onto the gpui-flowtext projection +/// mirror (`gpui_flowtext::SectionPageAttrs`). gpui-flowtext cannot depend on +/// `flowstate-document`, so the two types are defined field-for-field +/// identically and this is a direct copy. Fully-qualified paths disambiguate the +/// clashing type names. Takes the canonical attrs by value so the owned +/// header/footer flow id strings move rather than clone. +fn project_section_page_attrs(attrs: crate::loro_schema::SectionPageAttrs) -> gpui_flowtext::SectionPageAttrs { + gpui_flowtext::SectionPageAttrs { + page_size: gpui_flowtext::SectionPageSize { + width_twips: attrs.page_size.width_twips, + height_twips: attrs.page_size.height_twips, + }, + margins: gpui_flowtext::SectionMargins { + top_twips: attrs.margins.top_twips, + right_twips: attrs.margins.right_twips, + bottom_twips: attrs.margins.bottom_twips, + left_twips: attrs.margins.left_twips, + }, + columns: attrs.columns, + orientation: match attrs.orientation { + crate::loro_schema::SectionOrientation::Portrait => gpui_flowtext::SectionOrientation::Portrait, + crate::loro_schema::SectionOrientation::Landscape => gpui_flowtext::SectionOrientation::Landscape, + }, + page_numbering: gpui_flowtext::SectionPageNumbering { + format: match attrs.page_numbering.format { + crate::loro_schema::PageNumberFormat::None => gpui_flowtext::PageNumberFormat::None, + crate::loro_schema::PageNumberFormat::Decimal => gpui_flowtext::PageNumberFormat::Decimal, + crate::loro_schema::PageNumberFormat::LowerRoman => gpui_flowtext::PageNumberFormat::LowerRoman, + crate::loro_schema::PageNumberFormat::UpperRoman => gpui_flowtext::PageNumberFormat::UpperRoman, + crate::loro_schema::PageNumberFormat::LowerAlpha => gpui_flowtext::PageNumberFormat::LowerAlpha, + crate::loro_schema::PageNumberFormat::UpperAlpha => gpui_flowtext::PageNumberFormat::UpperAlpha, + }, + start: attrs.page_numbering.start, + }, + header_flow_id: attrs.header_flow_id, + footer_flow_id: attrs.footer_flow_id, + } +} + +fn invalid(message: impl Into) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{document_to_loro, flowstate_document_theme, loro_schema::body_text}; + use gpui_flowtext::{ + InputBlock, InputBlockAlignment, InputImageBlock, InputImageSizing, InputParagraph, InputRun, RunStyles, document_from_input_blocks, + }; + + /// §act-four M4 cold viewport load: `materialize_viewport` materializes an + /// arbitrary body region byte-identically to the corresponding slice of the + /// full `document_from_loro` rebuild — WITHOUT building the whole projection. + #[test] + fn materialize_viewport_matches_the_full_rebuild_slice() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + (0..12) + .map(|ix| { + InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: vec![InputRun { + text: format!("paragraph {ix} — naïve café ☃"), + styles: RunStyles::default(), + }], + }) + }) + .collect(), + ); + let doc = document_to_loro(&source, "Viewport load")?; + let body = body_text(&doc); + let full = projection_blocks_from_loro(&doc)?.blocks; // full rebuild, Vec + + // (1) The whole-doc viewport equals the full rebuild, block-for-block. + let whole = materialize_viewport(&doc, 0, body.len_unicode())?; + assert_eq!(whole.blocks, full, "whole-doc viewport == full rebuild"); + + // (2) A mid-doc sub-viewport equals the corresponding slice of the full + // rebuild — the cold random-scroll case. Boundaries (sorted by position) + // index paragraphs; [boundary[6], boundary[9]) covers paragraphs 6,7,8. + let mut boundaries: Vec = paragraph_ids_by_boundary(&doc, &body) + .keys() + .copied() + .collect(); + boundaries.sort_unstable(); + assert!(boundaries.len() >= 10, "enough rows to sub-viewport"); + let viewport = materialize_viewport(&doc, boundaries[6], boundaries[9])?; + assert_eq!(viewport.blocks, full[6..9].to_vec(), "sub-viewport == full[6..9], byte-identical"); + Ok(()) + } + + /// The vendored Loro batch resolver (`query_text_id_positions`, driving + /// `boundary_cursor_positions`) must produce EXACTLY the positions the per-id + /// `get_cursor_pos` produces — that equivalence is the entire correctness basis + /// for replacing the O(records²) per-cursor scan. Resolves every boundary cursor + /// in one batch call (exercising the by-peer grouping + binary search) and checks + /// each against `get_cursor_pos`; multibyte content is included so any + /// unicode/event index mismatch would surface. + #[test] + fn batch_cursor_resolver_matches_per_cursor_get_cursor_pos() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + (0..40) + .map(|ix| { + InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: vec![InputRun { + text: format!("paragraph {ix} — naïve café ☃ tail"), + styles: RunStyles::default(), + }], + }) + }) + .collect(), + ); + let doc = document_to_loro(&source, "Batch cursor equivalence")?; + let root = doc.get_map(ROOT); + let body = body_text(&doc); + let paragraphs = child_map(&root, PARAGRAPHS_BY_ID)?.expect("paragraphs map"); + + let mut cursors: Vec<(ID, Cursor)> = Vec::new(); + for key in map_keys(¶graphs) { + let Some(record) = child_map(¶graphs, &key)? else { + continue; + }; + for field in ["boundary_cursor", "start_cursor"] { + let Some(bytes) = map_binary_opt(&record, field)? else { + continue; + }; + let Ok(cursor) = Cursor::decode(&bytes) else { + continue; + }; + if cursor.container == body.id() + && let Some(id) = cursor.id + { + cursors.push((id, cursor)); + } + } + } + let ids: Vec = cursors.iter().map(|(id, _)| *id).collect(); + let batch = doc.inner().query_text_id_positions(&body.id(), &ids); + assert_eq!(batch.len(), ids.len(), "one result per queried id"); + + let mut checked = 0usize; + for ((_, cursor), batch_pos) in cursors.iter().zip(&batch) { + let per_cursor = doc + .get_cursor_pos(cursor) + .ok() + .map(|result| result.current.pos); + if let Some(pos) = batch_pos { + assert_eq!(Some(*pos), per_cursor, "batch resolver disagreed with get_cursor_pos"); + checked += 1; + } + } + assert!( + checked >= 40, + "expected >=40 live boundary cursors resolved by the batch path, got {checked}" + ); + Ok(()) + } + + /// Stronger equivalence: the batch resolver must still equal `get_cursor_pos` + /// after fragmenting edits (insert/delete/re-insert) AND a concurrent multi-peer + /// merge — the live collab state the fresh-doc test above doesn't reach. If the + /// vendored resolver were wrong here, the full projection rebuild would assign + /// different ids than the per-cursor path and show up as projection divergence. + #[test] + fn batch_resolver_matches_per_cursor_after_edits_and_merge() { + use loro::{ExportMode, cursor::Side}; + let a = LoroDoc::new(); + a.set_peer_id(1).unwrap(); + let ta = a.get_text("t"); + ta.insert(0, "the quick brown fox jumps over").unwrap(); + a.commit(); + ta.delete(4, 6).unwrap(); // remove "quick " + ta.insert(4, "SLOW ").unwrap(); + a.commit(); + ta.insert(0, "well, ").unwrap(); + a.commit(); + // Peer 2: snapshot from A, edit concurrently, merge back so `a` holds chunks + // authored by two different peers. + let b = LoroDoc::new(); + b.set_peer_id(2).unwrap(); + b.import(&a.export(ExportMode::Snapshot).unwrap()).unwrap(); + let tb = b.get_text("t"); + tb.insert(0, "PREFIX ").unwrap(); + let tb_end = tb.len_unicode(); + tb.insert(tb_end, " suffix").unwrap(); + b.commit(); + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + a.commit(); + + let text = a.get_text("t"); + let container = text.id(); + let len = text.len_unicode(); + let mut ids: Vec = Vec::new(); + let mut cursors = Vec::new(); + for pos in 0..=len { + for side in [Side::Left, Side::Right] { + if let Some(cursor) = text.get_cursor(pos, side) + && let Some(id) = cursor.id + { + ids.push(id); + cursors.push(cursor); + } + } + } + assert!(!ids.is_empty(), "should have collected some cursors"); + + let batch = a.inner().query_text_id_positions(&container, &ids); + assert_eq!(batch.len(), ids.len()); + let mut compared = 0usize; + for (cursor, batch_pos) in cursors.iter().zip(&batch) { + let per_cursor = a + .get_cursor_pos(cursor) + .ok() + .map(|result| result.current.pos); + if let Some(pos) = batch_pos { + assert_eq!( + Some(*pos), + per_cursor, + "batch resolver diverged from get_cursor_pos on multi-peer/edited text" + ); + compared += 1; + } + } + assert!(compared > 0, "should have compared at least some live cursors"); + } + + /// The fabricated/repair id derivation must be POSITION-INDEPENDENT: the same + /// boundary newline keeps the same key after text is inserted ahead of it (it + /// shifts position but its `OpID` is unchanged). This is what lets a fabricated id + /// and a later-repaired record's id converge instead of diverging as the old + /// `block_ix` / unicode-offset keys did. + #[test] + fn stable_boundary_keys_are_position_independent() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + vec![ + InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "alpha".into(), + styles: RunStyles::default(), + }], + }), + InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "bravo".into(), + styles: RunStyles::default(), + }], + }), + ], + ); + let doc = document_to_loro(&source, "Stable boundary keys")?; + let body = body_text(&doc); + + // A non-zero boundary (so it exercises the OpID path, not the boundary-0 seed). + let boundary = body + .to_string() + .chars() + .enumerate() + .filter_map(|(i, c)| (c == '\n').then_some(i)) + .find(|&i| i > 0) + .expect("a non-zero boundary"); + let before = stable_boundary_metadata_keys(&body, boundary).expect("keys for a live boundary"); + assert!( + before.0.contains("op-") && before.1.contains("op-"), + "non-zero boundary keys derive from the OpID: {before:?}" + ); + assert_ne!(before.0, before.1, "paragraph and block keys must be distinct"); + + // Insert ahead of the boundary so it shifts by 4 unicode positions. + body.insert(1, "XXXX").expect("insert"); + doc.commit(); + let after = stable_boundary_metadata_keys(&body, boundary + 4).expect("keys after shift"); + assert_eq!(before, after, "the same newline keeps the same key after shifting"); + Ok(()) + } + + #[test] + fn projection_preserves_loro_paragraph_and_block_ids() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + vec![ + InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "before".to_string(), + styles: RunStyles::default(), + }], + }), + InputBlock::Image(InputImageBlock { + asset_id: AssetId(42), + alt_text: "alt".into(), + caption: None, + sizing: InputImageSizing::FitWidth, + alignment: InputBlockAlignment::Left, + external_url: None, + }), + ], + ); + let doc = document_to_loro(&source, "Projection ids")?; + let body = body_text(&doc); + let root = doc.get_map(ROOT); + let blocks = child_map(&root, BLOCKS_BY_ID)?.expect("blocks map"); + let first_paragraph_id = paragraph_ids_by_boundary(&doc, &body) + .get(&0) + .cloned() + .expect("first paragraph id"); + let first_block_id = paragraph_block_ids_by_boundary(&doc, &body) + .get(&0) + .cloned() + .expect("first paragraph block id"); + let image_id = map_keys(&blocks) + .into_iter() + .find(|key| { + child_map(&blocks, key) + .ok() + .flatten() + .and_then(|block| map_string_opt(&block, "kind").ok().flatten()) + .as_deref() + == Some("image") + }) + .expect("image block id"); + + let projected = document_from_loro(&doc)?; + + assert_eq!(projected.ids.paragraph_ids[0], ParagraphId(loro_id_u128(&first_paragraph_id))); + assert_eq!(projected.ids.block_ids[0], BlockId(loro_id_u128(&first_block_id))); + assert_eq!(projected.ids.block_ids[1], BlockId(loro_id_u128(&image_id))); + Ok(()) + } + + #[test] + fn section_page_attrs_read_back_from_loro() { + let doc = crate::loro_schema::new_loro_document("Sections").expect("new Loro document"); + let expected = crate::loro_schema::SectionPageAttrs { + columns: 3, + orientation: crate::loro_schema::SectionOrientation::Landscape, + ..crate::loro_schema::SectionPageAttrs::default() + }; + crate::loro_schema::set_section_page_attrs(&doc, "section.alpha", &expected).expect("set section page attrs"); + + assert_eq!(section_page_attrs(&doc, "section.alpha"), Some(expected)); + assert_eq!(section_page_attrs(&doc, "section.missing"), None); + } + + #[test] + fn missing_paragraph_metadata_is_deterministic_and_quarantines_content() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + vec![InputBlock::Paragraph(InputParagraph { + style: gpui_flowtext::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "before".to_string(), + styles: RunStyles::default(), + }], + })], + ); + let doc = document_to_loro(&source, "Missing metadata")?; + let body = body_text(&doc); + // Append a second paragraph boundary that carries a style mark but has no + // durable paragraph metadata record. + let end = body.len_unicode(); + body.insert(end, "\nextra").unwrap(); + body + .mark(end..end + 1, MARK_PARAGRAPH_STYLE, 0_i64) + .unwrap(); + doc.commit(); + + let (projection, defects) = document_from_loro_with_defects(&doc)?; + let (_, defects_again) = document_from_loro_with_defects(&doc)?; + assert_eq!(defects, defects_again, "projection defects must be deterministic across passes"); + assert!(defects.iter().any(|defect| matches!( + defect, + ProjectionDefect::MissingParagraphMetadata { + boundary_unicode: Some(_), + .. + } + ))); + // The paragraph is quarantined with a fabricated id — its content survives. + assert_eq!(projection.paragraphs.len(), 2); + assert_eq!(gpui_flowtext::paragraph_text(&projection, 1), "extra"); + Ok(()) + } + + #[test] + fn unresolved_object_anchor_is_quarantined_not_dropped() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + vec![InputBlock::Image(InputImageBlock { + asset_id: AssetId(7), + alt_text: "alt".into(), + caption: None, + sizing: InputImageSizing::FitWidth, + alignment: InputBlockAlignment::Left, + external_url: None, + })], + ); + let doc = document_to_loro(&source, "Unresolved anchor")?; + let body = body_text(&doc); + // Delete the object placeholder, leaving the block's anchor dangling. + let placeholder = body + .to_string() + .chars() + .position(|ch| ch == OBJECT_REPLACEMENT) + .expect("object placeholder"); + body.delete(placeholder, 1).unwrap(); + doc.commit(); + + let (projection, defects) = document_from_loro_with_defects(&doc)?; + let (_, defects_again) = document_from_loro_with_defects(&doc)?; + assert_eq!(defects, defects_again, "quarantine reporting must be deterministic"); + assert!( + defects + .iter() + .any(|defect| matches!(defect, ProjectionDefect::UnresolvedObjectAnchor { .. })) + ); + // The block is quarantined (appended), not silently dropped. + assert!( + projection + .blocks + .iter() + .any(|block| matches!(block, gpui_flowtext::Block::Image(_))) + ); + Ok(()) + } + + #[test] + fn invalid_asset_id_is_reported_and_placeholdered() -> io::Result<()> { + let source = document_from_input_blocks( + DocumentTheme::clone(&flowstate_document_theme()), + vec![InputBlock::Image(InputImageBlock { + asset_id: AssetId(42), + alt_text: "alt".into(), + caption: None, + sizing: InputImageSizing::FitWidth, + alignment: InputBlockAlignment::Left, + external_url: None, + })], + ); + let doc = document_to_loro(&source, "Invalid asset")?; + let root = doc.get_map(ROOT); + let blocks = child_map(&root, BLOCKS_BY_ID)?.expect("blocks map"); + let image_key = map_keys(&blocks) + .into_iter() + .find(|key| { + child_map(&blocks, key) + .ok() + .flatten() + .and_then(|block| map_string_opt(&block, "kind").ok().flatten()) + .as_deref() + == Some("image") + }) + .expect("image block"); + let image = child_map(&blocks, &image_key)?.expect("image block map"); + image.insert("asset_id", "not-a-number").unwrap(); + doc.commit(); + + let (projection, defects) = document_from_loro_with_defects(&doc)?; + assert!( + defects + .iter() + .any(|defect| matches!(defect, ProjectionDefect::InvalidAssetId { .. })) + ); + // Never silently coerced away: projected as the deterministic AssetId(0) placeholder. + assert!( + projection + .blocks + .iter() + .any(|block| matches!(block, gpui_flowtext::Block::Image(image) if image.asset_id == AssetId(0))) + ); + Ok(()) + } +} diff --git a/crates/flowstate-document/src/loro_schema.rs b/crates/flowstate-document/src/loro_schema.rs new file mode 100644 index 00000000..7f4c188a --- /dev/null +++ b/crates/flowstate-document/src/loro_schema.rs @@ -0,0 +1,820 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use loro::{ + ContainerTrait as _, ExpandType, LoroDoc, LoroMap, LoroResult, LoroText, LoroValue, StyleConfig, StyleConfigMap, ValueOrContainer, + cursor::Side, +}; +use uuid::Uuid; + +use crate::LORO_SCHEMA_VERSION; + +pub const ROOT: &str = "flowstate.root"; +pub const META: &str = "meta"; +pub const FLOWS_BY_ID: &str = "flows_by_id"; +pub const BLOCKS_BY_ID: &str = "blocks_by_id"; +pub const PARAGRAPHS_BY_ID: &str = "paragraphs_by_id"; +pub const SECTIONS_BY_ID: &str = "sections_by_id"; +pub const ASSETS_BY_ID: &str = "assets_by_id"; +pub const REVISIONS: &str = "revisions"; +pub const USERS_BY_ID: &str = "users_by_id"; +pub const COMMENTS_BY_ID: &str = "comments_by_id"; +pub const REPLICAS_BY_ID: &str = "replicas_by_id"; + +pub const FLOW_TEXT_KEY: &str = "text"; +pub const FLOW_ATTRS_KEY: &str = "attrs"; +pub const FLOW_KIND_KEY: &str = "kind"; +pub const FLOW_ID_KEY: &str = "id"; + +pub const BODY_FLOW_ID: &str = "body"; +pub const ROOT_BODY_FLOW_ID: &str = "body"; +pub const ROOT_FIRST_PARAGRAPH_ID: &str = "paragraph.initial"; +pub const MAIN_BODY_BLOCK_ID: &str = "block.body.initial"; + +pub const MARK_PARAGRAPH_STYLE: &str = "paragraph_style"; +pub const MARK_RUN_SEMANTIC_STYLE: &str = "run_semantic_style_id"; +pub const MARK_HIGHLIGHT_STYLE: &str = "highlight_style_id"; +pub const MARK_DIRECT_UNDERLINE: &str = "direct_underline"; +pub const MARK_STRIKETHROUGH: &str = "strikethrough"; +/// Superscript/subscript run mark (OOXML `w:vertAlign`). Value: `1` superscript, +/// `2` subscript; absent ⇒ baseline. Currently written by the `.docx` import and +/// read by the projection (round-trips), but not yet emitted by the live editor — +/// see [`gpui_flowtext::VertAlign`]. +pub const MARK_VERT_ALIGN: &str = "vert_align"; + +pub const OBJECT_REPLACEMENT: char = '\u{FFFC}'; +pub const SENTINEL_NEWLINE: &str = "\n"; + +// §P2b durable-table containers (plan §12). Centralized so the writer, the +// projector, and the collab runtime address the identical child containers. +pub const TABLE_KEY: &str = "table"; +pub const TABLE_ROW_ORDER: &str = "row_order"; +pub const TABLE_COLUMN_ORDER: &str = "column_order"; +pub const TABLE_ROWS_BY_ID: &str = "rows_by_id"; +pub const TABLE_COLUMNS_BY_ID: &str = "columns_by_id"; +pub const TABLE_CELLS_BY_ID: &str = "cells_by_id"; + +/// Loro string id for a durable table row (§P2b): `row.{u128}`. The dotted form +/// ends in the row's unique numeric id so the canonical `loro_id_u128` decoder +/// (which parses the trailing segment) round-trips it. +#[must_use] +pub fn row_loro_id(row: gpui_flowtext::RowId) -> String { + format!("row.{}", row.0) +} + +/// Loro string id for a durable table column (§P2b): `column.{u128}`. +#[must_use] +pub fn column_loro_id(column: gpui_flowtext::ColumnId) -> String { + format!("column.{}", column.0) +} + +/// Loro string id for a durable table cell (§P2b): `cell.{u128}` where the u128 +/// is the deterministic [`gpui_flowtext::CellId`] mix of its coordinate. +#[must_use] +pub fn cell_loro_id(cell: gpui_flowtext::CellId) -> String { + format!("cell.{}", cell.0) +} + +/// Loro string id for the cell at coordinate `(row, column)` (§P2b). +#[must_use] +pub fn cell_loro_id_for(row: gpui_flowtext::RowId, column: gpui_flowtext::ColumnId) -> String { + cell_loro_id(gpui_flowtext::CellId::from_coordinate(row, column)) +} + +/// Cell text flow id for a cell string id (§P2b): `{cell_id}.flow`. +#[must_use] +pub fn cell_flow_loro_id(cell_loro_id: &str) -> String { + format!("{cell_loro_id}.flow") +} + +/// Parse the trailing numeric segment of a dotted Loro id into a `u128`. +pub fn loro_id_trailing_u128(id: &str) -> Option { + parse_trailing_u128(id) +} + +fn parse_trailing_u128(id: &str) -> Option { + id.rsplit('.') + .next() + .and_then(|segment| segment.parse().ok()) +} + +/// Decode a `row.{u128}` Loro id into a [`gpui_flowtext::RowId`] (§P2b). +#[must_use] +pub fn parse_row_loro_id(id: &str) -> Option { + parse_trailing_u128(id).map(gpui_flowtext::RowId) +} + +/// Decode a `column.{u128}` Loro id into a [`gpui_flowtext::ColumnId`] (§P2b). +#[must_use] +pub fn parse_column_loro_id(id: &str) -> Option { + parse_trailing_u128(id).map(gpui_flowtext::ColumnId) +} + +/// Decode a `cell.{u128}` Loro id into a [`gpui_flowtext::CellId`] (§P2b). +#[must_use] +pub fn parse_cell_loro_id(id: &str) -> Option { + parse_trailing_u128(id).map(gpui_flowtext::CellId) +} + +/// §11 page-structure attribute keys stored on a `SectionMap`'s `attrs` map. +/// +/// Encoding (documented and locked): +/// * page size: two `i64` keys in twips (1/1440 inch) — `page_width_twips`, +/// `page_height_twips`. US Letter defaults to `12240 x 15840`. +/// * margins: four `i64` keys in twips — `margin_{top,right,bottom,left}_twips`. +/// The default 1-inch margin is `1440` twips. +/// * columns: one `i64` key — `columns` (default `1`). +/// * orientation: one string key — `orientation` (`"portrait"` | `"landscape"`). +/// * page numbering: a small struct stored as `page_numbering_format` (string) +/// plus `page_numbering_start` (`i64`, default `1`). +/// * header/footer: optional `header_flow_id` / `footer_flow_id` string keys whose +/// values name independent header/footer text flows (kinds `"header"`/`"footer"`). +/// +/// Twips are used because they are the DOCX-native, integer-exact unit, keeping the +/// first-class DOCX import/export pipeline lossless. +pub const SECTION_ATTR_PAGE_WIDTH: &str = "page_width_twips"; +pub const SECTION_ATTR_PAGE_HEIGHT: &str = "page_height_twips"; +pub const SECTION_ATTR_MARGIN_TOP: &str = "margin_top_twips"; +pub const SECTION_ATTR_MARGIN_RIGHT: &str = "margin_right_twips"; +pub const SECTION_ATTR_MARGIN_BOTTOM: &str = "margin_bottom_twips"; +pub const SECTION_ATTR_MARGIN_LEFT: &str = "margin_left_twips"; +pub const SECTION_ATTR_COLUMNS: &str = "columns"; +pub const SECTION_ATTR_ORIENTATION: &str = "orientation"; +pub const SECTION_ATTR_PAGE_NUMBERING_FORMAT: &str = "page_numbering_format"; +pub const SECTION_ATTR_PAGE_NUMBERING_START: &str = "page_numbering_start"; +pub const SECTION_ATTR_HEADER_FLOW_ID: &str = "header_flow_id"; +pub const SECTION_ATTR_FOOTER_FLOW_ID: &str = "footer_flow_id"; + +/// US Letter width in twips (8.5in x 1440). +const DEFAULT_PAGE_WIDTH_TWIPS: i64 = 12_240; +/// US Letter height in twips (11in x 1440). +const DEFAULT_PAGE_HEIGHT_TWIPS: i64 = 15_840; +/// One-inch margin in twips. +const DEFAULT_MARGIN_TWIPS: i64 = 1_440; + +/// Section page orientation (§11). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SectionOrientation { + Portrait, + Landscape, +} + +impl SectionOrientation { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Portrait => "portrait", + Self::Landscape => "landscape", + } + } + + #[must_use] + pub fn from_attr(value: Option<&str>) -> Self { + match value { + Some("landscape") => Self::Landscape, + _ => Self::Portrait, + } + } +} + +/// Page-number rendering format for a section (§11). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PageNumberFormat { + None, + Decimal, + LowerRoman, + UpperRoman, + LowerAlpha, + UpperAlpha, +} + +impl PageNumberFormat { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Decimal => "decimal", + Self::LowerRoman => "lower_roman", + Self::UpperRoman => "upper_roman", + Self::LowerAlpha => "lower_alpha", + Self::UpperAlpha => "upper_alpha", + } + } + + #[must_use] + pub fn from_attr(value: Option<&str>) -> Self { + match value { + Some("decimal") => Self::Decimal, + Some("lower_roman") => Self::LowerRoman, + Some("upper_roman") => Self::UpperRoman, + Some("lower_alpha") => Self::LowerAlpha, + Some("upper_alpha") => Self::UpperAlpha, + _ => Self::None, + } + } +} + +/// Page size for a section, in twips (§11). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SectionPageSize { + pub width_twips: i64, + pub height_twips: i64, +} + +impl Default for SectionPageSize { + fn default() -> Self { + Self { + width_twips: DEFAULT_PAGE_WIDTH_TWIPS, + height_twips: DEFAULT_PAGE_HEIGHT_TWIPS, + } + } +} + +/// Section margins, in twips (§11). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SectionMargins { + pub top_twips: i64, + pub right_twips: i64, + pub bottom_twips: i64, + pub left_twips: i64, +} + +impl Default for SectionMargins { + fn default() -> Self { + Self { + top_twips: DEFAULT_MARGIN_TWIPS, + right_twips: DEFAULT_MARGIN_TWIPS, + bottom_twips: DEFAULT_MARGIN_TWIPS, + left_twips: DEFAULT_MARGIN_TWIPS, + } + } +} + +/// Section page-numbering descriptor (§11). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SectionPageNumbering { + pub format: PageNumberFormat, + pub start: i64, +} + +impl Default for SectionPageNumbering { + fn default() -> Self { + Self { + format: PageNumberFormat::None, + start: 1, + } + } +} + +/// §11 page-structure attributes carried by a `SectionMap`. +/// +/// These are canonical in Loro regardless of whether a projection type can hold +/// them. They round-trip losslessly through [`write_section_page_attrs`] / +/// [`read_section_page_attrs`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SectionPageAttrs { + pub page_size: SectionPageSize, + pub margins: SectionMargins, + pub columns: i64, + pub orientation: SectionOrientation, + pub page_numbering: SectionPageNumbering, + pub header_flow_id: Option, + pub footer_flow_id: Option, +} + +impl Default for SectionPageAttrs { + fn default() -> Self { + Self { + page_size: SectionPageSize::default(), + margins: SectionMargins::default(), + columns: 1, + orientation: SectionOrientation::Portrait, + page_numbering: SectionPageNumbering::default(), + header_flow_id: None, + footer_flow_id: None, + } + } +} + +pub fn new_loro_document(title: &str) -> LoroResult { + let doc = LoroDoc::new(); + init_loro_document(&doc, title)?; + Ok(doc) +} + +pub(crate) fn new_loro_import_document(title: &str) -> LoroResult { + let doc = LoroDoc::new(); + init_loro_document_structure(&doc, title, false)?; + Ok(doc) +} + +pub fn init_loro_document(doc: &LoroDoc, title: &str) -> LoroResult<()> { + init_loro_document_structure(doc, title, true)?; + doc.commit(); + Ok(()) +} + +fn init_loro_document_structure(doc: &LoroDoc, title: &str, include_initial_paragraph: bool) -> LoroResult<()> { + configure_text_styles(doc); + + let root = doc.get_map(ROOT); + let meta = root.ensure_mergeable_map(META)?; + let flows = root.ensure_mergeable_map(FLOWS_BY_ID)?; + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; + let sections = root.ensure_mergeable_map(SECTIONS_BY_ID)?; + let assets = root.ensure_mergeable_map(ASSETS_BY_ID)?; + let revisions = root.ensure_mergeable_list(REVISIONS)?; + let users = root.ensure_mergeable_map(USERS_BY_ID)?; + let replicas = root.ensure_mergeable_map(REPLICAS_BY_ID)?; + + init_meta(&meta, title)?; + meta.insert("root_container_id", root.id().to_string())?; + meta.insert("flows_container_id", flows.id().to_string())?; + meta.insert("blocks_container_id", blocks.id().to_string())?; + meta.insert("paragraphs_container_id", paragraphs.id().to_string())?; + meta.insert("sections_container_id", sections.id().to_string())?; + meta.insert("assets_container_id", assets.id().to_string())?; + meta.insert("revisions_container_id", revisions.id().to_string())?; + meta.insert("users_container_id", users.id().to_string())?; + meta.insert("replicas_container_id", replicas.id().to_string())?; + // `ensure_flow` establishes the body flow's text + attrs containers for both + // the seeded and the import (`include_initial_paragraph == false`) paths. + ensure_flow(&flows, ROOT_BODY_FLOW_ID, "body")?; + if include_initial_paragraph { + // §P2a: route fresh-document creation through the unified seed so new + // documents and the runtime empty-body repair path converge on identical + // canonical state. + seed_document_body(doc)?; + } + Ok(()) +} + +/// §P2a unified canonical document seed. Establishes exactly the canonical body +/// state that both fresh-document creation and the runtime empty-body / +/// `MissingParagraph*` repair path must converge on: +/// * the boundary-0 [`SENTINEL_NEWLINE`] carrying a `Normal` paragraph-style mark, +/// * the first paragraph record ([`ROOT_FIRST_PARAGRAPH_ID`]) and its paired +/// paragraph block record ([`MAIN_BODY_BLOCK_ID`]). +/// +/// Section page defaults are supplied lazily by [`read_section_page_attrs`] +/// (which substitutes documented US-Letter defaults for any missing section +/// `attrs`), so the seed does not need to materialize a section container to make +/// a new/repaired document render with correct page structure. +/// +/// Idempotent: every step is an `ensure_*` / starts-with check, so seeding an +/// already-seeded document — or two peers seeding the same body concurrently — +/// converges via Loro map LWW semantics without duplicating structure. This +/// function does **not** commit; the caller controls the commit (and, for the +/// runtime repair path, its origin). +pub fn seed_document_body(doc: &LoroDoc) -> LoroResult<()> { + let root = doc.get_map(ROOT); + let flows = root.ensure_mergeable_map(FLOWS_BY_ID)?; + let blocks = root.ensure_mergeable_map(BLOCKS_BY_ID)?; + let paragraphs = root.ensure_mergeable_map(PARAGRAPHS_BY_ID)?; + let body_flow = ensure_flow(&flows, ROOT_BODY_FLOW_ID, "body")?; + let body_text = body_flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + body_flow.ensure_mergeable_map(FLOW_ATTRS_KEY)?; + ensure_sentinel(&body_text)?; + ensure_initial_paragraph(¶graphs, &blocks, &body_text)?; + // §P2a fidelity: the canonical seed establishes exactly the sentinel + + // first-paragraph state that the projection integrity and import invariants + // assert. Trace each seeding so a convergence/repair trail is visible. + fidelity::event(FidelityClass::Structure, "seed-body", || { + format!( + "seeded canonical body sentinel + first paragraph (body_unicode_len={})", + body_text.len_unicode() + ) + }); + Ok(()) +} + +/// Per-key mark expand table (Loro-first spec §9, D7). +/// +/// Run-style marks expand `After`: typing at the end of a styled run inherits +/// the style natively (Peritext semantics — what a word processor does), so a +/// plain text insert emits ZERO style operations. `paragraph_style` stays +/// `None`: it anchors the boundary sentinel newline and must never bleed into +/// text. Two riders enforced by the write path: +/// * caret style *overrides* (style toggled at the caret) explicitly +/// mark/unmark exactly the inserted range; +/// * `split_paragraph` unmarks all run-style keys on its inserted `\n` +/// (sentinel hygiene) — an expand-`After` run ending at the split point +/// would otherwise absorb the sentinel and bleed into the next paragraph. +pub fn configure_text_styles(doc: &LoroDoc) { + let mut styles = StyleConfigMap::new(); + let no_expand = StyleConfig::new().expand(ExpandType::None); + let expand_after = StyleConfig::new().expand(ExpandType::After); + styles.insert(MARK_PARAGRAPH_STYLE.into(), no_expand); + styles.insert(MARK_RUN_SEMANTIC_STYLE.into(), expand_after); + styles.insert(MARK_HIGHLIGHT_STYLE.into(), expand_after); + styles.insert(MARK_DIRECT_UNDERLINE.into(), expand_after); + styles.insert(MARK_STRIKETHROUGH.into(), expand_after); + styles.insert(MARK_VERT_ALIGN.into(), expand_after); + doc.config_text_style(styles); +} + +pub fn root_map(doc: &LoroDoc) -> LoroMap { + doc.get_map(ROOT) +} + +pub fn body_text(doc: &LoroDoc) -> LoroText { + let root = root_map(doc); + let flows = root + .ensure_mergeable_map(FLOWS_BY_ID) + .expect("root flows map should be initialized"); + let body = flows + .ensure_mergeable_map(ROOT_BODY_FLOW_ID) + .expect("body flow should be initialized"); + body + .ensure_mergeable_text(FLOW_TEXT_KEY) + .expect("body text should be initialized") +} + +pub fn record_revision( + doc: &LoroDoc, + revision_id: u128, + frontier: Vec, + title: &str, + summary: &str, + author_user_id: Option, +) -> LoroResult<()> { + let root = root_map(doc); + let revisions = root.ensure_mergeable_list(REVISIONS)?; + let revision = revisions.insert_container(revisions.len(), LoroMap::new())?; + revision.insert("id", revision_id.to_string())?; + revision.insert("timestamp", unix_time_secs())?; + revision.insert("replica_id", doc.peer_id().to_string())?; + revision.insert("frontier", frontier)?; + revision.insert("title", title)?; + revision.insert("summary", summary)?; + if let Some(author_user_id) = author_user_id { + revision.insert("author_user_id", author_user_id.to_string())?; + } + // §act-ten A10.2: revision/metadata commits are INERT for undo — they touch + // only the meta/revisions containers, never the body. The "meta" origin is + // excluded from the UndoManager (like "remote"/"repair"), so an autosave + // checkpoint no longer pushes a bogus undo step; the runtime re-arms the + // recorded-inverse stacks across this commit for the same reason. + doc.commit_with(loro::CommitOptions::new().origin("meta")); + Ok(()) +} + +pub fn set_document_id(doc: &LoroDoc, document_id: Uuid) -> LoroResult<()> { + let meta = root_map(doc).ensure_mergeable_map(META)?; + meta.insert("document_id", document_id.to_string())?; + touch_document_metadata(doc)?; + Ok(()) +} + +pub fn document_id(doc: &LoroDoc) -> Option { + let ValueOrContainer::Container(container) = root_map(doc).get(META)? else { + return None; + }; + let meta = container.into_map().ok()?; + let ValueOrContainer::Value(LoroValue::String(value)) = meta.get("document_id")? else { + return None; + }; + Uuid::parse_str(&value).ok() +} + +pub fn document_schema_version(doc: &LoroDoc) -> Option { + let ValueOrContainer::Container(container) = root_map(doc).get(META)? else { + return None; + }; + let meta = container.into_map().ok()?; + let ValueOrContainer::Value(LoroValue::I64(version)) = meta.get("loro_schema_version")? else { + return None; + }; + u32::try_from(version).ok() +} + +pub fn fork_document_lineage(doc: &LoroDoc) -> LoroResult { + let root = root_map(doc); + let meta = root.ensure_mergeable_map(META)?; + if let Some(parent_id) = document_id(doc) { + meta.insert("parent_document_id", parent_id.to_string())?; + } + let document_id = Uuid::new_v4(); + meta.insert("document_id", document_id.to_string())?; + meta.insert("forked_at", unix_time_secs())?; + meta.insert("modified_at", unix_time_secs())?; + meta.insert("last_written_by_app_version", env!("CARGO_PKG_VERSION"))?; + doc.commit(); + Ok(document_id) +} + +pub fn touch_document_metadata(doc: &LoroDoc) -> LoroResult<()> { + let meta = root_map(doc).ensure_mergeable_map(META)?; + meta.insert("modified_at", unix_time_secs())?; + meta.insert("last_written_by_app_version", env!("CARGO_PKG_VERSION"))?; + Ok(()) +} + +/// §act-twelve A12.1.1: the OPEN-time replica invariant without the commit. +/// `register_replica` unconditionally rewrites `last_seen_at`/`app_version` +/// and commits — which advanced the frontier on EVERY open, permanently +/// invalidating the package's frontier-stamped projection/search caches for +/// all later opens. This variant writes (and commits) only when the record is +/// genuinely absent or from another app version; the `last_seen_at` touch +/// rides the first real local edit instead (see the collab layer's deferred +/// author-identity hook). Returns whether it committed. +pub fn register_replica_if_absent(doc: &LoroDoc) -> LoroResult { + let root = root_map(doc); + let replicas = root.ensure_mergeable_map(REPLICAS_BY_ID)?; + let replica_id = doc.peer_id().to_string(); + let existing = replicas.get(&replica_id).and_then(|value| match value { + loro::ValueOrContainer::Container(loro::Container::Map(map)) => Some(map), + _ => None, + }); + if let Some(replica) = existing { + let same_version = matches!( + replica.get("app_version"), + Some(loro::ValueOrContainer::Value(loro::LoroValue::String(version))) if version.as_str() == env!("CARGO_PKG_VERSION") + ); + if same_version { + return Ok(false); + } + } + register_replica(doc, None) +} + +pub fn register_replica(doc: &LoroDoc, user_id: Option) -> LoroResult { + let root = root_map(doc); + let replicas = root.ensure_mergeable_map(REPLICAS_BY_ID)?; + let replica_id = doc.peer_id().to_string(); + let replica = replicas.ensure_mergeable_map(&replica_id)?; + replica.insert("id", replica_id.as_str())?; + replica.insert("app_version", env!("CARGO_PKG_VERSION"))?; + if replica.get("created_at").is_none() { + replica.insert("created_at", unix_time_secs())?; + } + replica.insert("last_seen_at", unix_time_secs())?; + if let Some(user_id) = user_id { + replica.insert("user_id", user_id.to_string())?; + } + doc.commit(); + Ok(true) +} + +/// Register (or refresh) a durable user identity in `users_by_id` (§15/§31). +/// +/// Mirrors [`register_replica`]: it writes a `UserMap` record (`id`, +/// `container_id`, optional `display_name`, `created_at`, `last_seen_at`) and +/// links the *current* editing replica to this user so authorship/blame can be +/// resolved from replica to user. Returns `true` once the record is present. +/// +/// The collaboration layer calls this with a real identity; the schema only +/// guarantees the record shape and the replica→user link. +pub fn register_user(doc: &LoroDoc, user_id: u128, display_name: Option<&str>) -> LoroResult { + let root = root_map(doc); + let users = root.ensure_mergeable_map(USERS_BY_ID)?; + let user_key = user_id.to_string(); + let user = users.ensure_mergeable_map(&user_key)?; + user.insert("id", user_key.as_str())?; + if let Some(display_name) = display_name { + user.insert("display_name", display_name)?; + } + if user.get("created_at").is_none() { + user.insert("created_at", unix_time_secs())?; + } + user.insert("last_seen_at", unix_time_secs())?; + + // Link the current editing replica to this durable user identity (§15). + let replicas = root.ensure_mergeable_map(REPLICAS_BY_ID)?; + let replica_id = doc.peer_id().to_string(); + let replica = replicas.ensure_mergeable_map(&replica_id)?; + replica.insert("id", replica_id.as_str())?; + replica.insert("user_id", user_key.as_str())?; + if replica.get("created_at").is_none() { + replica.insert("created_at", unix_time_secs())?; + } + replica.insert("last_seen_at", unix_time_secs())?; + doc.commit(); + Ok(true) +} + +/// Ensure a `SectionMap` exists in `sections_by_id` with its identity and an +/// `attrs` child map (§11). Returns the section map. +pub fn ensure_section(doc: &LoroDoc, section_id: &str) -> LoroResult { + let root = root_map(doc); + let sections = root.ensure_mergeable_map(SECTIONS_BY_ID)?; + let section = sections.ensure_mergeable_map(section_id)?; + section.insert("id", section_id)?; + let _attrs = section.ensure_mergeable_map("attrs")?; + Ok(section) +} + +/// Create/update a section's §11 page-structure attributes and, when referenced, +/// its independent header/footer text flows. Commits the change. +pub fn set_section_page_attrs(doc: &LoroDoc, section_id: &str, attrs: &SectionPageAttrs) -> LoroResult<()> { + let section = ensure_section(doc, section_id)?; + let attrs_map = section.ensure_mergeable_map("attrs")?; + let flows = root_map(doc).ensure_mergeable_map(FLOWS_BY_ID)?; + write_section_page_attrs(&attrs_map, &flows, attrs)?; + doc.commit(); + Ok(()) +} + +/// Write §11 page-structure attributes onto a section's `attrs` map using the +/// documented twip/string encoding. When `header_flow_id`/`footer_flow_id` are +/// set, the corresponding header/footer flows are created in `flows`. +pub(crate) fn write_section_page_attrs(attrs_map: &LoroMap, flows: &LoroMap, attrs: &SectionPageAttrs) -> LoroResult<()> { + attrs_map.insert(SECTION_ATTR_PAGE_WIDTH, attrs.page_size.width_twips)?; + attrs_map.insert(SECTION_ATTR_PAGE_HEIGHT, attrs.page_size.height_twips)?; + attrs_map.insert(SECTION_ATTR_MARGIN_TOP, attrs.margins.top_twips)?; + attrs_map.insert(SECTION_ATTR_MARGIN_RIGHT, attrs.margins.right_twips)?; + attrs_map.insert(SECTION_ATTR_MARGIN_BOTTOM, attrs.margins.bottom_twips)?; + attrs_map.insert(SECTION_ATTR_MARGIN_LEFT, attrs.margins.left_twips)?; + attrs_map.insert(SECTION_ATTR_COLUMNS, attrs.columns)?; + attrs_map.insert(SECTION_ATTR_ORIENTATION, attrs.orientation.as_str())?; + attrs_map.insert(SECTION_ATTR_PAGE_NUMBERING_FORMAT, attrs.page_numbering.format.as_str())?; + attrs_map.insert(SECTION_ATTR_PAGE_NUMBERING_START, attrs.page_numbering.start)?; + if let Some(header_flow_id) = &attrs.header_flow_id { + attrs_map.insert(SECTION_ATTR_HEADER_FLOW_ID, header_flow_id.as_str())?; + ensure_flow(flows, header_flow_id, "header")?; + } + if let Some(footer_flow_id) = &attrs.footer_flow_id { + attrs_map.insert(SECTION_ATTR_FOOTER_FLOW_ID, footer_flow_id.as_str())?; + ensure_flow(flows, footer_flow_id, "footer")?; + } + Ok(()) +} + +/// Read §11 page-structure attributes back from a section's `attrs` map, +/// substituting the documented defaults (US Letter, 1-inch margins, 1 column, +/// portrait, no numbering, no header/footer) for any missing keys. +#[must_use] +pub fn read_section_page_attrs(attrs_map: &LoroMap) -> SectionPageAttrs { + let defaults = SectionPageAttrs::default(); + SectionPageAttrs { + page_size: SectionPageSize { + width_twips: map_i64_value(attrs_map, SECTION_ATTR_PAGE_WIDTH).unwrap_or(defaults.page_size.width_twips), + height_twips: map_i64_value(attrs_map, SECTION_ATTR_PAGE_HEIGHT).unwrap_or(defaults.page_size.height_twips), + }, + margins: SectionMargins { + top_twips: map_i64_value(attrs_map, SECTION_ATTR_MARGIN_TOP).unwrap_or(defaults.margins.top_twips), + right_twips: map_i64_value(attrs_map, SECTION_ATTR_MARGIN_RIGHT).unwrap_or(defaults.margins.right_twips), + bottom_twips: map_i64_value(attrs_map, SECTION_ATTR_MARGIN_BOTTOM).unwrap_or(defaults.margins.bottom_twips), + left_twips: map_i64_value(attrs_map, SECTION_ATTR_MARGIN_LEFT).unwrap_or(defaults.margins.left_twips), + }, + columns: map_i64_value(attrs_map, SECTION_ATTR_COLUMNS).unwrap_or(defaults.columns), + orientation: SectionOrientation::from_attr(map_string_value(attrs_map, SECTION_ATTR_ORIENTATION).as_deref()), + page_numbering: SectionPageNumbering { + format: PageNumberFormat::from_attr(map_string_value(attrs_map, SECTION_ATTR_PAGE_NUMBERING_FORMAT).as_deref()), + start: map_i64_value(attrs_map, SECTION_ATTR_PAGE_NUMBERING_START).unwrap_or(defaults.page_numbering.start), + }, + header_flow_id: map_string_value(attrs_map, SECTION_ATTR_HEADER_FLOW_ID), + footer_flow_id: map_string_value(attrs_map, SECTION_ATTR_FOOTER_FLOW_ID), + } +} + +fn map_i64_value(map: &LoroMap, key: &str) -> Option { + match map.get(key)? { + ValueOrContainer::Value(LoroValue::I64(value)) => Some(value), + _ => None, + } +} + +fn map_string_value(map: &LoroMap, key: &str) -> Option { + match map.get(key)? { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + } +} + +fn init_meta(meta: &LoroMap, title: &str) -> LoroResult<()> { + let now = unix_time_secs(); + if meta.get("document_id").is_none() { + meta.insert("document_id", Uuid::new_v4().to_string())?; + } + meta.insert("loro_schema_version", i64::from(LORO_SCHEMA_VERSION))?; + meta.insert("schema_features", "flow-v1")?; + meta.insert("title", title)?; + meta.insert("created_by_app_version", env!("CARGO_PKG_VERSION"))?; + meta.insert("last_written_by_app_version", env!("CARGO_PKG_VERSION"))?; + if meta.get("created_at").is_none() { + meta.insert("created_at", now)?; + } + meta.insert("modified_at", now)?; + Ok(()) +} + +fn ensure_flow(flows: &LoroMap, flow_id: &str, kind: &str) -> LoroResult { + let flow = flows.ensure_mergeable_map(flow_id)?; + flow.insert(FLOW_ID_KEY, flow_id)?; + flow.insert(FLOW_KIND_KEY, kind)?; + let text = flow.ensure_mergeable_text(FLOW_TEXT_KEY)?; + let _attrs = flow.ensure_mergeable_map(FLOW_ATTRS_KEY)?; + flow.insert("text_container_id", text.id().to_string())?; + Ok(flow) +} + +fn ensure_sentinel(text: &LoroText) -> LoroResult<()> { + if text.len_unicode() == 0 || !text.to_string().starts_with(SENTINEL_NEWLINE) { + text.insert(0, SENTINEL_NEWLINE)?; + text.mark(0..1, MARK_PARAGRAPH_STYLE, 0_i64)?; + } + Ok(()) +} + +fn ensure_initial_paragraph(paragraphs: &LoroMap, blocks: &LoroMap, body: &LoroText) -> LoroResult<()> { + let paragraph = paragraphs.ensure_mergeable_map(ROOT_FIRST_PARAGRAPH_ID)?; + paragraph.insert("id", ROOT_FIRST_PARAGRAPH_ID)?; + paragraph.insert("flow_id", ROOT_BODY_FLOW_ID)?; + if let Some(cursor) = body.get_cursor(0, Side::Left) { + paragraph.insert("start_cursor", cursor.encode())?; + } + if let Some(cursor) = body.get_cursor(0, Side::Left) { + paragraph.insert("boundary_cursor", cursor.encode())?; + } + let _paragraph_attrs = paragraph.ensure_mergeable_map("attrs")?; + + let block = blocks.ensure_mergeable_map(MAIN_BODY_BLOCK_ID)?; + block.insert("id", MAIN_BODY_BLOCK_ID)?; + block.insert("kind", "paragraph")?; + block.insert("flow_id", ROOT_BODY_FLOW_ID)?; + if let Some(cursor) = body.get_cursor(0, Side::Left) { + block.insert("anchor_cursor", cursor.encode())?; + } + let _block_attrs = block.ensure_mergeable_map("attrs")?; + let _nested_refs = block.ensure_mergeable_map("nested_refs")?; + Ok(()) +} + +fn unix_time_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn section_page_attrs_round_trip_through_loro() -> LoroResult<()> { + let doc = new_loro_document("Sections")?; + let attrs = SectionPageAttrs { + page_size: SectionPageSize { + width_twips: 15_840, + height_twips: 12_240, + }, + margins: SectionMargins { + top_twips: 720, + right_twips: 720, + bottom_twips: 1_440, + left_twips: 1_440, + }, + columns: 2, + orientation: SectionOrientation::Landscape, + page_numbering: SectionPageNumbering { + format: PageNumberFormat::LowerRoman, + start: 3, + }, + header_flow_id: Some("section.s1.header".to_string()), + footer_flow_id: Some("section.s1.footer".to_string()), + }; + set_section_page_attrs(&doc, "section.s1", &attrs)?; + + let root = root_map(&doc); + let sections = root.ensure_mergeable_map(SECTIONS_BY_ID)?; + let section = sections.ensure_mergeable_map("section.s1")?; + let attrs_map = section.ensure_mergeable_map("attrs")?; + assert_eq!(read_section_page_attrs(&attrs_map), attrs); + + // §11: header/footer are independent flows of their respective kinds. + let flows = root.ensure_mergeable_map(FLOWS_BY_ID)?; + let header = flows.ensure_mergeable_map("section.s1.header")?; + assert_eq!(map_string_value(&header, FLOW_KIND_KEY).as_deref(), Some("header")); + let footer = flows.ensure_mergeable_map("section.s1.footer")?; + assert_eq!(map_string_value(&footer, FLOW_KIND_KEY).as_deref(), Some("footer")); + Ok(()) + } + + #[test] + fn section_page_attrs_default_to_letter_when_unset() -> LoroResult<()> { + let doc = new_loro_document("Sections")?; + let section = ensure_section(&doc, "section.bare")?; + let attrs_map = section.ensure_mergeable_map("attrs")?; + assert_eq!(read_section_page_attrs(&attrs_map), SectionPageAttrs::default()); + Ok(()) + } + + #[test] + fn register_user_writes_record_and_links_replica() -> LoroResult<()> { + let doc = new_loro_document("Users")?; + register_user(&doc, 0x42, Some("Ada"))?; + + let root = root_map(&doc); + let users = root.ensure_mergeable_map(USERS_BY_ID)?; + let user = users.ensure_mergeable_map("66")?; + assert_eq!(map_string_value(&user, "display_name").as_deref(), Some("Ada")); + assert_eq!(map_string_value(&user, "id").as_deref(), Some("66")); + + let replicas = root.ensure_mergeable_map(REPLICAS_BY_ID)?; + let replica = replicas.ensure_mergeable_map(&doc.peer_id().to_string())?; + assert_eq!(map_string_value(&replica, "user_id").as_deref(), Some("66")); + Ok(()) + } +} diff --git a/crates/flowstate-document/src/package.rs b/crates/flowstate-document/src/package.rs new file mode 100644 index 00000000..fe8c7fc5 --- /dev/null +++ b/crates/flowstate-document/src/package.rs @@ -0,0 +1,4648 @@ +use std::{ + collections::HashSet, + fs::{self, OpenOptions}, + io::{self, Cursor, Read as _, Seek as _, SeekFrom, Write as _}, + path::Path, +}; + +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use loro::{Container, ExportMode, Frontiers, LoroDoc, LoroValue, ValueOrContainer, VersionVector}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +pub const LORO_PACKAGE_FORMAT_VERSION: u32 = 1; +/// §A12.1.3: format version stamped on packages that CARRY a shallow +/// accelerator chunk. Old builds reject unknown chunk kinds with a +/// corruption-shaped error; the version bump turns that into the clean +/// "unsupported format version" one. Packages without a shallow chunk keep +/// v1 (fully old-build readable). +pub const LORO_PACKAGE_FORMAT_VERSION_SHALLOW: u32 = 2; + +/// §A13.1.0: env-gated read-stage probe (`FLOWSTATE_OPEN_PROBE`). +fn open_probe() -> bool { + static PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + *PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_OPEN_PROBE").is_some()) +} + +fn package_format_version_supported(version: u32) -> bool { + version == LORO_PACKAGE_FORMAT_VERSION || version == LORO_PACKAGE_FORMAT_VERSION_SHALLOW +} +pub const LORO_SCHEMA_VERSION: u32 = 1; +pub const DEFAULT_UPDATE_SEGMENT_COMPACTION_THRESHOLD: usize = 256; + +const PACKAGE_MAGIC: &[u8; 16] = b"FLOWDB8-LORO\0\0\0\0"; +const PACKAGE_HEADER_VERSION: u32 = 1; +const JOURNAL_MAGIC: &[u8; 16] = b"FLOWDB8-JOURNAL\0"; +const JOURNAL_HEADER_VERSION: u32 = 1; +const JOURNAL_TXN_MAGIC: &[u8; 8] = b"DB8TXN01"; +const JOURNAL_COMMIT_MAGIC: &[u8; 8] = b"DB8DONE1"; +const JOURNAL_DELTA_MAGIC: &[u8; 8] = b"DB8DELTA"; +const JOURNAL_GENERATION_COMPACTION_THRESHOLD: usize = 16; + +const CHUNK_MANIFEST: u32 = 1; +const CHUNK_LORO_SNAPSHOT: u32 = 2; +const CHUNK_LORO_UPDATE_SEGMENT: u32 = 3; +const CHUNK_ASSET: u32 = 4; +const CHUNK_REVISION_INDEX: u32 = 5; +const CHUNK_PROJECTION_CACHE: u32 = 6; +const CHUNK_SEARCH_UNIT: u32 = 7; +const CHUNK_THUMBNAIL: u32 = 8; +const CHUNK_INTEGRITY_INDEX: u32 = 9; +const CHUNK_PREVIEW_HEADER: u32 = 10; +/// §A12.1.3 slice 1: shallow (history-truncated) Loro snapshot — an OPEN +/// ACCELERATOR chunk, never the only copy of any history (invariant S1 in +/// `flowstate_shallow_open_design.md`). Old builds reject packages carrying +/// it ("unknown Flowstate package chunk kind"), so it is only written behind +/// `FLOWSTATE_WRITE_SHALLOW_SNAPSHOT` until the format-version bump. +const CHUNK_SHALLOW_SNAPSHOT: u32 = 11; + +const PACKAGE_CHUNK_TABLE_ENTRY_BYTES: usize = 4 + 8 + 8 + 32; +const MAX_PACKAGE_CHUNK_COUNT: usize = 1_048_576; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DocumentPackage { + pub manifest: DocumentPackageManifest, + pub loro_snapshots: Vec, + pub loro_update_segments: Vec, + pub assets: Vec, + pub revisions: Vec, + pub projection_caches: Vec, + pub search_units: Vec, + pub thumbnails: Vec, + /// §19 named integrity index: one entry per durable chunk (snapshots, update + /// segments, assets). Older packages without the index decode to an empty + /// vector and still load; new packages always rebuild a complete index. + #[serde(default)] + pub integrity_index: Vec, + /// §act-four M2: the always-written PREVIEW HEADER — a tiny projection of the + /// document's first `PREVIEW_HEADER_BLOCKS` blocks, written on every + /// checkpoint even when the full projection cache is stale or absent. Preview + /// reads ONLY this chunk (`read_preview_header`), never the snapshot, making + /// cold preview `O(viewport)` and independent of full-cache freshness. Older + /// packages without it decode to an empty vector. + #[serde(default)] + pub preview_headers: Vec, + /// §A12.1.3: shallow snapshot accelerator chunks (at most one is kept). + /// Postcard is positional — this field must stay in APPEND order. + #[serde(default)] + pub shallow_snapshots: Vec, + /// §A12.1.3 slice 2b: full-snapshot chunks whose BLAKE3 verification was + /// DEFERRED at read time (open-path cost). Keyed by `snapshot_id` → the + /// chunk-table checksum; consumers that decode those bytes verify first + /// via [`Self::verify_snapshot_chunk`]. Never serialized. + #[serde(skip)] + pub(crate) deferred_snapshot_checksums: std::collections::HashMap, +} + +/// §act-four M2: the maximum number of leading blocks stored in the preview +/// header — enough to fill a preview screen; independent of document size. +pub const PREVIEW_HEADER_BLOCKS: usize = 16; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DocumentPackageManifest { + pub package_format_version: u32, + pub loro_schema_version: u32, + pub document_id: u128, + pub latest_frontier: Vec, + pub latest_version_vector: Vec, + pub latest_snapshot_id: u128, + pub update_segment_index: Vec, + pub asset_index: Vec, + pub projection_cache_frontier: Option>, + pub search_cache_frontier: Option>, + /// §act-four M2: frontier the preview header was written at. Older packages + /// without it decode to `None` (preview falls back to the projection cache / + /// full read). + #[serde(default)] + pub preview_header_frontier: Option>, + /// §act-ten A10.5: RETIRED — always written empty. The M4 cold-scroll consumer + /// was never wired, and persisting it re-ran both whole-registry scans per + /// checkpoint. The field survives because postcard is positional: removing it + /// would break decoding of every existing package. Old packages carrying + /// boundaries still decode; the data is ignored. + #[serde(default)] + pub block_boundaries: Vec, + pub created_at_unix_secs: i64, + pub modified_at_unix_secs: i64, + /// §27 schema migration log. Empty at schema v1 (no migrations exist yet). + /// Future schema bumps append a [`SchemaMigrationRecord`] here, and any Loro + /// document mutation a migration performs is an ordinary Loro change committed + /// with the `"migration"` origin. Older packages without this field decode to + /// an empty vector. + #[serde(default)] + pub schema_migrations: Vec, + /// §act-twelve A12.1.1: frontier at which the body/flow paragraph-style + /// marks were last known CLEAN (a checkpoint from a live runtime always + /// qualifies — the runtime maintains the marks). Open compares this to the + /// doc frontier and SKIPS the whole-corpus style-mark verification scan on + /// a match (measured 1.1-1.2s per open on the 2.6M-char impact doc — the + /// single largest open cost). `None` (older packages) scans once; the next + /// checkpoint stamps. Postcard is positional: this field must stay in + /// APPEND order. + #[serde(default)] + pub style_marks_clean_frontier: Option>, + /// §A12.1.3: frontier of the newest shallow snapshot chunk, `None` when the + /// package carries none. The shallow chunk is an accelerator only — every + /// consumer must fall back to the full snapshot + segments when it is + /// absent or stale. Postcard is positional: APPEND order only. + #[serde(default)] + pub shallow_snapshot_frontier: Option>, +} + +/// §19 integrity-index entry recording one durable chunk's identity, kind, +/// BLAKE3 checksum, and byte length so package integrity can be cross-checked +/// against the actual chunk payloads on read. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct IntegrityIndexEntry { + pub chunk_kind: u32, + pub id: u128, + pub checksum: [u8; 32], + pub byte_length: u64, +} + +/// §27 record of a schema migration that was applied to this package. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SchemaMigrationRecord { + pub id: u128, + pub from_version: u32, + pub to_version: u32, + pub applied_at_unix_secs: i64, + pub description: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ChunkRef { + pub id: u128, + pub checksum: [u8; 32], + pub byte_length: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LoroSnapshotChunk { + pub snapshot_id: u128, + pub frontier: Vec, + pub version_vector: Vec, + #[serde(with = "serde_bytes")] + pub bytes: Vec, + pub created_at_unix_secs: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LoroShallowSnapshotChunk { + pub snapshot_id: u128, + /// The shallow ROOT: history strictly before this frontier is absent from + /// `bytes`. A doc decoded from this chunk cannot check out, serve, or + /// import ops below it (`flowstate_shallow_open_design.md` §1). + pub root_frontier: Vec, + pub root_version_vector: Vec, + /// Frontier the snapshot was taken AT (its state tip). + pub frontier: Vec, + pub version_vector: Vec, + #[serde(with = "serde_bytes")] + pub bytes: Vec, + pub created_at_unix_secs: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LoroUpdateSegmentChunk { + pub segment_id: u128, + pub from_frontier: Vec, + pub from_version_vector: Vec, + pub to_frontier: Vec, + pub to_version_vector: Vec, + #[serde(with = "serde_bytes")] + pub bytes: Vec, + pub checksum: [u8; 32], + pub created_at_unix_secs: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AssetChunk { + pub asset_id: u128, + pub content_hash: [u8; 32], + pub mime_type: String, + pub byte_length: u64, + #[serde(with = "serde_bytes")] + pub bytes: Vec, + #[serde(with = "serde_bytes")] + pub metadata: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PackageRevision { + pub revision_id: u128, + pub frontier: Vec, + pub version_vector: Vec, + pub title: String, + pub summary: String, + pub author_user_id: Option, + pub replica_id: Option, + pub created_at_unix_secs: i64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ProjectionCacheChunk { + pub frontier: Vec, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +/// §act-four M2: a tiny first-`PREVIEW_HEADER_BLOCKS`-blocks projection for +/// `O(viewport)` cold preview. `bytes` is an encoded [`ProjectionBlocks`] +/// trimmed to the leading blocks. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PreviewHeaderChunk { + pub frontier: Vec, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SearchUnitChunk { + pub frontier: Vec, + pub unit_id: u128, + pub unit_kind: String, + #[serde(default)] + pub flow_id: Option, + #[serde(default)] + pub block_id: Option, + #[serde(default)] + pub table_id: Option, + #[serde(default)] + pub cell_id: Option, + pub heading_path: Vec, + pub heading: String, + pub body: String, + pub insert_text: String, + #[serde(default)] + pub unit_start_cursor: Vec, + #[serde(default)] + pub unit_end_cursor: Vec, + pub paragraph_start_cursor: Vec, + pub paragraph_end_cursor: Vec, +} + +#[derive(Clone, Debug)] +struct Chunk { + kind: u32, + payload: Vec, +} + +#[derive(Clone, Debug)] +struct ChunkEntry { + kind: u32, + offset: u64, + len: u64, + checksum: [u8; 32], +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ThumbnailChunk { + pub thumbnail_id: u128, + pub revision_id: Option, + pub frontier: Vec, + pub mime_type: String, + #[serde(with = "serde_bytes")] + pub bytes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +enum PackageJournalDelta { + Update { + manifest: DocumentPackageManifest, + segment: LoroUpdateSegmentChunk, + /// §act-four M2: the fresh preview header for this edit, so cold preview of + /// an edited-but-unflushed document stays `O(viewport)` (the header rides + /// the journal delta rather than waiting for the next full checkpoint). + /// `None` on older journals / when no header was maintained. + #[serde(default)] + preview_header: Option, + }, + Assets { + manifest: DocumentPackageManifest, + assets: Vec, + }, +} + +impl DocumentPackage { + pub fn from_loro_snapshot(doc: &LoroDoc, title: &str) -> io::Result { + Self::from_loro_snapshot_with_assets(doc, title, Vec::new()) + } + + pub fn from_loro_snapshot_with_assets(doc: &LoroDoc, title: &str, assets: Vec) -> io::Result { + doc.commit(); + let now = unix_time_secs(); + let document_id = crate::loro_schema::document_id(doc) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Loro document has no valid canonical document ID"))? + .as_u128(); + let revision_id = Uuid::new_v4().as_u128(); + let revision_frontiers = doc.state_frontiers(); + let revision_version_vector = encode_version_vector(&frontiers_version_vector(doc, &revision_frontiers)?); + crate::loro_schema::record_revision(doc, revision_id, encode_frontiers(&revision_frontiers), title, "Initial snapshot", None) + .map_err(loro_io_error)?; + let snapshot_id = Uuid::new_v4().as_u128(); + let frontier = encode_frontiers(&doc.state_frontiers()); + let version_vector = encode_version_vector(&doc.state_vv()); + let snapshot = doc.export(ExportMode::Snapshot).map_err(loro_io_error)?; + let mut package = Self { + manifest: DocumentPackageManifest { + package_format_version: LORO_PACKAGE_FORMAT_VERSION, + loro_schema_version: LORO_SCHEMA_VERSION, + document_id, + latest_frontier: frontier.clone(), + latest_version_vector: version_vector.clone(), + latest_snapshot_id: snapshot_id, + update_segment_index: Vec::new(), + asset_index: Vec::new(), + projection_cache_frontier: None, + search_cache_frontier: None, + preview_header_frontier: None, + block_boundaries: Vec::new(), + created_at_unix_secs: now, + modified_at_unix_secs: now, + schema_migrations: Vec::new(), + // NOT stamped here: this constructor packages ARBITRARY docs (tests, + // converters, raw Loro states) whose marks were never maintained. + // The RUNTIME stamps after constructing from its own maintained doc + // (checkpoint path) and on every runtime persist (segment append). + style_marks_clean_frontier: None, + shallow_snapshot_frontier: None, + }, + loro_snapshots: vec![ + LoroSnapshotChunk { + snapshot_id, + frontier: frontier.clone(), + version_vector: version_vector.clone(), + bytes: snapshot, + created_at_unix_secs: now, + }, + LoroSnapshotChunk { + snapshot_id: Uuid::new_v4().as_u128(), + frontier: encode_frontiers(&revision_frontiers), + version_vector: revision_version_vector.clone(), + // Full-history snapshot exported at the revision frontier directly + // from the live doc — no O(doc) `fork_at`. We deliberately keep full + // `SnapshotAt` bytes in the package; switching revision chunks to the + // lighter `StateOnly` variant is an explicitly deferred decision. + bytes: doc + .export(ExportMode::snapshot_at(&revision_frontiers)) + .map_err(loro_io_error)?, + created_at_unix_secs: now, + }, + ], + loro_update_segments: Vec::new(), + assets, + revisions: vec![PackageRevision { + revision_id, + frontier: encode_frontiers(&revision_frontiers), + version_vector: revision_version_vector, + title: title.to_string(), + summary: "Initial snapshot".to_string(), + author_user_id: None, + replica_id: None, + created_at_unix_secs: now, + }], + projection_caches: Vec::new(), + search_units: Vec::new(), + thumbnails: Vec::new(), + integrity_index: Vec::new(), + preview_headers: Vec::new(), + shallow_snapshots: Vec::new(), + deferred_snapshot_checksums: std::collections::HashMap::new(), + } + .with_manifest_indexes()?; + package.rebuild_projection_cache_from_loro(doc)?; + package.rebuild_search_units_from_loro(doc)?; + fidelity::event(FidelityClass::Persistence, "save-snapshot", || { + format!( + "snapshots={} segments={} revisions={} assets={} frontier={}", + package.loro_snapshots.len(), + package.loro_update_segments.len(), + package.revisions.len(), + package.assets.len(), + frontier_tag(&package.manifest.latest_frontier) + ) + }); + package.fidelity_check_roundtrip(doc); + Ok(package) + } + + pub fn load_loro_doc(&self) -> io::Result { + self.validate()?; + self.load_loro_doc_from_validated() + } + + /// [`Self::load_loro_doc`] minus the leading `validate()` — for callers + /// holding a package that `read`/`from_bytes` JUST validated. The doubled + /// validate re-hashed every update segment and asset (a second full + /// integrity sweep over a multi-hundred-MB package) on every open. + pub fn load_loro_doc_from_validated(&self) -> io::Result { + let doc = self.load_loro_doc_unvalidated()?; + fidelity::event(FidelityClass::Persistence, "load", || { + format!( + "snapshots={} segments={} revisions={} frontier={}", + self.loro_snapshots.len(), + self.loro_update_segments.len(), + self.revisions.len(), + frontier_tag(&self.manifest.latest_frontier) + ) + }); + Ok(doc) + } + + /// §fidelity persistence round-trip invariant. Read-only and gated on + /// [`fidelity::expensive_checks_enabled`]: it reloads this package's canonical Loro state into a + /// fresh document and asserts the projection built from the reloaded state + /// matches the projection of the `source` document that was persisted + /// (paragraph count + texts, paragraph/block ids, and section count). Never + /// mutates `self` or `source`; failures fire the `package-roundtrip-mismatch` + /// violation. + fn fidelity_check_roundtrip(&self, source: &LoroDoc) { + if !fidelity::expensive_checks_enabled() { + return; + } + let expected = match crate::loro_projection::document_from_loro(source) { + Ok(document) => document, + Err(error) => { + fidelity::violation(FidelityClass::Persistence, "package-roundtrip-mismatch", || { + format!("could not project the source document for the round-trip check: {error}") + }); + return; + }, + }; + let loaded = match self + .load_loro_doc_unvalidated() + .and_then(|doc| crate::loro_projection::document_from_loro(&doc)) + { + Ok(document) => document, + Err(error) => { + fidelity::violation(FidelityClass::Persistence, "package-roundtrip-mismatch", || { + format!("could not reload and project the persisted package: {error}") + }); + return; + }, + }; + let same_shape = expected.paragraphs.len() == loaded.paragraphs.len() + && expected.ids.paragraph_ids == loaded.ids.paragraph_ids + && expected.ids.block_ids == loaded.ids.block_ids + && expected.sections.len() == loaded.sections.len(); + let same_text = + same_shape && (0..expected.paragraphs.len()).all(|ix| crate::paragraph_text(&expected, ix) == crate::paragraph_text(&loaded, ix)); + fidelity::check(same_shape && same_text, FidelityClass::Persistence, "package-roundtrip-mismatch", || { + format!( + "reloaded projection differs from persisted source: paragraphs {}->{}, sections {}->{}, ids_match={}, text_match={}", + expected.paragraphs.len(), + loaded.paragraphs.len(), + expected.sections.len(), + loaded.sections.len(), + expected.ids.paragraph_ids == loaded.ids.paragraph_ids && expected.ids.block_ids == loaded.ids.block_ids, + same_text + ) + }); + } + + /// §A12.1.3 slice 2b: verify a snapshot chunk whose hash was deferred at + /// read time. No-op for chunks verified eagerly (or built in memory). The + /// deferred value is the chunk-TABLE checksum — a hash of the postcard + /// envelope — so verification re-encodes the chunk (a memcpy-scale pass; + /// postcard is deterministic) and hashes that. + fn verify_snapshot_chunk(&self, snapshot: &LoroSnapshotChunk) -> io::Result<()> { + if let Some(expected) = self.deferred_snapshot_checksums.get(&snapshot.snapshot_id) { + let payload = encode_chunk(snapshot, "Loro snapshot")?; + if blake3_hash(&payload) != *expected { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Flowstate package chunk checksum mismatch")); + } + } + Ok(()) + } + + /// §A12.1.3 slice 2b: integrity-index checksum to VALIDATE against for a + /// snapshot chunk. For read-deferred chunks the stored entry is echoed + /// (its real verification happens at decode via + /// [`Self::verify_snapshot_chunk`]); everything else hashes for real. + fn integrity_checksum_for_snapshot(&self, snapshot: &LoroSnapshotChunk) -> [u8; 32] { + if self + .deferred_snapshot_checksums + .contains_key(&snapshot.snapshot_id) + && let Some(entry) = self + .integrity_index + .iter() + .find(|entry| entry.chunk_kind == CHUNK_LORO_SNAPSHOT && entry.id == snapshot.snapshot_id) + { + return entry.checksum; + } + blake3_hash(&snapshot.bytes) + } + + fn load_loro_doc_unvalidated(&self) -> io::Result { + let snapshot = self + .latest_snapshot() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package has no latest Loro snapshot"))?; + self.verify_snapshot_chunk(snapshot)?; + let doc = LoroDoc::new(); + crate::loro_schema::configure_text_styles(&doc); + doc.import(&snapshot.bytes).map_err(loro_io_error)?; + for segment in &self.loro_update_segments { + doc.import(&segment.bytes).map_err(loro_io_error)?; + } + let document_id = crate::loro_schema::document_id(&doc) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "package Loro state has no valid document ID"))?; + if document_id.as_u128() != self.manifest.document_id { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "package manifest document ID does not match canonical Loro lineage", + )); + } + if crate::loro_schema::document_schema_version(&doc) != Some(self.manifest.loro_schema_version) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "package manifest schema version does not match canonical Loro metadata", + )); + } + Ok(doc) + } + /// §A12.1.3 slice 2: load the live doc from the SHALLOW accelerator chunk + /// plus the update segments, skipping the full-history decode. Returns + /// `Ok(None)` when the package has no usable shallow chunk; the caller + /// falls back to [`Self::load_loro_doc_from_validated`]. Any import/lineage + /// anomaly is an `Err` the caller likewise treats as fall-back-to-full + /// (design: `flowstate_shallow_open_design.md` §4). + /// + /// Segment overlap is safe by construction: a shallow doc's oplog VV covers + /// genesis→tip (only its HISTORY is truncated), so Loro's update decode + /// drops already-known ops; segments at-or-below the shallow frontier + /// decode to nothing. A segment op with deps below the shallow ROOT (a + /// pre-shallow far-behind remote import persisted before this open) errors + /// cleanly (`ImportUpdatesThatDependsOnOutdatedVersion`) → full fallback. + pub fn load_loro_doc_shallow(&self) -> io::Result> { + let Some(stamp) = self.manifest.shallow_snapshot_frontier.as_deref() else { + return Ok(None); + }; + let Some(shallow) = self + .shallow_snapshots + .iter() + .rev() + .find(|shallow| shallow.frontier == stamp) + else { + return Ok(None); + }; + let doc = LoroDoc::new(); + crate::loro_schema::configure_text_styles(&doc); + doc.import(&shallow.bytes).map_err(loro_io_error)?; + for segment in &self.loro_update_segments { + doc.import(&segment.bytes).map_err(loro_io_error)?; + } + // The reconstructed tip must be exactly the manifest tip — anything else + // (a stale chunk, a segment the shallow state couldn't accept) demotes to + // the full path rather than opening a wrong version. + if encode_frontiers(&doc.oplog_frontiers()) != self.manifest.latest_frontier { + return Ok(None); + } + let document_id = crate::loro_schema::document_id(&doc) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "package Loro state has no valid document ID"))?; + if document_id.as_u128() != self.manifest.document_id { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "package manifest document ID does not match canonical Loro lineage", + )); + } + if crate::loro_schema::document_schema_version(&doc) != Some(self.manifest.loro_schema_version) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "package manifest schema version does not match canonical Loro metadata", + )); + } + Ok(Some(doc)) + } + + pub fn replace_assets_from_document(&mut self, document: &crate::DocumentProjection) -> io::Result<()> { + let mut next = self.clone(); + next.assets = crate::loro_import::assets_from_document(document); + next.manifest.modified_at_unix_secs = unix_time_secs(); + next.refresh_manifest_indexes(); + next.validate()?; + *self = next; + Ok(()) + } + + pub fn current_search_units(&self) -> &[SearchUnitChunk] { + if self.manifest.search_cache_frontier.as_deref() == Some(self.manifest.latest_frontier.as_slice()) { + &self.search_units + } else { + &[] + } + } + + pub fn read_cached_search_units(path: impl AsRef) -> io::Result>> { + let bytes = fs::read(path)?; + Self::cached_search_units_from_bytes(&bytes) + } + + pub fn cached_search_units_from_bytes(bytes: &[u8]) -> io::Result>> { + let (manifest, units) = if bytes.starts_with(JOURNAL_MAGIC) { + cached_search_units_from_journal_bytes(bytes)? + } else { + cached_search_units_from_compact_bytes(bytes)? + }; + if !package_format_version_supported(manifest.package_format_version) || manifest.loro_schema_version != LORO_SCHEMA_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported Flowstate cached search package version", + )); + } + let Some(search_frontier) = manifest.search_cache_frontier.as_deref() else { + return Ok(None); + }; + validate_frontiers(search_frontier, "manifest search cache frontier")?; + validate_frontiers(&manifest.latest_frontier, "manifest latest frontier")?; + if search_frontier != manifest.latest_frontier.as_slice() { + return Ok(None); + } + if units.iter().any(|unit| unit.frontier != search_frontier) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "search unit frontier does not match package cache frontier", + )); + } + Ok(Some(units)) + } + + pub fn append_update_segment( + &mut self, + from_frontier: &Frontiers, + from_version_vector: &VersionVector, + to_frontier: &Frontiers, + to_version_vector: &VersionVector, + bytes: Vec, + ) -> io::Result { + if bytes.is_empty() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "cannot append an empty Loro update segment")); + } + let segment_id = Uuid::new_v4().as_u128(); + let now = unix_time_secs(); + let checksum = blake3_hash(&bytes); + // Append IN PLACE with a rollback guard instead of `self.clone()`: the + // clone copied every snapshot/asset chunk — ~86MB on a large document — + // on EVERY keystroke inside the write gate (the .db8 first-edit freeze). + // Only the manifest and the cleared caches need restoring on a validation + // failure; the pushed segment pops back off. + let manifest_backup = self.manifest.clone(); + let projection_caches_backup = std::mem::take(&mut self.projection_caches); + let search_units_backup = std::mem::take(&mut self.search_units); + self.loro_update_segments.push(LoroUpdateSegmentChunk { + segment_id, + from_frontier: encode_frontiers(from_frontier), + from_version_vector: encode_version_vector(from_version_vector), + to_frontier: encode_frontiers(to_frontier), + to_version_vector: encode_version_vector(to_version_vector), + bytes, + checksum, + created_at_unix_secs: now, + }); + self.manifest.latest_frontier = encode_frontiers(to_frontier); + self.manifest.latest_version_vector = encode_version_vector(to_version_vector); + self.manifest.projection_cache_frontier = None; + self.manifest.search_cache_frontier = None; + // §act-twelve A12.1.1: only live runtimes write packages, and the runtime + // maintains paragraph-style marks through every local/import/repair flow — + // the appended segment's frontier is therefore marks-clean, and open can + // skip the whole-corpus verification scan. + self.manifest.style_marks_clean_frontier = Some(encode_frontiers(to_frontier)); + self.manifest.modified_at_unix_secs = now; + self.refresh_manifest_indexes(); + // Validate ONLY the delta: full `validate()` re-hashes every stored + // segment and decodes every cached projection — O(package) per keystroke + // (the other half of the .db8 first-edit freeze). Full validation still + // runs at read/write time. + if let Err(error) = self.validate_appended_segment() { + self.loro_update_segments.pop(); + self.manifest = manifest_backup; + self.projection_caches = projection_caches_backup; + self.search_units = search_units_backup; + self.refresh_manifest_indexes(); + return Err(error); + } + fidelity::event(FidelityClass::Persistence, "append-segment", || { + format!( + "segment={segment_id:032x} segments={} frontier={}", + self.loro_update_segments.len(), + frontier_tag(&self.manifest.latest_frontier) + ) + }); + Ok(segment_id) + } + + pub fn create_named_revision( + &mut self, + doc: &LoroDoc, + title: impl Into, + summary: impl Into, + author_user_id: Option, + replica_id: Option, + ) -> io::Result { + self.create_named_revision_with_id(doc, Uuid::new_v4().as_u128(), title, summary, author_user_id, replica_id) + } + + pub fn create_named_revision_with_id( + &mut self, + doc: &LoroDoc, + revision_id: u128, + title: impl Into, + summary: impl Into, + author_user_id: Option, + replica_id: Option, + ) -> io::Result { + self.create_named_revision_at_with_id(doc, revision_id, &doc.state_frontiers(), title, summary, author_user_id, replica_id) + } + + pub fn create_named_revision_at_with_id( + &mut self, + doc: &LoroDoc, + revision_id: u128, + frontiers: &Frontiers, + title: impl Into, + summary: impl Into, + author_user_id: Option, + replica_id: Option, + ) -> io::Result { + doc.commit(); + let title = title.into(); + let summary = summary.into(); + if self + .revisions + .iter() + .any(|revision| revision.revision_id == revision_id) + { + return Ok(revision_id); + } + let doc_frontier_before_revision_record = doc.state_frontiers(); + let doc_vv_before_revision_record = doc.state_vv(); + let frontier = encode_frontiers(frontiers); + let version_vector = encode_version_vector(&frontiers_version_vector(doc, frontiers)?); + if !loro_revision_exists(doc, revision_id) { + crate::loro_schema::record_revision(doc, revision_id, frontier.clone(), &title, &summary, author_user_id).map_err(loro_io_error)?; + let update = doc + .export(ExportMode::updates(&doc_vv_before_revision_record)) + .map_err(loro_io_error)?; + if !update.is_empty() { + self.append_update_segment( + &doc_frontier_before_revision_record, + &doc_vv_before_revision_record, + &doc.state_frontiers(), + &doc.state_vv(), + update, + )?; + } + } + let revision = PackageRevision { + revision_id, + frontier: frontier.clone(), + version_vector, + title, + summary, + author_user_id, + replica_id, + created_at_unix_secs: unix_time_secs(), + }; + let revision_id = revision.revision_id; + // §A12.1.3: `SnapshotAt` export is upstream-unimplemented on shallow + // docs. The stored chunk is only a revision-LOAD accelerator — without + // it, `load_revision_loro_doc` falls back to forking the package's full + // doc at the revision frontier (S1 keeps that history), so a shallow + // session simply skips the accelerator. + if self.snapshot_for_frontier(&frontier).is_none() && !doc.is_shallow() { + self.loro_snapshots.push(LoroSnapshotChunk { + snapshot_id: Uuid::new_v4().as_u128(), + frontier: frontier.clone(), + version_vector: revision.version_vector.clone(), + // Full `SnapshotAt` bytes from the live doc (no fork); the lighter + // `StateOnly` variant is an explicitly deferred format decision. + bytes: doc + .export(ExportMode::snapshot_at(frontiers)) + .map_err(loro_io_error)?, + created_at_unix_secs: unix_time_secs(), + }); + } + self.revisions.push(revision); + self.manifest.modified_at_unix_secs = unix_time_secs(); + // §19: a revision may add a snapshot chunk; keep the integrity index complete. + self.integrity_index = self.build_integrity_index(); + self.validate()?; + fidelity::event(FidelityClass::Persistence, "checkpoint", || { + format!( + "revision={revision_id:032x} revisions={} snapshots={} frontier={}", + self.revisions.len(), + self.loro_snapshots.len(), + frontier_tag(&frontier) + ) + }); + Ok(revision_id) + } + + pub fn load_revision_loro_doc(&self, revision_id: u128) -> io::Result { + let revision = self + .revisions + .iter() + .find(|revision| revision.revision_id == revision_id) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "Flowstate package revision is missing"))?; + fidelity::event(FidelityClass::Persistence, "load-revision", || { + format!("revision={:032x} frontier={}", revision.revision_id, frontier_tag(&revision.frontier)) + }); + if let Some(snapshot) = self.snapshot_for_frontier(&revision.frontier) { + self.verify_snapshot_chunk(snapshot)?; + let doc = LoroDoc::new(); + crate::loro_schema::configure_text_styles(&doc); + doc.import(&snapshot.bytes).map_err(loro_io_error)?; + return Ok(doc); + } + let doc = self.load_loro_doc()?; + let frontiers = decode_frontiers(&revision.frontier)?; + doc.fork_at(&frontiers).map_err(loro_io_error) + } + + pub fn compact_to_named_snapshot( + &mut self, + doc: &LoroDoc, + title: impl Into, + summary: impl Into, + author_user_id: Option, + replica_id: Option, + ) -> io::Result<(u128, u128)> { + self.compact_to_named_snapshot_with_id(doc, Uuid::new_v4().as_u128(), title, summary, author_user_id, replica_id) + } + + pub fn sync_revisions_from_loro(&mut self, doc: &LoroDoc) -> io::Result { + let root = doc.get_map(crate::loro_schema::ROOT); + let Some(ValueOrContainer::Container(Container::List(revisions))) = root.get(crate::loro_schema::REVISIONS) else { + return Ok(0); + }; + let mut added = 0usize; + for index in 0..revisions.len() { + let Some(ValueOrContainer::Container(Container::Map(revision))) = revisions.get(index) else { + continue; + }; + let Some(revision_id) = package_map_string(&revision, "id").and_then(|id| id.parse::().ok()) else { + continue; + }; + if self + .revisions + .iter() + .any(|existing| existing.revision_id == revision_id) + { + continue; + } + let Some(frontier) = package_map_binary(&revision, "frontier") else { + continue; + }; + let frontiers = decode_frontiers(&frontier)?; + let version_vector = encode_version_vector(&frontiers_version_vector(doc, &frontiers)?); + if self.snapshot_for_frontier(&frontier).is_none() { + self.loro_snapshots.push(LoroSnapshotChunk { + snapshot_id: Uuid::new_v4().as_u128(), + frontier: frontier.clone(), + version_vector: version_vector.clone(), + // This runs on every remote import that learns a new revision, so it + // must stay O(revision) — export full `SnapshotAt` bytes straight + // from the live doc instead of paying an O(doc) `fork_at`. The + // lighter `StateOnly` variant is an explicitly deferred decision. + bytes: doc + .export(ExportMode::snapshot_at(&frontiers)) + .map_err(loro_io_error)?, + created_at_unix_secs: package_map_i64(&revision, "timestamp").unwrap_or_else(unix_time_secs), + }); + } + self.revisions.push(PackageRevision { + revision_id, + frontier, + version_vector, + title: package_map_string(&revision, "title").unwrap_or_else(|| "Revision".to_string()), + summary: package_map_string(&revision, "summary").unwrap_or_default(), + author_user_id: package_map_string(&revision, "author_user_id").and_then(|id| id.parse().ok()), + replica_id: package_map_string(&revision, "replica_id").and_then(|id| id.parse().ok()), + created_at_unix_secs: package_map_i64(&revision, "timestamp").unwrap_or_else(unix_time_secs), + }); + added += 1; + } + if added > 0 { + self + .revisions + .sort_by_key(|revision| revision.created_at_unix_secs); + self.manifest.modified_at_unix_secs = unix_time_secs(); + // §19: syncing revisions may add snapshot chunks; keep the index complete. + self.integrity_index = self.build_integrity_index(); + self.validate()?; + } + Ok(added) + } + + pub fn compact_to_named_snapshot_with_id( + &mut self, + doc: &LoroDoc, + revision_id: u128, + title: impl Into, + summary: impl Into, + author_user_id: Option, + replica_id: Option, + ) -> io::Result<(u128, u128)> { + let snapshot_id = self.compact_to_snapshot_any(doc)?; + let revision_id = self.create_named_revision_with_id(doc, revision_id, title, summary, author_user_id, replica_id)?; + Ok((revision_id, snapshot_id)) + } + + /// §A12.1.3 slice 1 (dark chunk): export a shallow snapshot of `doc` + /// rooted at (a causal floor of) `requested_root` and store it as the + /// package's SINGLE shallow accelerator chunk, stamping + /// `manifest.shallow_snapshot_frontier`. The full snapshot and update + /// segments are untouched — invariant S1 (`flowstate_shallow_open_design + /// .md` §2): the shallow chunk is never the only copy of any history. + /// + /// Loro floors the requested root to a causally-closed cut, so the TRUE + /// root is read back from a scratch decode of the exported bytes (bounded + /// by O(state + window); this runs on checkpoint paths, never per edit). + pub fn record_shallow_snapshot(&mut self, doc: &LoroDoc, requested_root: &Frontiers) -> io::Result { + doc.commit(); + let bytes = doc + .export(ExportMode::shallow_snapshot(requested_root)) + .map_err(loro_io_error)?; + // §A13.4.3: the true (floored) root used to be read back from a scratch + // decode of the exported bytes (~60ms per checkpoint). For a SINGLE-HEAD + // requested root the floor is the identity (`calc_shallow_doc_start` + // only reduces multi-head frontiers), and the export's start-vv law is + // `frontiers_to_vv(root)` with each head clamped DOWN to its own counter + // (the root op itself rides the snapshot). Multi-head roots — and + // `FLOWSTATE_SHALLOW_ROOT_VERIFY`, which decodes and asserts equality — + // take the probe path. + static ROOT_VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + let verify = *ROOT_VERIFY.get_or_init(|| std::env::var_os("FLOWSTATE_SHALLOW_ROOT_VERIFY").is_some()); + let analytic = if requested_root.len() <= 1 { + let mut root_version_vector = doc + .frontiers_to_vv(requested_root) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "shallow root frontier is not in the document history"))?; + for id in requested_root.iter() { + root_version_vector.insert(id.peer, id.counter); + } + Some((requested_root.clone(), root_version_vector, doc.oplog_frontiers(), doc.oplog_vv())) + } else { + None + }; + let (root_frontier, root_version_vector, at_frontiers, at_vv) = match analytic { + Some(analytic) if !verify => analytic, + analytic => { + let probe = LoroDoc::new(); + probe.import(&bytes).map_err(loro_io_error)?; + let decoded = ( + probe.shallow_since_frontiers(), + probe.shallow_since_vv().to_vv(), + probe.oplog_frontiers(), + probe.oplog_vv(), + ); + if verify && let Some(analytic) = analytic { + assert!( + analytic.0 == decoded.0 && analytic.1 == decoded.1 && analytic.2 == decoded.2 && analytic.3 == decoded.3, + "[flowstate-shallow-root-verify] analytic shallow root diverges from the decoded chunk" + ); + } + decoded + }, + }; + let snapshot_id = Uuid::new_v4().as_u128(); + self.shallow_snapshots.clear(); + let frontier = encode_frontiers(&at_frontiers); + self.shallow_snapshots.push(LoroShallowSnapshotChunk { + snapshot_id, + root_frontier: encode_frontiers(&root_frontier), + root_version_vector: encode_version_vector(&root_version_vector), + frontier: frontier.clone(), + version_vector: encode_version_vector(&at_vv), + bytes, + created_at_unix_secs: unix_time_secs(), + }); + self.manifest.shallow_snapshot_frontier = Some(frontier); + self.manifest.package_format_version = LORO_PACKAGE_FORMAT_VERSION_SHALLOW; + self.refresh_manifest_indexes(); + Ok(snapshot_id) + } + + /// §A12.4.1: refresh the shallow accelerator at the live tip WITHOUT + /// compacting (root policy: previous checkpoint frontier, as in + /// [`Self::compact_to_snapshot`]'s mint). Non-fatal by design — the chunk + /// is an accelerator (S1); on failure the previous chunk is dropped + /// rather than left stale-stamped. + pub fn refresh_shallow_snapshot(&mut self, live: &LoroDoc) { + // §A13.4.3: the shallow export costs ~200ms on a large doc — do NOT pay + // it per routine checkpoint. A slightly stale accelerator only costs the + // next open a replay of the segments past it (Loro drops known ops at + // decode), so refresh only once enough segments have accumulated beyond + // the chunk. Consolidations (`compact_to_snapshot_any`) always re-mint. + const MAX_SEGMENTS_PAST_SHALLOW: usize = 64; + if let Some(shallow) = self.latest_shallow_snapshot() { + let past = match self + .loro_update_segments + .iter() + .position(|segment| segment.from_frontier == shallow.frontier) + { + Some(position) => self.loro_update_segments.len() - position, + // Chunk frontier not on the segment chain: at the base snapshot + // frontier (all segments are past it) or off-chain (stale) — count + // everything either way. + None => self.loro_update_segments.len(), + }; + if past < MAX_SEGMENTS_PAST_SHALLOW { + return; + } + } + let previous_checkpoint_frontier = self.manifest.latest_frontier.clone(); + let requested_root = decode_frontiers(&previous_checkpoint_frontier).unwrap_or_default(); + if let Err(error) = self.record_shallow_snapshot(live, &requested_root) { + eprintln!("[flowstate-shallow-open] refreshing the shallow snapshot failed (package keeps the full path): {error}"); + self.clear_shallow_snapshot(); + } + } + + /// §A12.1.3: the package's shallow accelerator chunk, if present. + pub fn latest_shallow_snapshot(&self) -> Option<&LoroShallowSnapshotChunk> { + self.shallow_snapshots.last() + } + + /// §A12.1.3: drop the shallow accelerator chunk (e.g. when it goes stale + /// relative to a rewritten history). Always safe — it is never the only + /// copy of anything (invariant S1). + pub fn clear_shallow_snapshot(&mut self) { + self.shallow_snapshots.clear(); + self.manifest.shallow_snapshot_frontier = None; + self.manifest.package_format_version = LORO_PACKAGE_FORMAT_VERSION; + self.refresh_manifest_indexes(); + } + + /// §A12.1.3 slice 3: reconstruct the FULL-history doc from this package's + /// full snapshot chunk + update segments, caught up to `tip`'s frontier via + /// a post-root updates import (always legal: every op past the package's + /// persisted state is post-root by construction). This is the + /// load-serve-drop sidecar — transient, never held by the editor. + pub fn reconstruct_full_doc(&self, tip: &LoroDoc) -> io::Result { + let full = self.load_loro_doc_from_validated()?; + if full.is_shallow() { + // The package's "full" chunk is itself shallow — S1 was violated by an + // earlier writer. Fail loudly; compaction from this doc would compound + // the history loss. + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "package full-history snapshot is shallow (S1 violation); refusing sidecar reconstruction", + )); + } + let missing = tip + .export(ExportMode::updates(&full.oplog_vv())) + .map_err(loro_io_error)?; + if !missing.is_empty() { + full.import(&missing).map_err(loro_io_error)?; + } + if full.oplog_frontiers() != tip.oplog_frontiers() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "sidecar reconstruction did not reach the live tip", + )); + } + Ok(full) + } + + /// §A12.1.3 slice 3: compaction that is correct for BOTH full and shallow + /// live docs. A shallow live doc cannot full-export (its snapshot export is + /// silently shallow — see the guard in [`Self::compact_to_snapshot`]), so + /// the full-history doc is reconstructed from the package (sidecar) and + /// compaction runs from IT; the shallow accelerator is then re-minted at + /// the new checkpoint so the next open stays fast. + pub fn compact_to_snapshot_any(&mut self, live: &LoroDoc) -> io::Result { + if !live.is_shallow() { + return self.compact_to_snapshot(live); + } + let previous_checkpoint_frontier = self.manifest.latest_frontier.clone(); + let full = self.reconstruct_full_doc(live)?; + let snapshot_id = self.compact_to_snapshot(&full)?; + let requested_root = decode_frontiers(&previous_checkpoint_frontier).unwrap_or_default(); + self.record_shallow_snapshot(live, &requested_root)?; + Ok(snapshot_id) + } + + pub fn compact_to_snapshot(&mut self, doc: &LoroDoc) -> io::Result { + // §A12.1.3 invariant S1: a shallow doc's `Snapshot` export is silently + // SHALLOW — storing it as the full chunk and clearing segments would + // orphan all pre-root history. Consolidation of a shallow live doc must + // come from the full-history sidecar (design §3); until that lands, + // refuse loudly rather than lose history. + if doc.is_shallow() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "refusing to compact from a shallow live doc: its snapshot export would orphan pre-root history", + )); + } + doc.commit(); + // §A12.1.3 §7 root policy: the shallow root trails by one checkpoint — + // the manifest frontier BEFORE this compaction. + let previous_checkpoint_frontier = self.manifest.latest_frontier.clone(); + let snapshot_id = Uuid::new_v4().as_u128(); + let now = unix_time_secs(); + let frontier = encode_frontiers(&doc.state_frontiers()); + let version_vector = encode_version_vector(&doc.state_vv()); + let bytes = doc.export(ExportMode::Snapshot).map_err(loro_io_error)?; + let mut next = self.clone(); + next.loro_snapshots.push(LoroSnapshotChunk { + snapshot_id, + frontier: frontier.clone(), + version_vector, + bytes, + created_at_unix_secs: now, + }); + next.loro_update_segments.clear(); + next.manifest.latest_snapshot_id = snapshot_id; + next.manifest.style_marks_clean_frontier = Some(frontier.clone()); + next.manifest.latest_frontier = frontier; + next.manifest.latest_version_vector = encode_version_vector(&doc.state_vv()); + next.manifest.modified_at_unix_secs = now; + let retained_revision_frontiers = next + .revisions + .iter() + .map(|revision| revision.frontier.clone()) + .collect::>(); + next.loro_snapshots.retain(|snapshot| { + snapshot.snapshot_id == snapshot_id + || retained_revision_frontiers + .iter() + .any(|frontier| frontier.as_slice() == snapshot.frontier.as_slice()) + }); + next.refresh_manifest_indexes(); + next.rebuild_projection_cache_from_loro(doc)?; + next.rebuild_search_units_from_loro(doc)?; + // §A12.1.3 (default ON): also mint the shallow accelerator chunk so the + // next open takes the shallow path. Failures are non-fatal — the chunk + // is an accelerator, never a required artifact (invariant S1). + static WRITE_SHALLOW: std::sync::OnceLock = std::sync::OnceLock::new(); + if *WRITE_SHALLOW.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_SHALLOW_SNAPSHOT").is_none()) { + let requested_root = decode_frontiers(&previous_checkpoint_frontier).unwrap_or_default(); + if let Err(error) = next.record_shallow_snapshot(doc, &requested_root) { + eprintln!("[flowstate-shallow-open] minting the shallow snapshot failed (package keeps the full path): {error}"); + next.clear_shallow_snapshot(); + } + } + *self = next; + // §fidelity persistence: record the compaction and assert that compaction did + // not drop the ability to reload any named revision, and that the compacted + // package still round-trips to the source projection. The revision reload + // loop is eager, so gate the whole block on `enabled()`. + if fidelity::enabled() { + fidelity::event(FidelityClass::Persistence, "compact", || { + format!( + "snapshot={snapshot_id:032x} snapshots={} segments={} revisions={} frontier={}", + self.loro_snapshots.len(), + self.loro_update_segments.len(), + self.revisions.len(), + frontier_tag(&self.manifest.latest_frontier) + ) + }); + for revision in &self.revisions { + let loadable = self.load_revision_loro_doc(revision.revision_id).is_ok(); + fidelity::check(loadable, FidelityClass::Persistence, "revision-unloadable-after-compact", || { + format!("named revision {:032x} is not loadable after compaction", revision.revision_id) + }); + } + self.fidelity_check_roundtrip(doc); + } + Ok(snapshot_id) + } + + pub fn compact_update_segments_if_needed(&mut self, doc: &LoroDoc, max_update_segments: usize) -> io::Result> { + if max_update_segments == 0 || self.loro_update_segments.len() <= max_update_segments { + return Ok(None); + } + + self.compact_to_snapshot_any(doc).map(Some) + } + + pub fn rebuild_search_units_from_loro(&mut self, doc: &LoroDoc) -> io::Result<()> { + doc.commit(); + let frontier = encode_frontiers(&doc.state_frontiers()); + self.search_units = crate::package_search::search_units_from_loro(doc, self.manifest.document_id, &frontier)?; + self.manifest.search_cache_frontier = Some(frontier); + self.manifest.modified_at_unix_secs = unix_time_secs(); + self.validate()?; + Ok(()) + } + + pub fn read(path: impl AsRef) -> io::Result { + let path = path.as_ref(); + let probe_t0 = std::time::Instant::now(); + let bytes = fs::read(path)?; + if open_probe() { + eprintln!("[flowstate-read-probe] fs_read={:?} bytes={}", probe_t0.elapsed(), bytes.len()); + } + let package = Self::from_bytes(&bytes)?; + if bytes.starts_with(JOURNAL_MAGIC) { + let (_, committed_end) = committed_journal_transactions(&bytes)?; + if committed_end != bytes.len() { + let file = OpenOptions::new().write(true).open(path)?; + file.set_len(committed_end as u64)?; + file.sync_all()?; + } + } + Ok(package) + } + + /// Preview fast path (§P3): materialize a [`crate::DocumentProjection`] + /// from a package file using ONLY its manifest, frontier-current projection + /// cache, and asset chunks. Snapshot/segment chunks are neither hashed, + /// copied, nor decoded, and no `validate()` sweep runs — on a large + /// package the full read spent seconds in exactly those (the ".db8 preview + /// slower than the equivalent .docx" field report). Returns `Ok(None)` + /// when no frontier-current cache exists; callers fall back to the full + /// read + Loro materialization. Read-only: never truncates a journal tail. + pub fn read_cached_projection(path: impl AsRef) -> io::Result> { + let bytes = fs::read(path)?; + let cached = if bytes.starts_with(JOURNAL_MAGIC) { + cached_projection_from_journal_bytes(&bytes)? + } else { + cached_projection_from_compact_bytes(&bytes)? + }; + let Some((manifest, caches, assets)) = cached else { + return Ok(None); + }; + let Some(frontier) = manifest.projection_cache_frontier.as_deref() else { + return Ok(None); + }; + if frontier != manifest.latest_frontier.as_slice() { + return Ok(None); + } + let Some(cache) = caches.iter().find(|cache| cache.frontier == frontier) else { + return Ok(None); + }; + let projection = decode_chunk::(&cache.bytes, "projection cache payload")?; + let mut document = crate::loro_projection::document_from_projection_blocks(projection); + if document.ids.document_id == 0 { + document.ids.document_id = manifest.document_id; + } + document.frontier = frontier.to_vec(); + crate::attach_package_assets(&mut document, &assets); + Ok(Some(document)) + } + + /// §act-four M2: the CHEAPEST cold preview — decode ONLY the manifest + the + /// frontier-current preview header (the leading `PREVIEW_HEADER_BLOCKS` + /// blocks), never the snapshot, segments, projection cache, or assets. This + /// is `O(viewport)` regardless of document size, and — because the header + /// rides journal deltas — stays fresh for edited-but-unflushed documents + /// where the full projection cache is absent. Returns `Ok(None)` when no + /// frontier-current header exists (the caller falls back to + /// `read_cached_projection` → full read). Read-only. + pub fn read_preview_header(path: impl AsRef) -> io::Result> { + // §act-ten A10.13: POSITIONED reads. The old `fs::read` pulled the whole + // package (377 MB for a large .db8) to decode a 16-block header — the M2 + // "O(viewport)" claim was false on the I/O axis. Now: seek-walk the journal + // transaction headers (tiny reads), pread only the last full generation's + // chunk TABLE plus its manifest/preview chunks, and read the (small) delta + // payloads after it. Falls back to the whole-file path on any read error + // so a torn/foreign file degrades exactly as before. + let extracted = match preview_header_positioned(path.as_ref()) { + Ok(extracted) => extracted, + Err(_) => { + let bytes = fs::read(path)?; + if bytes.starts_with(JOURNAL_MAGIC) { + preview_header_from_journal_bytes(&bytes)? + } else { + preview_header_from_compact_bytes(&bytes)? + } + }, + }; + let Some((manifest, headers)) = extracted else { + return Ok(None); + }; + let Some(frontier) = manifest.preview_header_frontier.as_deref() else { + return Ok(None); + }; + if frontier != manifest.latest_frontier.as_slice() { + return Ok(None); // stale header — the caller falls back. + } + let Some(header) = headers.iter().find(|header| header.frontier == frontier) else { + return Ok(None); + }; + let projection = decode_chunk::(&header.bytes, "preview header payload")?; + let mut document = crate::loro_projection::document_from_projection_blocks(projection); + if document.ids.document_id == 0 { + document.ids.document_id = manifest.document_id; + } + document.frontier = frontier.to_vec(); + Ok(Some(document)) + } + + pub fn write(&self, path: impl AsRef) -> io::Result<()> { + let path = path.as_ref(); + reject_foreign_package_path(path)?; + fidelity::event(FidelityClass::Persistence, "write", || { + format!( + "snapshots={} segments={} revisions={} assets={} frontier={}", + self.loro_snapshots.len(), + self.loro_update_segments.len(), + self.revisions.len(), + self.assets.len(), + frontier_tag(&self.manifest.latest_frontier) + ) + }); + let payload = self.to_bytes()?; + if file_has_journal_header(path)? { + let bytes = fs::read(path)?; + let (transactions, committed_end) = committed_journal_transactions(&bytes)?; + let rewrite = committed_end != bytes.len() + || transactions.len() >= JOURNAL_GENERATION_COMPACTION_THRESHOLD + || bytes.len() > journal_transaction_len(payload.len()).saturating_mul(4); + if rewrite { + write_journal_generation(path, &payload) + } else { + append_journal_transaction(path, &payload) + } + } else { + write_journal_generation(path, &payload) + } + } + + pub fn append_latest_update_to_path(&self, path: impl AsRef) -> io::Result<()> { + let path = path.as_ref(); + reject_foreign_package_path(path)?; + let Some(segment) = self.loro_update_segments.last().cloned() else { + return self.write(path); + }; + if !file_has_journal_header(path)? { + return self.write(path); + } + let payload = encode_journal_delta(&PackageJournalDelta::Update { + manifest: self.manifest.clone(), + segment, + preview_header: self.preview_headers.last().cloned(), + })?; + append_journal_transaction(path, &payload) + } + + pub fn append_latest_update_to_prepared_path(&self, path: impl AsRef) -> io::Result<()> { + self.append_latest_update_to_prepared_path_with_durability(path, true) + } + + /// §act-twelve A12.5.1: the repair-persist lane. Canonical REPAIRS are + /// re-derivable (a crash that loses one re-detects the defect and repairs + /// convergently on the next projection — the whole §P2a design), so their + /// journal append may skip the per-transaction `sync_all` that dominated + /// heal cost on contended disks. The journal's transaction framing already + /// drops a torn unsynced tail at replay, and ANY later synced append + /// (edits, checkpoints) syncs the whole file, covering earlier lazy + /// transactions. + pub fn append_latest_update_to_prepared_path_lazy(&self, path: impl AsRef) -> io::Result<()> { + self.append_latest_update_to_prepared_path_with_durability(path, false) + } + + fn append_latest_update_to_prepared_path_with_durability(&self, path: impl AsRef, sync: bool) -> io::Result<()> { + let path = path.as_ref(); + reject_foreign_package_path(path)?; + let Some(segment) = self.loro_update_segments.last().cloned() else { + return self.write(path); + }; + if !file_has_journal_header(path)? { + return self.write(path); + } + let payload = encode_journal_delta(&PackageJournalDelta::Update { + manifest: self.manifest.clone(), + segment, + preview_header: self.preview_headers.last().cloned(), + })?; + append_journal_transaction_to_prepared_file_with_durability(path, &payload, sync) + } + + pub fn append_assets_to_path(&self, path: impl AsRef) -> io::Result<()> { + let path = path.as_ref(); + reject_foreign_package_path(path)?; + if !file_has_journal_header(path)? { + return self.write(path); + } + let payload = encode_journal_delta(&PackageJournalDelta::Assets { + manifest: self.manifest.clone(), + assets: self.assets.clone(), + })?; + append_journal_transaction(path, &payload) + } + + pub fn from_bytes(bytes: &[u8]) -> io::Result { + if bytes.starts_with(JOURNAL_MAGIC) { + return Self::from_journal_bytes(bytes); + } + Self::from_compact_bytes(bytes) + } + + fn from_journal_bytes(bytes: &[u8]) -> io::Result { + let probe_t0 = std::time::Instant::now(); + let payloads = committed_journal_payloads(bytes)?; + let probe_scan = probe_t0.elapsed(); + // Decode from the LAST full generation: everything before it is + // superseded, and decoding every stacked generation made opening a large + // journal (~5 × 86MB on the impact-doc package) tens of seconds. + let latest_generation = payloads + .iter() + .rposition(|payload| payload.starts_with(PACKAGE_MAGIC)) + .unwrap_or(0); + let mut package = None; + for payload in &payloads[latest_generation..] { + let payload = *payload; + if payload.starts_with(PACKAGE_MAGIC) { + package = Some(Self::from_compact_bytes(payload)?); + continue; + } + let delta = decode_journal_delta(payload)?; + let current = package + .as_mut() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package journal delta precedes a full generation"))?; + match delta { + PackageJournalDelta::Update { + manifest, + segment, + preview_header, + } => { + if !current + .loro_update_segments + .iter() + .any(|existing| existing.segment_id == segment.segment_id) + { + current.loro_update_segments.push(segment); + } + current.manifest = manifest; + if current.manifest.projection_cache_frontier.is_none() { + current.projection_caches.clear(); + } + if current.manifest.search_cache_frontier.is_none() { + current.search_units.clear(); + } + // §act-four M2: adopt the delta's fresh preview header (edited docs + // keep an up-to-date header even without a full checkpoint). + if let Some(header) = preview_header { + current.preview_headers = vec![header]; + } + }, + PackageJournalDelta::Assets { manifest, assets } => { + current.manifest = manifest; + current.assets = assets; + }, + } + } + let mut package = + package.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package journal has no complete full generation"))?; + // §19: journal deltas append update segments after the base generation, so + // rebuild the integrity index over the reconstructed chunk set before + // validating it for consistency. The manifest's own segment/asset indexes + // arrive with the delta and remain checked by `validate_manifest_indexes`. + package.integrity_index = package.build_integrity_index(); + let probe_t1 = std::time::Instant::now(); + package.validate()?; + if open_probe() { + eprintln!( + "[flowstate-read-probe] journal_scan+hash={probe_scan:?} generations={} journal_validate={:?}", + payloads.len(), + probe_t1.elapsed() + ); + } + Ok(package) + } + + fn from_compact_bytes(bytes: &[u8]) -> io::Result { + // Decode each chunk straight from its slice into the owned structs — the + // former shape copied every payload into an intermediate `Vec` first, + // a full extra pass over a multi-hundred-MB package on every open. + let probe_t0 = std::time::Instant::now(); + let chunks = read_chunk_slices_deferring(bytes, |_| true, |kind| kind == CHUNK_LORO_SNAPSHOT)?; + let probe_slices = probe_t0.elapsed(); + let probe_t1 = std::time::Instant::now(); + let mut manifest = None; + let mut deferred_snapshot_checksums = std::collections::HashMap::new(); + let mut loro_snapshots = Vec::new(); + let mut loro_update_segments = Vec::new(); + let mut assets = Vec::new(); + let mut revisions = Vec::new(); + let mut projection_caches = Vec::new(); + let mut search_units = Vec::new(); + let mut thumbnails = Vec::new(); + let mut integrity_index = Vec::new(); + let mut preview_headers = Vec::new(); + let mut shallow_snapshots = Vec::new(); + + for (kind, payload, deferred_checksum) in chunks { + match kind { + CHUNK_MANIFEST => manifest = Some(decode_chunk(payload, "manifest")?), + CHUNK_LORO_SNAPSHOT => { + let snapshot: LoroSnapshotChunk = decode_chunk(payload, "Loro snapshot")?; + if let Some(checksum) = deferred_checksum { + deferred_snapshot_checksums.insert(snapshot.snapshot_id, checksum); + } + loro_snapshots.push(snapshot); + }, + CHUNK_LORO_UPDATE_SEGMENT => loro_update_segments.push(decode_chunk(payload, "Loro update segment")?), + CHUNK_ASSET => assets.push(decode_chunk(payload, "asset")?), + CHUNK_REVISION_INDEX => revisions.push(decode_chunk(payload, "revision")?), + CHUNK_PROJECTION_CACHE => projection_caches.push(decode_chunk(payload, "projection cache")?), + CHUNK_SEARCH_UNIT => search_units.push(decode_chunk(payload, "search unit")?), + CHUNK_THUMBNAIL => thumbnails.push(decode_chunk(payload, "thumbnail")?), + CHUNK_INTEGRITY_INDEX => integrity_index.push(decode_chunk(payload, "integrity index entry")?), + CHUNK_PREVIEW_HEADER => preview_headers.push(decode_chunk(payload, "preview header")?), + CHUNK_SHALLOW_SNAPSHOT => shallow_snapshots.push(decode_chunk(payload, "shallow Loro snapshot")?), + _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "unknown Flowstate package chunk kind")), + } + } + + let package = Self { + manifest: manifest.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package has no manifest"))?, + loro_snapshots, + loro_update_segments, + assets, + revisions, + projection_caches, + search_units, + thumbnails, + integrity_index, + preview_headers, + shallow_snapshots, + deferred_snapshot_checksums, + }; + let probe_decode = probe_t1.elapsed(); + let probe_t2 = std::time::Instant::now(); + package.validate()?; + if open_probe() { + eprintln!( + "[flowstate-read-probe] table+hash={probe_slices:?} chunk_decode={probe_decode:?} validate={:?}", + probe_t2.elapsed() + ); + } + Ok(package) + } + + pub fn to_bytes(&self) -> io::Result> { + // §A12.1.3 slice 2b: a write mints FRESH table checksums over the + // in-memory payloads — verify any read-deferred snapshot hashes first so + // on-disk corruption can never be laundered into a "valid" package. + for snapshot in &self.loro_snapshots { + self.verify_snapshot_chunk(snapshot)?; + } + self.validate()?; + let mut chunks = Vec::new(); + chunks.push(Chunk { + kind: CHUNK_MANIFEST, + payload: encode_chunk(&self.manifest, "manifest")?, + }); + for snapshot in &self.loro_snapshots { + chunks.push(Chunk { + kind: CHUNK_LORO_SNAPSHOT, + payload: encode_chunk(snapshot, "Loro snapshot")?, + }); + } + for segment in &self.loro_update_segments { + chunks.push(Chunk { + kind: CHUNK_LORO_UPDATE_SEGMENT, + payload: encode_chunk(segment, "Loro update segment")?, + }); + } + for asset in &self.assets { + chunks.push(Chunk { + kind: CHUNK_ASSET, + payload: encode_chunk(asset, "asset")?, + }); + } + for revision in &self.revisions { + chunks.push(Chunk { + kind: CHUNK_REVISION_INDEX, + payload: encode_chunk(revision, "revision")?, + }); + } + for cache in &self.projection_caches { + chunks.push(Chunk { + kind: CHUNK_PROJECTION_CACHE, + payload: encode_chunk(cache, "projection cache")?, + }); + } + for header in &self.preview_headers { + chunks.push(Chunk { + kind: CHUNK_PREVIEW_HEADER, + payload: encode_chunk(header, "preview header")?, + }); + } + for shallow in &self.shallow_snapshots { + chunks.push(Chunk { + kind: CHUNK_SHALLOW_SNAPSHOT, + payload: encode_chunk(shallow, "shallow Loro snapshot")?, + }); + } + for unit in &self.search_units { + chunks.push(Chunk { + kind: CHUNK_SEARCH_UNIT, + payload: encode_chunk(unit, "search unit")?, + }); + } + for thumbnail in &self.thumbnails { + chunks.push(Chunk { + kind: CHUNK_THUMBNAIL, + payload: encode_chunk(thumbnail, "thumbnail")?, + }); + } + for entry in &self.integrity_index { + chunks.push(Chunk { + kind: CHUNK_INTEGRITY_INDEX, + payload: encode_chunk(entry, "integrity index entry")?, + }); + } + write_chunks(&chunks) + } + + /// Delta-scoped validation for [`Self::append_update_segment`]: the new + /// segment must chain from the previous persisted tip, carry a correct + /// checksum, and land the manifest on its `to` frontier. Everything already + /// stored was validated when it entered the package. + fn validate_appended_segment(&self) -> io::Result<()> { + if !package_format_version_supported(self.manifest.package_format_version) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported Flowstate package format version")); + } + if self.manifest.loro_schema_version != LORO_SCHEMA_VERSION { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported Flowstate Loro schema version")); + } + let segment = self + .loro_update_segments + .last() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "appended update segment is missing"))?; + if segment.checksum != blake3_hash(&segment.bytes) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Loro update segment checksum mismatch")); + } + validate_frontiers(&segment.from_frontier, "update segment from frontier")?; + validate_version_vector(&segment.from_version_vector, "update segment from version vector")?; + validate_frontiers(&segment.to_frontier, "update segment to frontier")?; + validate_version_vector(&segment.to_version_vector, "update segment to version vector")?; + if segment.to_frontier != self.manifest.latest_frontier { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "latest update segment frontier does not match manifest", + )); + } + let previous_tip = if self.loro_update_segments.len() >= 2 { + &self.loro_update_segments[self.loro_update_segments.len() - 2].to_frontier + } else { + let snapshot = self + .latest_snapshot() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "latest Loro snapshot is missing"))?; + &snapshot.frontier + }; + if &segment.from_frontier != previous_tip { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "appended update segment does not chain from the persisted tip", + )); + } + Ok(()) + } + + pub fn validate(&self) -> io::Result<()> { + if !package_format_version_supported(self.manifest.package_format_version) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported Flowstate package format version")); + } + if self.manifest.loro_schema_version != LORO_SCHEMA_VERSION { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported Flowstate Loro schema version")); + } + let snapshot = self + .latest_snapshot() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "latest Loro snapshot is missing"))?; + if self.loro_update_segments.is_empty() && snapshot.frontier != self.manifest.latest_frontier { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "latest snapshot frontier does not match manifest", + )); + } + if let Some(last_segment) = self.loro_update_segments.last() + && last_segment.to_frontier != self.manifest.latest_frontier + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "latest update segment frontier does not match manifest", + )); + } + validate_frontiers(&self.manifest.latest_frontier, "manifest latest frontier")?; + validate_version_vector(&self.manifest.latest_version_vector, "manifest latest version vector")?; + if let Some(frontier) = &self.manifest.projection_cache_frontier { + validate_frontiers(frontier, "manifest projection cache frontier")?; + if !self + .projection_caches + .iter() + .any(|cache| cache.frontier == *frontier) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "manifest projection cache frontier has no cache chunk", + )); + } + } + if let Some(frontier) = &self.manifest.search_cache_frontier { + validate_frontiers(frontier, "manifest search cache frontier")?; + } + for snapshot in &self.loro_snapshots { + validate_frontiers(&snapshot.frontier, "snapshot frontier")?; + validate_version_vector(&snapshot.version_vector, "snapshot version vector")?; + if snapshot.bytes.is_empty() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "empty Loro snapshot bytes")); + } + } + for revision in &self.revisions { + validate_frontiers(&revision.frontier, "revision frontier")?; + validate_version_vector(&revision.version_vector, "revision version vector")?; + } + for shallow in &self.shallow_snapshots { + validate_frontiers(&shallow.root_frontier, "shallow snapshot root frontier")?; + validate_version_vector(&shallow.root_version_vector, "shallow snapshot root version vector")?; + validate_frontiers(&shallow.frontier, "shallow snapshot frontier")?; + validate_version_vector(&shallow.version_vector, "shallow snapshot version vector")?; + if shallow.bytes.is_empty() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "empty shallow Loro snapshot bytes")); + } + } + if let Some(stamp) = self.manifest.shallow_snapshot_frontier.as_deref() + && !self + .shallow_snapshots + .iter() + .any(|shallow| shallow.frontier == stamp) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "manifest names a shallow snapshot frontier with no matching chunk", + )); + } + for cache in &self.projection_caches { + validate_frontiers(&cache.frontier, "projection cache frontier")?; + // Structural decode of the cache payload is DEFERRED to its consumer + // (`current_projection_document`, which already fails soft to a full + // materialization on a bad payload): decoding the whole document model + // here made every routine validate — package open, save, append — pay + // seconds on a large document. + } + for thumbnail in &self.thumbnails { + validate_frontiers(&thumbnail.frontier, "thumbnail frontier")?; + if thumbnail.bytes.is_empty() || thumbnail.mime_type.is_empty() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid empty thumbnail chunk")); + } + if let Some(revision_id) = thumbnail.revision_id + && !self + .revisions + .iter() + .any(|revision| revision.revision_id == revision_id) + { + return Err(io::Error::new(io::ErrorKind::InvalidData, "thumbnail references an unknown revision")); + } + } + if self + .loro_update_segments + .iter() + .any(|segment| segment.checksum != blake3_hash(&segment.bytes)) + { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Loro update segment checksum mismatch")); + } + for segment in &self.loro_update_segments { + validate_frontiers(&segment.from_frontier, "update segment from frontier")?; + validate_version_vector(&segment.from_version_vector, "update segment from version vector")?; + validate_frontiers(&segment.to_frontier, "update segment to frontier")?; + validate_version_vector(&segment.to_version_vector, "update segment to version vector")?; + } + let mut expected_frontier = snapshot.frontier.as_slice(); + let mut expected_version_vector = snapshot.version_vector.as_slice(); + for segment in &self.loro_update_segments { + if segment.from_frontier != expected_frontier { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Loro update segment frontier chain is broken")); + } + if segment.from_version_vector != expected_version_vector { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Loro update segment version-vector chain is broken", + )); + } + expected_frontier = &segment.to_frontier; + expected_version_vector = &segment.to_version_vector; + } + if self + .assets + .iter() + .any(|asset| asset.content_hash != blake3_hash(&asset.bytes) || asset.byte_length != asset.bytes.len() as u64) + { + return Err(io::Error::new(io::ErrorKind::InvalidData, "asset hash or length mismatch")); + } + self.validate_manifest_indexes()?; + self.validate_integrity_index()?; + self.validate_schema_migrations()?; + // NOT validated here: a full `load_loro_doc_unvalidated()` round-trip. + // Importing the whole Loro snapshot as a validation step made every + // routine validate — open, save, compaction, and (via append) every + // keystroke — pay a multi-second CRDT import on a large document (the + // .db8 first-edit freeze). Corruption in the snapshot bytes still + // surfaces at the one place the doc is actually loaded. + Ok(()) + } + + pub fn rebuild_projection_cache_from_loro(&mut self, doc: &LoroDoc) -> io::Result<()> { + doc.commit(); + let frontier = encode_frontiers(&doc.state_frontiers()); + let projection = crate::loro_projection::projection_blocks_from_loro(doc)?; + // §act-four M2: the preview header is the leading blocks of the same + // materialization — O(K) to trim + serialize on top of the full cache. + self.write_preview_header(&frontier, &projection)?; + // §act-ten A10.5: `manifest.block_boundaries` is NOT persisted anymore. It + // fed only `cold_viewport_blocks`, which has no production caller and its + // own lazy derive fallback — while `body_block_boundaries` re-ran BOTH + // whole-registry scans the projection walk above just performed, per + // checkpoint (~every 900ms while typing; the pegged package thread). + self.projection_caches.clear(); + self.projection_caches.push(ProjectionCacheChunk { + frontier: frontier.clone(), + bytes: encode_chunk(&projection, "projection cache payload")?, + }); + self.manifest.projection_cache_frontier = Some(frontier); + self.manifest.modified_at_unix_secs = unix_time_secs(); + self.validate()?; + Ok(()) + } + + /// §act-five P9: rebuild BOTH the projection cache and the search units from a + /// SINGLE `document_from_loro` materialization. The separate + /// `rebuild_projection_cache_from_loro` + `rebuild_search_units_from_loro` + /// calls each ran a full projection walk — a double materialization per + /// checkpoint that pegged the off-gate package thread during collaboration. + /// The search units derive from the SAME materialized blocks the cache does. + /// §A13.4.1: cache rebuild that uses the runtime's MAINTAINED projection + /// when it matches the doc frontier — skipping the O(doc) + /// `document_from_loro` walk (365ms of a routine checkpoint on the impact + /// doc). Mismatch (or `FLOWSTATE_DISABLE_PROJECTION_CACHE_FASTPATH`) falls + /// back to the from-Loro walk; `FLOWSTATE_CHECKPOINT_CACHE_VERIFY` runs + /// both and asserts payload equality. + pub fn rebuild_caches_from_projection(&mut self, doc: &LoroDoc, document: &crate::DocumentProjection) -> io::Result<()> { + doc.commit(); + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + let disabled = *DISABLED.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_PROJECTION_CACHE_FASTPATH").is_some()); + let frontier = encode_frontiers(&doc.state_frontiers()); + if disabled || document.frontier != frontier { + return self.rebuild_caches_from_loro(doc); + } + let probe_t = std::time::Instant::now(); + let projection = projection_blocks_from_document(document); + if open_probe() { + eprintln!("[flowstate-cache-probe] projection_convert={:?}", probe_t.elapsed()); + } + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + if *VERIFY.get_or_init(|| std::env::var_os("FLOWSTATE_CHECKPOINT_CACHE_VERIFY").is_some()) { + let from_loro = crate::loro_projection::projection_blocks_from_loro(doc)?; + let ours = encode_chunk(&projection, "projection cache payload")?; + let theirs = encode_chunk(&from_loro, "projection cache payload")?; + assert!( + ours == theirs, + "[flowstate-checkpoint-cache-verify] projection-built cache payload diverges from the from-Loro walk" + ); + } + self.finish_cache_rebuild(doc, frontier, projection) + } + + pub fn rebuild_caches_from_loro(&mut self, doc: &LoroDoc) -> io::Result<()> { + doc.commit(); + let frontier = encode_frontiers(&doc.state_frontiers()); + let projection = crate::loro_projection::projection_blocks_from_loro(doc)?; + self.finish_cache_rebuild(doc, frontier, projection) + } + + fn finish_cache_rebuild(&mut self, doc: &LoroDoc, frontier: Vec, projection: crate::loro_projection::ProjectionBlocks) -> io::Result<()> { + let probe_t = std::time::Instant::now(); + self.write_preview_header(&frontier, &projection)?; + let probe_preview = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + // §act-ten A10.5: block-boundary persistence removed (see + // `rebuild_projection_cache_from_loro`). + self.search_units = crate::package_search::search_units_from_input_blocks(doc, &projection.blocks, self.manifest.document_id, &frontier)?; + let probe_search = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + self.manifest.search_cache_frontier = Some(frontier.clone()); + self.projection_caches.clear(); + self.projection_caches.push(ProjectionCacheChunk { + frontier: frontier.clone(), + bytes: encode_chunk(&projection, "projection cache payload")?, + }); + let probe_encode = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + self.manifest.projection_cache_frontier = Some(frontier); + self.manifest.modified_at_unix_secs = unix_time_secs(); + self.validate()?; + if open_probe() { + eprintln!( + "[flowstate-cache-probe] preview={probe_preview:?} search={probe_search:?} encode={probe_encode:?} validate={:?}", + probe_t.elapsed() + ); + } + Ok(()) + } + + /// §act-four M2: (re)write the preview header from a materialized + /// [`ProjectionBlocks`] — the leading `PREVIEW_HEADER_BLOCKS` blocks + their + /// ids, serialized into a tiny chunk at `frontier`. Idempotent: replaces any + /// existing header. The header is written on every checkpoint so cold preview + /// is always fresh and `O(viewport)`, independent of the full cache. + fn write_preview_header(&mut self, frontier: &[u8], projection: &crate::loro_projection::ProjectionBlocks) -> io::Result<()> { + let block_count = projection.blocks.len().min(PREVIEW_HEADER_BLOCKS); + // `paragraph_ids` is parallel to the PARAGRAPH blocks in order; count how + // many of the leading blocks are paragraphs to trim the ids in lockstep. + let paragraph_count = projection.blocks[..block_count] + .iter() + .filter(|block| matches!(block, crate::InputBlock::Paragraph(_))) + .count(); + let header = crate::loro_projection::ProjectionBlocks { + document_id: projection.document_id, + blocks: projection.blocks[..block_count].to_vec(), + paragraph_ids: projection.paragraph_ids[..paragraph_count.min(projection.paragraph_ids.len())].to_vec(), + block_ids: projection.block_ids[..block_count.min(projection.block_ids.len())].to_vec(), + sections: Vec::new(), + }; + self.preview_headers.clear(); + self.preview_headers.push(PreviewHeaderChunk { + frontier: frontier.to_vec(), + bytes: encode_chunk(&header, "preview header payload")?, + }); + self.manifest.preview_header_frontier = Some(frontier.to_vec()); + Ok(()) + } + + /// §act-four M2: refresh the preview header from a LIVE, already-materialized + /// projection (the runtime's), keyed to the package's current latest + /// frontier — `O(PREVIEW_HEADER_BLOCKS)`, no re-materialization. Called on + /// every edit-persist so the header stays fresh even between full checkpoints + /// (it then rides the journal delta to disk). The caller guarantees the + /// projection reflects `latest_frontier`. + pub fn refresh_preview_header(&mut self, projection: &crate::DocumentProjection) -> io::Result<()> { + let frontier = self.manifest.latest_frontier.clone(); + let header_blocks = preview_header_blocks_from_document(projection, PREVIEW_HEADER_BLOCKS); + self.preview_headers.clear(); + self.preview_headers.push(PreviewHeaderChunk { + frontier: frontier.clone(), + bytes: encode_chunk(&header_blocks, "preview header payload")?, + }); + self.manifest.preview_header_frontier = Some(frontier); + Ok(()) + } + + pub fn current_projection_document(&self) -> io::Result> { + let Some(frontier) = self.manifest.projection_cache_frontier.as_deref() else { + return Ok(None); + }; + if frontier != self.manifest.latest_frontier.as_slice() { + return Ok(None); + } + let Some(cache) = self + .projection_caches + .iter() + .find(|cache| cache.frontier == frontier) + else { + return Ok(None); + }; + let projection = decode_chunk::(&cache.bytes, "projection cache payload")?; + let mut document = crate::loro_projection::document_from_projection_blocks(projection); + if document.ids.document_id == 0 { + document.ids.document_id = self.manifest.document_id; + } + document.frontier = frontier.to_vec(); + Ok(Some(document)) + } + + fn validate_manifest_indexes(&self) -> io::Result<()> { + if self.manifest.update_segment_index.len() != self.loro_update_segments.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "manifest update segment index length mismatch", + )); + } + for (index, segment) in self + .manifest + .update_segment_index + .iter() + .zip(&self.loro_update_segments) + { + if index.id != segment.segment_id || index.checksum != segment.checksum || index.byte_length != segment.bytes.len() as u64 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "manifest update segment index mismatch")); + } + } + if self.manifest.asset_index.len() != self.assets.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "manifest asset index length mismatch")); + } + for (index, asset) in self.manifest.asset_index.iter().zip(&self.assets) { + if index.id != asset.asset_id || index.checksum != asset.content_hash || index.byte_length != asset.byte_length { + return Err(io::Error::new(io::ErrorKind::InvalidData, "manifest asset index mismatch")); + } + } + Ok(()) + } + + /// §19: cross-check the named integrity index against the actual durable + /// chunks. An absent index (older packages) is accepted for backward + /// compatibility; a present index must be complete and consistent. + fn validate_integrity_index(&self) -> io::Result<()> { + // §19: an empty integrity index is the legacy compatibility case for older + // packages that predate the named index. Preserve that behavior explicitly. + if self.integrity_index.is_empty() { + return Ok(()); + } + let expected = self.build_integrity_index(); + if self.integrity_index.len() != expected.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "integrity index entry count mismatch")); + } + let expected = Self::integrity_index_key_set(&expected)?; + let actual = Self::integrity_index_key_set(&self.integrity_index)?; + if actual != expected { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "integrity index entries do not match durable chunks", + )); + } + for entry in &self.integrity_index { + let actual = match entry.chunk_kind { + CHUNK_LORO_SNAPSHOT => self + .loro_snapshots + .iter() + .find(|snapshot| snapshot.snapshot_id == entry.id) + .map(|snapshot| (self.integrity_checksum_for_snapshot(snapshot), snapshot.bytes.len() as u64)), + CHUNK_LORO_UPDATE_SEGMENT => self + .loro_update_segments + .iter() + .find(|segment| segment.segment_id == entry.id) + .map(|segment| (segment.checksum, segment.bytes.len() as u64)), + CHUNK_ASSET => self + .assets + .iter() + .find(|asset| asset.asset_id == entry.id) + .map(|asset| (asset.content_hash, asset.byte_length)), + CHUNK_SHALLOW_SNAPSHOT => self + .shallow_snapshots + .iter() + .find(|shallow| shallow.snapshot_id == entry.id) + .map(|shallow| (blake3_hash(&shallow.bytes), shallow.bytes.len() as u64)), + _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "integrity index has an unknown chunk kind")), + }; + let Some((checksum, byte_length)) = actual else { + return Err(io::Error::new(io::ErrorKind::InvalidData, "integrity index references a missing chunk")); + }; + if entry.checksum != checksum || entry.byte_length != byte_length { + return Err(io::Error::new(io::ErrorKind::InvalidData, "integrity index checksum or length mismatch")); + } + } + Ok(()) + } + + /// §27: validate the schema migration log. Empty at schema v1. Each record + /// must describe a forward migration whose target does not exceed this + /// package's schema version. + fn validate_schema_migrations(&self) -> io::Result<()> { + for migration in &self.manifest.schema_migrations { + if migration.from_version >= migration.to_version || migration.to_version > self.manifest.loro_schema_version { + return Err(io::Error::new(io::ErrorKind::InvalidData, "inconsistent schema migration record")); + } + } + Ok(()) + } + + /// §27: record a schema migration event in package metadata. The migration's + /// Loro document mutation (if any) must be committed separately as an ordinary + /// Loro change with the `"migration"` origin before/after calling this. + pub fn record_schema_migration(&mut self, from_version: u32, to_version: u32, description: impl Into) -> io::Result { + if from_version >= to_version || to_version > self.manifest.loro_schema_version { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "schema migration must move forward to a version supported by this package", + )); + } + let migration_id = Uuid::new_v4().as_u128(); + self.manifest.schema_migrations.push(SchemaMigrationRecord { + id: migration_id, + from_version, + to_version, + applied_at_unix_secs: unix_time_secs(), + description: description.into(), + }); + self.manifest.modified_at_unix_secs = unix_time_secs(); + self.validate()?; + Ok(migration_id) + } + + pub fn latest_snapshot(&self) -> Option<&LoroSnapshotChunk> { + self + .loro_snapshots + .iter() + .find(|snapshot| snapshot.snapshot_id == self.manifest.latest_snapshot_id) + } + + fn snapshot_for_frontier(&self, frontier: &[u8]) -> Option<&LoroSnapshotChunk> { + self + .loro_snapshots + .iter() + .find(|snapshot| snapshot.frontier == frontier) + } + + fn with_manifest_indexes(mut self) -> io::Result { + self.refresh_manifest_indexes(); + Ok(self) + } + + /// Rebuild the manifest's update-segment/asset indexes and the §19 integrity + /// index from the package's current durable chunks. Infallible and idempotent. + /// Public wrapper for external maintenance tooling (e.g. restoring a + /// package to its latest snapshot after stripping segments). + pub fn refresh_manifest_indexes_public(&mut self) { + self.refresh_manifest_indexes(); + } + + fn refresh_manifest_indexes(&mut self) { + let mut update_segment_index = Vec::with_capacity(self.loro_update_segments.len()); + for segment in &self.loro_update_segments { + update_segment_index.push(ChunkRef { + id: segment.segment_id, + checksum: segment.checksum, + byte_length: segment.bytes.len() as u64, + }); + } + let mut asset_index = Vec::with_capacity(self.assets.len()); + for asset in &self.assets { + asset_index.push(ChunkRef { + id: asset.asset_id, + checksum: asset.content_hash, + byte_length: asset.byte_length, + }); + } + self.manifest.update_segment_index = update_segment_index; + self.manifest.asset_index = asset_index; + self.integrity_index = self.build_integrity_index(); + } + + /// §19: build a complete integrity index covering every durable chunk — Loro + /// snapshots, update segments, and assets — with each chunk's id, kind, BLAKE3 + /// checksum, and byte length. + fn build_integrity_index(&self) -> Vec { + let mut entries = Vec::with_capacity(self.loro_snapshots.len() + self.loro_update_segments.len() + self.assets.len()); + for snapshot in &self.loro_snapshots { + entries.push(IntegrityIndexEntry { + chunk_kind: CHUNK_LORO_SNAPSHOT, + id: snapshot.snapshot_id, + checksum: self.integrity_checksum_for_snapshot(snapshot), + byte_length: snapshot.bytes.len() as u64, + }); + } + for segment in &self.loro_update_segments { + entries.push(IntegrityIndexEntry { + chunk_kind: CHUNK_LORO_UPDATE_SEGMENT, + id: segment.segment_id, + checksum: segment.checksum, + byte_length: segment.bytes.len() as u64, + }); + } + for asset in &self.assets { + entries.push(IntegrityIndexEntry { + chunk_kind: CHUNK_ASSET, + id: asset.asset_id, + checksum: asset.content_hash, + byte_length: asset.byte_length, + }); + } + for shallow in &self.shallow_snapshots { + entries.push(IntegrityIndexEntry { + chunk_kind: CHUNK_SHALLOW_SNAPSHOT, + id: shallow.snapshot_id, + checksum: blake3_hash(&shallow.bytes), + byte_length: shallow.bytes.len() as u64, + }); + } + entries + } + + fn integrity_index_key_set(entries: &[IntegrityIndexEntry]) -> io::Result> { + let mut keys = HashSet::with_capacity(entries.len()); + for entry in entries { + if !keys.insert((entry.chunk_kind, entry.id)) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "integrity index contains duplicate chunk identity", + )); + } + } + Ok(keys) + } +} + +pub fn read_loro_db8(path: impl AsRef) -> io::Result { + DocumentPackage::read(path)?.load_loro_doc() +} + +pub fn write_loro_db8(path: impl AsRef, doc: &LoroDoc, title: &str) -> io::Result<()> { + DocumentPackage::from_loro_snapshot(doc, title)?.write(path) +} + +pub fn loro_db8_bytes(doc: &LoroDoc, title: &str) -> io::Result> { + DocumentPackage::from_loro_snapshot(doc, title)?.to_bytes() +} + +/// Parse the package header + chunk table and return `(kind, payload)` SLICES +/// into `bytes` for every chunk `keep` selects, verifying the table checksum +/// of each kept chunk. Bounds are validated for EVERY chunk (kept or not); +/// checksums are only computed for kept ones — a preview that needs two small +/// chunks must not BLAKE3 an entire multi-hundred-MB package. Zero payload +/// copies happen here; callers decode straight from the returned slices. +fn read_chunk_slices(bytes: &[u8], keep: impl FnMut(u32) -> bool) -> io::Result> { + Ok( + read_chunk_slices_deferring(bytes, keep, |_| false)? + .into_iter() + .map(|(kind, payload, _)| (kind, payload)) + .collect(), + ) +} + +/// §A12.1.3 slice 2b: like [`read_chunk_slices`], but chunk kinds selected by +/// `defer_hash` skip the BLAKE3 verification here and return their TABLE +/// checksum instead — the consumer that actually decodes those bytes must +/// verify first ([`DocumentPackage::verify_snapshot_chunk`]). This moves the +/// dominant open cost (hashing the multi-MB full snapshot chunk) off the +/// open path and onto its rare consumers (sidecar/full loads). +/// One chunk from [`read_chunk_slices_deferring`]: kind, payload slice, and — +/// for hash-deferred kinds — the unverified chunk-table checksum. +type DeferredChunkSlice<'bytes> = (u32, &'bytes [u8], Option<[u8; 32]>); + +fn read_chunk_slices_deferring( + bytes: &[u8], + mut keep: impl FnMut(u32) -> bool, + defer_hash: impl Fn(u32) -> bool, +) -> io::Result>> { + let mut cursor = Cursor::new(bytes); + let mut magic = [0_u8; 16]; + cursor.read_exact(&mut magic)?; + if &magic != PACKAGE_MAGIC { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid Flowstate Loro package magic")); + } + let version = read_u32(&mut cursor)?; + if version != PACKAGE_HEADER_VERSION { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported Flowstate package header version")); + } + let chunk_count = read_u32(&mut cursor)?; + if usize::try_from(chunk_count).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "package chunk count overflows usize"))? + > MAX_PACKAGE_CHUNK_COUNT + { + return Err(io::Error::new(io::ErrorKind::InvalidData, "too many package chunks")); + } + let mut entries = Vec::with_capacity(chunk_count as usize); + for _ in 0..chunk_count { + let kind = read_u32(&mut cursor)?; + let offset = read_u64(&mut cursor)?; + let len = read_u64(&mut cursor)?; + let mut checksum = [0_u8; 32]; + cursor.read_exact(&mut checksum)?; + entries.push(ChunkEntry { kind, offset, len, checksum }); + } + let mut chunks = Vec::with_capacity(entries.len()); + for entry in entries { + let start = usize::try_from(entry.offset).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "chunk offset overflows usize"))?; + let len = usize::try_from(entry.len).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "chunk length overflows usize"))?; + if len > MAX_PACKAGE_CHUNK_COUNT.saturating_mul(PACKAGE_CHUNK_TABLE_ENTRY_BYTES) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "package chunk length is unreasonably large")); + } + let end = start + .checked_add(len) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "chunk range overflows usize"))?; + if end > bytes.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Flowstate package chunk is truncated")); + } + if !keep(entry.kind) { + continue; + } + let payload = &bytes[start..end]; + if defer_hash(entry.kind) { + chunks.push((entry.kind, payload, Some(entry.checksum))); + continue; + } + if blake3_hash(payload) != entry.checksum { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Flowstate package chunk checksum mismatch")); + } + chunks.push((entry.kind, payload, None)); + } + Ok(chunks) +} + +fn cached_search_units_from_compact_bytes(bytes: &[u8]) -> io::Result<(DocumentPackageManifest, Vec)> { + // Selective read: only the manifest + search-unit chunks are decoded (and + // checksum-verified); snapshots/segments/assets are neither hashed nor + // copied — this consumer only wants the search cache. + let chunks = read_chunk_slices(bytes, |kind| matches!(kind, CHUNK_MANIFEST | CHUNK_SEARCH_UNIT))?; + let mut manifest = None; + let mut search_units = Vec::new(); + for (kind, payload) in chunks { + match kind { + CHUNK_MANIFEST => manifest = Some(decode_chunk(payload, "manifest")?), + CHUNK_SEARCH_UNIT => search_units.push(decode_chunk(payload, "search unit")?), + _ => {}, + } + } + let manifest = manifest.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package has no manifest"))?; + Ok((manifest, search_units)) +} + +/// The preview triple: manifest + projection caches + assets. +type CachedProjectionParts = (DocumentPackageManifest, Vec, Vec); + +/// Preview fast path (§P3): decode ONLY the manifest, the frontier-matched +/// projection cache, and the asset chunks from raw package bytes — skipping +/// snapshot/segment decode, their BLAKE3 sweeps, and `validate()` entirely. +/// Returns `Ok(None)` when no frontier-current projection cache exists (the +/// caller falls back to the full package read + Loro materialization). +fn cached_projection_from_compact_bytes(bytes: &[u8]) -> io::Result> { + let chunks = read_chunk_slices(bytes, |kind| matches!(kind, CHUNK_MANIFEST | CHUNK_PROJECTION_CACHE | CHUNK_ASSET))?; + let mut manifest: Option = None; + let mut caches = Vec::new(); + let mut assets = Vec::new(); + for (kind, payload) in chunks { + match kind { + CHUNK_MANIFEST => manifest = Some(decode_chunk(payload, "manifest")?), + CHUNK_PROJECTION_CACHE => caches.push(decode_chunk(payload, "projection cache")?), + CHUNK_ASSET => assets.push(decode_chunk(payload, "asset")?), + _ => {}, + } + } + let manifest = manifest.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package has no manifest"))?; + Ok(Some((manifest, caches, assets))) +} + +/// §act-four M2: build a trimmed [`ProjectionBlocks`] from the leading +/// `k` blocks of a live projection — `O(k)`, the header's payload. +/// §A13.4.1: the FULL `DocumentProjection → ProjectionBlocks` converter — +/// the checkpoint's cache payload built from the projection the runtime +/// already maintains, instead of a whole-doc `document_from_loro` walk. +/// Paragraph text comes through the ranged reader (projection runs hold +/// lengths only); object blocks self-carry content; `sections` copy +/// wholesale (canonical page attrs ride the maintained projection). +fn projection_blocks_from_document(document: &crate::DocumentProjection) -> crate::loro_projection::ProjectionBlocks { + let block_count = document.blocks.len(); + let mut blocks = Vec::with_capacity(block_count); + let mut paragraph_ix = 0usize; + for block in document.blocks.range(0..block_count) { + match block { + crate::Block::Paragraph(_) => { + blocks.push(crate::InputBlock::Paragraph(crate::input_paragraph_from_document_range( + document, + paragraph_ix, + 0..usize::MAX, + ))); + paragraph_ix += 1; + }, + other => blocks.push(crate::input_block_from_block(other)), + } + } + crate::loro_projection::ProjectionBlocks { + document_id: document.ids.document_id, + blocks, + paragraph_ids: document.ids.paragraph_ids.as_ref().clone(), + block_ids: document.ids.block_ids.as_ref().clone(), + sections: document.sections.as_ref().clone(), + } +} + +fn preview_header_blocks_from_document(document: &crate::DocumentProjection, k: usize) -> crate::loro_projection::ProjectionBlocks { + let block_count = document.blocks.len().min(k); + // Paragraph blocks must carry their TEXT (a `Block::Paragraph`'s runs hold + // only lengths — the text lives in the document rope), so read each paragraph + // through the ranged reader; object blocks carry their content directly. + let mut blocks = Vec::with_capacity(block_count); + let mut paragraph_ix = 0usize; + for block in document.blocks.range(0..block_count) { + match block { + crate::Block::Paragraph(_) => { + blocks.push(crate::InputBlock::Paragraph(crate::input_paragraph_from_document_range( + document, + paragraph_ix, + 0..usize::MAX, + ))); + paragraph_ix += 1; + }, + other => blocks.push(crate::input_block_from_block(other)), + } + } + crate::loro_projection::ProjectionBlocks { + document_id: document.ids.document_id, + blocks, + paragraph_ids: document.ids.paragraph_ids[..paragraph_ix.min(document.ids.paragraph_ids.len())].to_vec(), + block_ids: document.ids.block_ids[..block_count.min(document.ids.block_ids.len())].to_vec(), + sections: Vec::new(), + } +} + +/// The preview-header pair: manifest + preview header chunks. +type PreviewHeaderParts = (DocumentPackageManifest, Vec); + +/// §act-four M2: decode ONLY the manifest + preview-header chunk(s) from raw +/// compact bytes — skipping every heavy chunk and the `validate()` sweep. +/// §act-ten A10.13: positioned preview-header extraction — never materializes +/// the whole file. Journal: walk transaction headers by seek (magic + len + +/// checksum reads only), sniff each payload's leading magic, then read the +/// last full generation SELECTIVELY (chunk table + manifest/preview chunks) +/// followed by the small delta payloads after it. Compact: selective chunks at +/// offset 0. Chunk checksums still verify everything actually read; the +/// journal transaction checksum (whole-payload) is NOT re-verified here — the +/// COMMIT marker gates torn tails, and preview is a fallback-tolerant read. +fn preview_header_positioned(path: &Path) -> io::Result> { + use std::io::{Read as _, Seek as _, SeekFrom}; + let mut file = fs::File::open(path)?; + let file_len = file.metadata()?.len(); + let mut magic = [0_u8; 16]; + file.read_exact(&mut magic)?; + if &magic == PACKAGE_MAGIC { + return preview_header_from_compact_positioned(&mut file, 0, file_len); + } + if &magic != JOURNAL_MAGIC { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid Flowstate package journal header")); + } + let mut version = [0_u8; 4]; + file.read_exact(&mut version)?; + if u32::from_le_bytes(version) != JOURNAL_HEADER_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported Flowstate package journal version", + )); + } + // Walk committed transactions: record (payload_offset, payload_len, is_generation). + let mut offset = (JOURNAL_MAGIC.len() + 4) as u64; + let mut committed: Vec<(u64, u64, bool)> = Vec::new(); + let fixed_len = (JOURNAL_TXN_MAGIC.len() + 8 + 32) as u64; + loop { + if file_len.saturating_sub(offset) < fixed_len { + break; + } + file.seek(SeekFrom::Start(offset))?; + let mut txn_magic = [0_u8; 8]; + file.read_exact(&mut txn_magic)?; + if &txn_magic != JOURNAL_TXN_MAGIC { + break; + } + let mut len_bytes = [0_u8; 8]; + file.read_exact(&mut len_bytes)?; + let payload_len = u64::from_le_bytes(len_bytes); + let payload_offset = offset + fixed_len; + let commit_end = match payload_offset + .checked_add(payload_len) + .and_then(|end| end.checked_add(JOURNAL_COMMIT_MAGIC.len() as u64)) + { + Some(end) if end <= file_len => end, + _ => break, + }; + file.seek(SeekFrom::Start(commit_end - JOURNAL_COMMIT_MAGIC.len() as u64))?; + let mut commit_magic = [0_u8; 8]; + file.read_exact(&mut commit_magic)?; + if &commit_magic != JOURNAL_COMMIT_MAGIC { + break; + } + // Sniff the payload's leading magic to classify generation vs delta. + file.seek(SeekFrom::Start(payload_offset))?; + let mut payload_magic = [0_u8; 16]; + let is_generation = payload_len >= 16 && { + file.read_exact(&mut payload_magic)?; + &payload_magic == PACKAGE_MAGIC + }; + committed.push((payload_offset, payload_len, is_generation)); + offset = commit_end; + } + let latest_generation = committed + .iter() + .rposition(|(_, _, is_generation)| *is_generation) + .unwrap_or(0); + let mut extracted: Option = None; + for &(payload_offset, payload_len, is_generation) in &committed[latest_generation..] { + if is_generation { + extracted = preview_header_from_compact_positioned(&mut file, payload_offset, payload_len)?; + continue; + } + // Delta payloads are the small per-edit appends — read them whole. + file.seek(SeekFrom::Start(payload_offset))?; + let mut payload = vec![ + 0_u8; + usize::try_from(payload_len) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "journal payload length overflows usize"))? + ]; + file.read_exact(&mut payload)?; + let delta = decode_journal_delta(&payload)?; + let Some((manifest, headers)) = extracted.as_mut() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Flowstate package journal delta precedes a full generation", + )); + }; + if let PackageJournalDelta::Update { + manifest: next_manifest, + preview_header, + .. + } = delta + { + *manifest = next_manifest; + if let Some(header) = preview_header { + *headers = vec![header]; + } + } + } + Ok(extracted) +} + +/// Selective chunk read of a compact package image at `base..base+len` inside +/// `file`: chunk table first, then pread of ONLY the manifest + preview-header +/// chunks (checksum-verified). +fn preview_header_from_compact_positioned(file: &mut fs::File, base: u64, region_len: u64) -> io::Result> { + use std::io::{Read as _, Seek as _, SeekFrom}; + file.seek(SeekFrom::Start(base))?; + let mut magic = [0_u8; 16]; + file.read_exact(&mut magic)?; + if &magic != PACKAGE_MAGIC { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid Flowstate Loro package magic")); + } + let mut word = [0_u8; 4]; + file.read_exact(&mut word)?; + if u32::from_le_bytes(word) != PACKAGE_HEADER_VERSION { + return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported Flowstate package header version")); + } + file.read_exact(&mut word)?; + let chunk_count = u32::from_le_bytes(word); + if chunk_count as usize > MAX_PACKAGE_CHUNK_COUNT { + return Err(io::Error::new(io::ErrorKind::InvalidData, "too many package chunks")); + } + let mut table = vec![0_u8; chunk_count as usize * PACKAGE_CHUNK_TABLE_ENTRY_BYTES]; + file.read_exact(&mut table)?; + let mut manifest: Option = None; + let mut headers = Vec::new(); + for entry in table.chunks_exact(PACKAGE_CHUNK_TABLE_ENTRY_BYTES) { + let kind = u32::from_le_bytes(entry[0..4].try_into().expect("sized")); + if !matches!(kind, CHUNK_MANIFEST | CHUNK_PREVIEW_HEADER) { + continue; + } + let chunk_offset = u64::from_le_bytes(entry[4..12].try_into().expect("sized")); + let chunk_len = u64::from_le_bytes(entry[12..20].try_into().expect("sized")); + let checksum: [u8; 32] = entry[20..52].try_into().expect("sized"); + if chunk_len as usize > MAX_PACKAGE_CHUNK_COUNT.saturating_mul(PACKAGE_CHUNK_TABLE_ENTRY_BYTES) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "package chunk length is unreasonably large")); + } + let end = chunk_offset + .checked_add(chunk_len) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "chunk range overflows"))?; + if end > region_len { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "Flowstate package chunk is truncated")); + } + file.seek(SeekFrom::Start(base + chunk_offset))?; + let mut payload = + vec![0_u8; usize::try_from(chunk_len).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "chunk length overflows usize"))?]; + file.read_exact(&mut payload)?; + if blake3_hash(&payload) != checksum { + return Err(io::Error::new(io::ErrorKind::InvalidData, "Flowstate package chunk checksum mismatch")); + } + match kind { + CHUNK_MANIFEST => manifest = Some(decode_chunk(&payload, "manifest")?), + CHUNK_PREVIEW_HEADER => headers.push(decode_chunk(&payload, "preview header")?), + _ => {}, + } + } + let manifest = manifest.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package has no manifest"))?; + Ok(Some((manifest, headers))) +} + +fn preview_header_from_compact_bytes(bytes: &[u8]) -> io::Result> { + let chunks = read_chunk_slices(bytes, |kind| matches!(kind, CHUNK_MANIFEST | CHUNK_PREVIEW_HEADER))?; + let mut manifest: Option = None; + let mut headers = Vec::new(); + for (kind, payload) in chunks { + match kind { + CHUNK_MANIFEST => manifest = Some(decode_chunk(payload, "manifest")?), + CHUNK_PREVIEW_HEADER => headers.push(decode_chunk(payload, "preview header")?), + _ => {}, + } + } + let manifest = manifest.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package has no manifest"))?; + Ok(Some((manifest, headers))) +} + +/// Journal-aware sibling: decode the last full generation's header, then fold +/// each trailing `Update` delta's fresh header (an edited doc's latest header +/// rides its journal delta). +fn preview_header_from_journal_bytes(bytes: &[u8]) -> io::Result> { + let payloads = committed_journal_payloads(bytes)?; + let latest_generation = payloads + .iter() + .rposition(|payload| payload.starts_with(PACKAGE_MAGIC)) + .unwrap_or(0); + let mut extracted: Option = None; + for payload in &payloads[latest_generation..] { + let payload = *payload; + if payload.starts_with(PACKAGE_MAGIC) { + extracted = preview_header_from_compact_bytes(payload)?; + continue; + } + let delta = decode_journal_delta(payload)?; + let Some((manifest, headers)) = extracted.as_mut() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Flowstate package journal delta precedes a full generation", + )); + }; + if let PackageJournalDelta::Update { + manifest: next_manifest, + preview_header, + .. + } = delta + { + *manifest = next_manifest; + if let Some(header) = preview_header { + *headers = vec![header]; + } + } + } + Ok(extracted) +} + +fn cached_search_units_from_journal_bytes(bytes: &[u8]) -> io::Result<(DocumentPackageManifest, Vec)> { + let payloads = committed_journal_payloads(bytes)?; + // Decode from the LAST full generation only — everything earlier is + // superseded (same law as `from_journal_bytes`); decoding every stacked + // generation re-parsed the whole journal history. + let latest_generation = payloads + .iter() + .rposition(|payload| payload.starts_with(PACKAGE_MAGIC)) + .unwrap_or(0); + let mut cached = None; + for payload in &payloads[latest_generation..] { + let payload = *payload; + if payload.starts_with(PACKAGE_MAGIC) { + cached = Some(cached_search_units_from_compact_bytes(payload)?); + continue; + } + let delta = decode_journal_delta(payload)?; + let Some((manifest, search_units)) = cached.as_mut() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Flowstate package journal delta precedes a full generation", + )); + }; + match delta { + PackageJournalDelta::Update { manifest: next_manifest, .. } | PackageJournalDelta::Assets { manifest: next_manifest, .. } => { + *manifest = next_manifest; + if manifest.search_cache_frontier.is_none() { + search_units.clear(); + } + }, + } + } + cached.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate package journal has no complete full generation")) +} + +/// Journal-aware sibling of [`cached_projection_from_compact_bytes`]: decode +/// the last full generation's manifest/projection-cache/assets, then fold the +/// trailing deltas' manifests (an Update after the cached frontier makes the +/// cache stale, which the caller detects via the frontier mismatch). +fn cached_projection_from_journal_bytes(bytes: &[u8]) -> io::Result> { + let payloads = committed_journal_payloads(bytes)?; + let latest_generation = payloads + .iter() + .rposition(|payload| payload.starts_with(PACKAGE_MAGIC)) + .unwrap_or(0); + let mut cached = None; + for payload in &payloads[latest_generation..] { + let payload = *payload; + if payload.starts_with(PACKAGE_MAGIC) { + cached = cached_projection_from_compact_bytes(payload)?; + continue; + } + let delta = decode_journal_delta(payload)?; + let Some((manifest, caches, assets)) = cached.as_mut() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Flowstate package journal delta precedes a full generation", + )); + }; + match delta { + PackageJournalDelta::Update { manifest: next_manifest, .. } => { + *manifest = next_manifest; + if manifest.projection_cache_frontier.is_none() { + caches.clear(); + } + }, + PackageJournalDelta::Assets { + manifest: next_manifest, + assets: next_assets, + } => { + *manifest = next_manifest; + *assets = next_assets; + }, + } + } + Ok(cached) +} + +fn write_chunks(chunks: &[Chunk]) -> io::Result> { + let table_len = chunks.len() * (4 + 8 + 8 + 32); + let header_len = PACKAGE_MAGIC.len() + 4 + 4 + table_len; + let payload_len = chunks + .iter() + .map(|chunk| chunk.payload.len()) + .sum::(); + let mut bytes = Vec::with_capacity(header_len + payload_len); + bytes.extend_from_slice(PACKAGE_MAGIC); + write_u32(&mut bytes, PACKAGE_HEADER_VERSION); + write_u32( + &mut bytes, + u32::try_from(chunks.len()).map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "too many package chunks"))?, + ); + let mut offset = header_len; + for chunk in chunks { + write_u32(&mut bytes, chunk.kind); + write_u64(&mut bytes, offset as u64); + write_u64(&mut bytes, chunk.payload.len() as u64); + bytes.extend_from_slice(&blake3_hash(&chunk.payload)); + offset += chunk.payload.len(); + } + for chunk in chunks { + bytes.extend_from_slice(&chunk.payload); + } + Ok(bytes) +} + +fn encode_chunk(value: &T, label: &'static str) -> io::Result> { + postcard::to_stdvec(value).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, format!("encoding {label} failed: {error}"))) +} + +fn decode_chunk<'a, T: Deserialize<'a>>(bytes: &'a [u8], label: &'static str) -> io::Result { + if bytes.len() > MAX_PACKAGE_CHUNK_COUNT.saturating_mul(PACKAGE_CHUNK_TABLE_ENTRY_BYTES) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("decoding {label} failed: chunk too large"), + )); + } + postcard::from_bytes(bytes).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, format!("decoding {label} failed: {error}"))) +} + +fn encode_frontiers(frontiers: &Frontiers) -> Vec { + frontiers.encode() +} + +fn decode_frontiers(bytes: &[u8]) -> io::Result { + Frontiers::decode(bytes).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, format!("decoding frontiers failed: {error}"))) +} + +fn encode_version_vector(version_vector: &VersionVector) -> Vec { + version_vector.encode() +} + +/// Resolve a frontier that must lie within `doc`'s history into its version +/// vector directly on the live doc, replacing the `fork_at` (O(doc)) pattern +/// that previously existed only to read the fork's `state_vv`. +fn frontiers_version_vector(doc: &LoroDoc, frontiers: &Frontiers) -> io::Result { + doc.frontiers_to_vv(frontiers).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "revision frontier is not contained in the Loro document history", + ) + }) +} + +fn validate_frontiers(bytes: &[u8], label: &'static str) -> io::Result<()> { + Frontiers::decode(bytes) + .map(|_| ()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, format!("decoding {label} failed: {error}"))) +} + +fn validate_version_vector(bytes: &[u8], label: &'static str) -> io::Result<()> { + VersionVector::decode(bytes) + .map(|_| ()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, format!("decoding {label} failed: {error}"))) +} + +fn blake3_hash(bytes: &[u8]) -> [u8; 32] { + // §A13.1.2: multithread the hash for multi-MB payloads (3-6GB/s vs ~1); + // below the threshold the rayon fan-out costs more than it saves. + const RAYON_HASH_MIN_BYTES: usize = 1024 * 1024; + if bytes.len() >= RAYON_HASH_MIN_BYTES { + let mut hasher = blake3::Hasher::new(); + hasher.update_rayon(bytes); + return *hasher.finalize().as_bytes(); + } + *blake3::hash(bytes).as_bytes() +} + +/// Compact `B:` tag for an encoded frontier / version vector. +/// Used only inside lazy fidelity detail closures, so it never runs unless +/// tracing is enabled; it truncates to keep firehose lines short. +fn frontier_tag(bytes: &[u8]) -> String { + use std::fmt::Write as _; + let mut hex = String::new(); + for byte in bytes.iter().take(12) { + let _ = write!(hex, "{byte:02x}"); + } + if bytes.len() > 12 { + hex.push('~'); + } + format!("{}B:{hex}", bytes.len()) +} + +fn loro_io_error(error: impl std::error::Error + Send + Sync + 'static) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error) +} + +fn unix_time_secs() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| i64::try_from(duration.as_secs()).unwrap_or(i64::MAX)) +} + +fn read_u32(cursor: &mut Cursor<&[u8]>) -> io::Result { + let mut bytes = [0; 4]; + cursor.read_exact(&mut bytes)?; + Ok(u32::from_le_bytes(bytes)) +} + +fn write_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_le_bytes()); +} + +fn read_u64(cursor: &mut Cursor<&[u8]>) -> io::Result { + let mut bytes = [0; 8]; + cursor.read_exact(&mut bytes)?; + Ok(u64::from_le_bytes(bytes)) +} + +fn write_u64(bytes: &mut Vec, value: u64) { + bytes.extend_from_slice(&value.to_le_bytes()); +} + +/// Extensions that name a foreign *source / export* document format — never a +/// Flowstate package. Writing package/journal bytes onto one of these would +/// clobber a user's source file (the 2026-07-08 docx-overwrite data-loss bug, +/// where a phase-V open leaked the source `.docx` path into the autosave +/// target). The package writers below refuse it, turning any future regression +/// of that class into a safe error instead of silent data loss. +/// +/// Export to `.docx`/`.pdf` is UNAFFECTED: it goes through the docx/pdf writers +/// (`write_docx` / `write_pdf_with_db8_bytes`) and the app's own +/// `write_bytes_to_path`, none of which call these package methods. +fn reject_foreign_package_path(path: &Path) -> io::Result<()> { + if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) { + let extension = extension.to_ascii_lowercase(); + if matches!(extension.as_str(), "docx" | "doc" | "docm" | "dotx" | "pdf" | "rtf" | "odt") { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "refusing to write a Flowstate package to a foreign document path (would overwrite the source): {}", + path.display() + ), + )); + } + } + Ok(()) +} + +fn file_has_journal_header(path: &Path) -> io::Result { + let mut file = match fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error), + }; + let mut magic = [0_u8; 16]; + match file.read_exact(&mut magic) { + Ok(()) => Ok(&magic == JOURNAL_MAGIC), + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => Ok(false), + Err(error) => Err(error), + } +} + +fn append_journal_transaction(path: &Path, payload: &[u8]) -> io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + let existing = fs::read(path)?; + let (_, committed_end) = committed_journal_transactions(&existing)?; + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + if committed_end != existing.len() { + file.set_len(committed_end as u64)?; + } + file.seek(SeekFrom::End(0))?; + let mut bytes = Vec::with_capacity(journal_transaction_len(payload.len())); + append_journal_transaction_bytes(&mut bytes, payload); + file.write_all(&bytes)?; + file.sync_all() +} + +fn append_journal_transaction_to_prepared_file_with_durability(path: &Path, payload: &[u8], sync: bool) -> io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + let mut file = OpenOptions::new().append(true).open(path)?; + let mut bytes = Vec::with_capacity(journal_transaction_len(payload.len())); + append_journal_transaction_bytes(&mut bytes, payload); + file.write_all(&bytes)?; + if sync { + file.sync_all()?; + } + Ok(()) +} + +fn append_journal_transaction_bytes(bytes: &mut Vec, payload: &[u8]) { + bytes.extend_from_slice(JOURNAL_TXN_MAGIC); + write_u64(bytes, payload.len() as u64); + bytes.extend_from_slice(&blake3_hash(payload)); + bytes.extend_from_slice(payload); + bytes.extend_from_slice(JOURNAL_COMMIT_MAGIC); +} + +fn journal_transaction_len(payload_len: usize) -> usize { + JOURNAL_TXN_MAGIC.len() + 8 + 32 + payload_len + JOURNAL_COMMIT_MAGIC.len() +} + +fn committed_journal_payloads(bytes: &[u8]) -> io::Result> { + committed_journal_transactions(bytes).map(|(payloads, _)| payloads) +} + +fn committed_journal_transactions(bytes: &[u8]) -> io::Result<(Vec<&[u8]>, usize)> { + if bytes.len() < JOURNAL_MAGIC.len() + 4 || !bytes.starts_with(JOURNAL_MAGIC) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid Flowstate package journal header")); + } + let mut cursor = Cursor::new(&bytes[JOURNAL_MAGIC.len()..]); + let version = read_u32(&mut cursor)?; + if version != JOURNAL_HEADER_VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unsupported Flowstate package journal version", + )); + } + let mut offset = JOURNAL_MAGIC.len() + 4; + let mut committed = Vec::new(); + while offset < bytes.len() { + let fixed_len = JOURNAL_TXN_MAGIC.len() + 8 + 32; + if bytes.len().saturating_sub(offset) < fixed_len { + break; + } + if &bytes[offset..offset + JOURNAL_TXN_MAGIC.len()] != JOURNAL_TXN_MAGIC { + break; + } + offset += JOURNAL_TXN_MAGIC.len(); + let payload_len = u64::from_le_bytes( + bytes[offset..offset + 8] + .try_into() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid Flowstate journal transaction length"))?, + ); + offset += 8; + let checksum: [u8; 32] = bytes[offset..offset + 32] + .try_into() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid Flowstate journal transaction checksum"))?; + offset += 32; + let payload_len = usize::try_from(payload_len) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "Flowstate journal transaction length overflows usize"))?; + let payload_end = match offset.checked_add(payload_len) { + Some(end) => end, + None => break, + }; + let commit_end = match payload_end.checked_add(JOURNAL_COMMIT_MAGIC.len()) { + Some(end) => end, + None => break, + }; + if commit_end > bytes.len() { + break; + } + if &bytes[payload_end..commit_end] != JOURNAL_COMMIT_MAGIC { + break; + } + let payload = &bytes[offset..payload_end]; + // §A13.1: a full-generation payload (starts with PACKAGE_MAGIC) skips + // the whole-payload hash — its integrity is layered inside it (chunk + // table BLAKE3 per chunk, deferred kinds verify-at-decode), and a torn + // append is caught structurally (length bound + commit magic above). + // Delta payloads stay hashed: they are small and carry no internal + // checksums. `FLOWSTATE_EAGER_JOURNAL_HASH` restores full hashing. + static EAGER_HASH: std::sync::OnceLock = std::sync::OnceLock::new(); + let eager = *EAGER_HASH.get_or_init(|| std::env::var_os("FLOWSTATE_EAGER_JOURNAL_HASH").is_some()); + if (eager || !payload.starts_with(PACKAGE_MAGIC)) && blake3_hash(payload) != checksum { + break; + } + committed.push(payload); + offset = commit_end; + } + if committed.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Flowstate package journal has no complete transaction", + )); + } + Ok((committed, offset)) +} + +fn write_journal_generation(path: &Path, payload: &[u8]) -> io::Result<()> { + let mut bytes = Vec::with_capacity(JOURNAL_MAGIC.len() + 4 + journal_transaction_len(payload.len())); + bytes.extend_from_slice(JOURNAL_MAGIC); + write_u32(&mut bytes, JOURNAL_HEADER_VERSION); + append_journal_transaction_bytes(&mut bytes, payload); + write_bytes_atomic(path, &bytes) +} + +fn encode_journal_delta(delta: &PackageJournalDelta) -> io::Result> { + let encoded = encode_chunk(delta, "package journal delta")?; + let mut payload = Vec::with_capacity(JOURNAL_DELTA_MAGIC.len() + encoded.len()); + payload.extend_from_slice(JOURNAL_DELTA_MAGIC); + payload.extend_from_slice(&encoded); + Ok(payload) +} + +fn decode_journal_delta(payload: &[u8]) -> io::Result { + let encoded = payload + .strip_prefix(JOURNAL_DELTA_MAGIC) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "unknown Flowstate package journal transaction"))?; + decode_chunk(encoded, "package journal delta") +} + +fn package_map_string(map: &loro::LoroMap, key: &str) -> Option { + match map.get(key)? { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + } +} + +fn loro_revision_exists(doc: &LoroDoc, revision_id: u128) -> bool { + let root = doc.get_map(crate::loro_schema::ROOT); + let Some(ValueOrContainer::Container(Container::List(revisions))) = root.get(crate::loro_schema::REVISIONS) else { + return false; + }; + for index in 0..revisions.len() { + let Some(ValueOrContainer::Container(Container::Map(revision))) = revisions.get(index) else { + continue; + }; + if package_map_string(&revision, "id").and_then(|id| id.parse::().ok()) == Some(revision_id) { + return true; + } + } + false +} + +fn package_map_binary(map: &loro::LoroMap, key: &str) -> Option> { + match map.get(key)? { + ValueOrContainer::Value(LoroValue::Binary(value)) => Some(value.to_vec()), + _ => None, + } +} + +fn package_map_i64(map: &loro::LoroMap, key: &str) -> Option { + match map.get(key)? { + ValueOrContainer::Value(LoroValue::I64(value)) => Some(value), + _ => None, + } +} + +fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> io::Result<()> { + let parent = path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + fs::create_dir_all(parent)?; + atomicwrites::AtomicFile::new(path, atomicwrites::AllowOverwrite) + .write(|file| file.write_all(bytes)) + .map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + use crate::{ + AssetId, AssetRecord, Block, CellId, ColumnId, InputBlock, InputBlockAlignment, InputEquationBlock, InputEquationDisplay, + InputEquationSyntax, InputImageBlock, InputImageSizing, InputParagraph, InputRun, InputTableBlock, InputTableCell, InputTableCellBlock, + InputTableColumn, InputTableColumnWidth, InputTableRow, InputTableStyle, RowId, RunStyles, TableCellBlock, document_from_loro, + document_to_loro, + loro_schema::{body_text, new_loro_document}, + read_db8_bytes, + }; + use loro::{Container, LoroDoc, LoroMap, LoroValue, ValueOrContainer}; + + #[test] + fn package_roundtrips_loro_snapshot() -> io::Result<()> { + let doc = new_loro_document("Roundtrip").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "Hello Loro") + .map_err(loro_test_error)?; + let bytes = loro_db8_bytes(&doc, "Roundtrip")?; + + let package = DocumentPackage::from_bytes(&bytes)?; + assert_eq!(package.manifest.package_format_version, LORO_PACKAGE_FORMAT_VERSION); + assert_eq!(package.manifest.loro_schema_version, LORO_SCHEMA_VERSION); + assert_eq!(package.loro_snapshots.len(), 2); + assert_eq!( + package.manifest.projection_cache_frontier.as_deref(), + Some(package.manifest.latest_frontier.as_slice()) + ); + assert_eq!(package.projection_caches.len(), 1); + + let loaded = package.load_loro_doc()?; + assert_eq!(body_text(&loaded).to_string(), "\nHello Loro"); + let projected = package + .current_projection_document()? + .expect("projection cache"); + assert_eq!(crate::paragraph_text(&projected, 0), "Hello Loro"); + Ok(()) + } + + #[test] + fn package_read_repairs_an_incomplete_journal_tail() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("tail-recovery.db8"); + let doc = new_loro_document("Tail recovery").map_err(loro_test_error)?; + let package = DocumentPackage::from_loro_snapshot(&doc, "Tail recovery")?; + package.write(&path)?; + let committed_len = fs::metadata(&path)?.len(); + + let mut file = OpenOptions::new().append(true).open(&path)?; + std::io::Write::write_all(&mut file, b"incomplete journal transaction")?; + drop(file); + assert!(fs::metadata(&path)?.len() > committed_len); + + let repaired = DocumentPackage::read(&path)?; + + repaired.validate()?; + assert_eq!(fs::metadata(&path)?.len(), committed_len); + Ok(()) + } + + #[test] + fn package_write_refuses_to_clobber_a_foreign_source_document() -> io::Result<()> { + // §data-loss regression (2026-07-08): a phase-V open leaked the source + // `.docx` path into the autosave target, so a checkpoint wrote a `.db8` + // journal ON TOP of the user's original docx. The guard makes every package + // writer refuse a foreign-format path, so this can never silently destroy a + // source file again — regardless of how the path was derived upstream. + let dir = tempfile::tempdir()?; + let doc = new_loro_document("Guard").map_err(loro_test_error)?; + let package = DocumentPackage::from_loro_snapshot(&doc, "Guard")?; + + // A pre-existing "source" docx (arbitrary non-package bytes) must survive. + let docx_path = dir.path().join("source.docx"); + let original = b"PK\x03\x04 pretend this is a real docx zip"; + fs::write(&docx_path, original)?; + + for foreign in ["source.docx", "paper.pdf", "brief.doc", "legacy.rtf", "notes.odt"] { + let path = dir.path().join(foreign); + let error = package + .write(&path) + .expect_err("package write to a foreign path must be rejected"); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput, "{foreign}"); + } + // The append entry points guard identically. + assert_eq!( + package + .append_latest_update_to_path(&docx_path) + .expect_err("append must reject foreign path") + .kind(), + io::ErrorKind::InvalidInput + ); + + // The original docx bytes are untouched — nothing was written. + assert_eq!(fs::read(&docx_path)?, original); + + // The same package writes cleanly to a real `.db8` path. + let db8_path = dir.path().join("source.db8"); + package.write(&db8_path)?; + assert!(file_has_journal_header(&db8_path)?); + Ok(()) + } + + #[test] + fn package_rejects_old_final_state_magic() { + let old_bytes = b"GPTX\x06\0\0\0old-format"; + let error = DocumentPackage::from_bytes(old_bytes).expect_err("old final-state bytes must not load"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn public_db8_reader_rejects_old_final_state_magic() { + let old_bytes = b"GPTX\x06\0\0\0old-format"; + let error = read_db8_bytes(old_bytes).expect_err("old final-state bytes must not load through public facade"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn public_db8_bytes_roundtrip_through_loro_package() -> io::Result<()> { + let source = crate::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![InputBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "Hello package".to_string(), + styles: RunStyles::default(), + }], + })], + ); + let doc = document_to_loro(&source, "Public facade").map_err(loro_test_error)?; + let bytes = + DocumentPackage::from_loro_snapshot_with_assets(&doc, "Public facade", crate::loro_import::assets_from_document(&source))?.to_bytes()?; + let package = DocumentPackage::from_bytes(&bytes)?; + assert_eq!(package.manifest.package_format_version, LORO_PACKAGE_FORMAT_VERSION); + let projected = read_db8_bytes(&bytes)?; + assert_eq!(crate::paragraph_text(&projected, 0), "Hello package"); + Ok(()) + } + + #[test] + fn package_loads_snapshot_plus_update_segment() -> io::Result<()> { + let doc = new_loro_document("Append").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Append")?; + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + + let text = body_text(&doc); + text + .insert(text.len_unicode(), "after save") + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + assert!(package.manifest.projection_cache_frontier.is_none()); + assert!(package.projection_caches.is_empty()); + + let bytes = package.to_bytes()?; + let loaded = DocumentPackage::from_bytes(&bytes)?.load_loro_doc()?; + assert_eq!(body_text(&loaded).to_string(), "\nafter save"); + Ok(()) + } + + #[test] + fn package_rejects_manifest_projection_cache_frontier_without_chunk() -> io::Result<()> { + let doc = new_loro_document("Projection cache").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Projection cache")?; + package.projection_caches.clear(); + + let error = package + .validate() + .expect_err("manifest projection cache frontier must point at a cache chunk"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + Ok(()) + } + + #[test] + fn package_rejects_manifest_update_segment_index_mismatch() -> io::Result<()> { + let doc = new_loro_document("Append").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Append")?; + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + + body_text(&doc) + .insert(1, "update") + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + package.manifest.update_segment_index[0].byte_length = 0; + + let error = package + .validate() + .expect_err("stale manifest segment index must fail validation"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + Ok(()) + } + + #[test] + fn package_rejects_duplicate_integrity_index_entries() -> io::Result<()> { + let doc = new_loro_document("Integrity").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Integrity")?; + package.integrity_index[1].chunk_kind = package.integrity_index[0].chunk_kind; + package.integrity_index[1].id = package.integrity_index[0].id; + + let error = package + .validate() + .expect_err("duplicate integrity entries must fail validation"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + Ok(()) + } + + #[test] + fn package_replace_assets_is_failure_atomic() -> io::Result<()> { + let source = crate::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![InputBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "asset".to_string(), + styles: RunStyles::default(), + }], + })], + ); + let doc = document_to_loro(&source, "Atomic assets").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot_with_assets(&doc, "Atomic assets", crate::loro_import::assets_from_document(&source))?; + let previous_modified_at = package.manifest.modified_at_unix_secs; + package.manifest.package_format_version = 0; + + let error = package + .replace_assets_from_document(&source) + .expect_err("invalid package version must fail validation"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(package.manifest.modified_at_unix_secs, previous_modified_at); + assert_eq!(package.manifest.package_format_version, 0); + Ok(()) + } + + #[test] + fn package_append_update_segment_is_failure_atomic() -> io::Result<()> { + let doc = new_loro_document("Atomic update").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Atomic update")?; + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc).insert(1, "x").map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + let previous_segments = package.loro_update_segments.len(); + let previous_latest_frontier = package.manifest.latest_frontier.clone(); + package.manifest.package_format_version = 0; + + let error = package + .append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update) + .expect_err("invalid package version must fail validation"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(package.loro_update_segments.len(), previous_segments); + assert_eq!(package.manifest.latest_frontier, previous_latest_frontier); + assert_eq!(package.manifest.package_format_version, 0); + Ok(()) + } + + #[test] + fn package_compact_to_snapshot_is_failure_atomic() -> io::Result<()> { + let doc = new_loro_document("Atomic compact").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Atomic compact")?; + let previous_snapshots = package.loro_snapshots.len(); + let previous_update_segments = package.loro_update_segments.len(); + package.manifest.package_format_version = 0; + + let error = package + .compact_to_snapshot(&doc) + .expect_err("invalid package version must fail validation"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + assert_eq!(package.loro_snapshots.len(), previous_snapshots); + assert_eq!(package.loro_update_segments.len(), previous_update_segments); + assert_eq!(package.manifest.package_format_version, 0); + Ok(()) + } + + #[test] + fn package_rejects_manifest_asset_index_mismatch() -> io::Result<()> { + let doc = new_loro_document("Assets").map_err(loro_test_error)?; + let bytes = b"asset bytes".to_vec(); + let asset = AssetChunk { + asset_id: 7, + content_hash: blake3_hash(&bytes), + mime_type: "application/octet-stream".to_string(), + byte_length: bytes.len() as u64, + bytes, + metadata: Vec::new(), + }; + let mut package = DocumentPackage::from_loro_snapshot_with_assets(&doc, "Assets", vec![asset])?; + package.manifest.asset_index[0].checksum = [0; 32]; + + let error = package + .validate() + .expect_err("stale manifest asset index must fail validation"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + Ok(()) + } + + #[test] + fn named_revisions_are_restorable_after_compaction() -> io::Result<()> { + let doc = new_loro_document("Revisions").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Revisions")?; + let first_revision = package.create_named_revision(&doc, "Blank", "Before edits", None, None)?; + + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + let text = body_text(&doc); + text.insert(1, "after").map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + assert!(package.current_search_units().is_empty()); + let second_revision = package.create_named_revision(&doc, "After", "After text insert", None, None)?; + + package.compact_to_snapshot(&doc)?; + + let first_doc = package.load_revision_loro_doc(first_revision)?; + assert_eq!(body_text(&first_doc).to_string(), "\n"); + let second_doc = package.load_revision_loro_doc(second_revision)?; + assert_eq!(body_text(&second_doc).to_string(), "\nafter"); + let latest_doc = package.load_loro_doc()?; + assert_eq!(body_text(&latest_doc).to_string(), "\nafter"); + Ok(()) + } + + #[test] + fn revision_chunk_from_snapshot_at_matches_a_fork_at_reference() -> io::Result<()> { + let doc = new_loro_document("SnapshotAt").map_err(loro_test_error)?; + let text = body_text(&doc); + text.insert(1, "first").map_err(loro_test_error)?; + doc.commit(); + let mut package = DocumentPackage::from_loro_snapshot(&doc, "SnapshotAt")?; + + // Edit past the package's latest frontier so the revision below gets a + // fresh `snapshot_at` chunk instead of reusing the initial snapshot. + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc) + .insert(body_text(&doc).len_unicode(), " second") + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + + let revision_frontiers = doc.state_frontiers(); + let revision_id = package.create_named_revision(&doc, "Mid", "After second insert", None, None)?; + + // Advance the live doc past the revision so the chunk genuinely captures + // an older frontier. + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc) + .insert(body_text(&doc).len_unicode(), " third") + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + + // The `SnapshotAt` chunk must survive package serialization. + let package = DocumentPackage::from_bytes(&package.to_bytes()?)?; + let revision_frontier_bytes = encode_frontiers(&revision_frontiers); + assert!( + package + .snapshot_for_frontier(&revision_frontier_bytes) + .is_some() + ); + + let revision_doc = package.load_revision_loro_doc(revision_id)?; + let reference = doc.fork_at(&revision_frontiers).map_err(loro_test_error)?; + assert_eq!(body_text(&revision_doc).to_string(), body_text(&reference).to_string()); + assert_eq!(body_text(&revision_doc).to_string(), "\nfirst second"); + assert_eq!(revision_doc.state_frontiers(), reference.state_frontiers()); + assert_eq!(revision_doc.state_vv(), reference.state_vv()); + let revision = package + .revisions + .iter() + .find(|revision| revision.revision_id == revision_id) + .expect("named revision entry"); + assert_eq!(revision.version_vector, encode_version_vector(&reference.state_vv())); + Ok(()) + } + + #[test] + fn sync_revisions_from_loro_snapshots_remote_revision_without_forking() -> io::Result<()> { + let doc = new_loro_document("Sync revisions").map_err(loro_test_error)?; + body_text(&doc) + .insert(1, "shared") + .map_err(loro_test_error)?; + doc.commit(); + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Sync revisions")?; + + // Simulate a remote peer that edits and records a revision at its own + // frontier, then arrives over sync as one imported update. + let remote = doc.fork(); + body_text(&remote) + .insert(body_text(&remote).len_unicode(), " remote") + .map_err(loro_test_error)?; + remote.commit(); + let remote_frontiers = remote.state_frontiers(); + let remote_revision_id = 77_u128; + crate::loro_schema::record_revision( + &remote, + remote_revision_id, + encode_frontiers(&remote_frontiers), + "Remote", + "Recorded remotely", + None, + ) + .map_err(loro_test_error)?; + + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + let update = remote + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + doc.import(&update).map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + + assert_eq!(package.sync_revisions_from_loro(&doc)?, 1); + let remote_frontier_bytes = encode_frontiers(&remote_frontiers); + assert!( + package + .snapshot_for_frontier(&remote_frontier_bytes) + .is_some() + ); + + let revision_doc = package.load_revision_loro_doc(remote_revision_id)?; + let reference = doc.fork_at(&remote_frontiers).map_err(loro_test_error)?; + assert_eq!(body_text(&revision_doc).to_string(), body_text(&reference).to_string()); + assert_eq!(body_text(&revision_doc).to_string(), "\nshared remote"); + assert_eq!(revision_doc.state_vv(), reference.state_vv()); + let revision = package + .revisions + .iter() + .find(|revision| revision.revision_id == remote_revision_id) + .expect("synced revision entry"); + assert_eq!(revision.version_vector, encode_version_vector(&reference.state_vv())); + Ok(()) + } + + #[test] + fn package_compacts_update_segments_after_threshold() -> io::Result<()> { + let doc = new_loro_document("Auto compact").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Auto compact")?; + let initial_revision = package.create_named_revision(&doc, "Blank", "Before edits", None, None)?; + + for text in ["one", " two"] { + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc) + .insert(body_text(&doc).len_unicode(), text) + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + } + + // Creating the named revision records it into the Loro `revisions` list and + // persists that op as its own update segment (Loro-native revisions), so the + // two body-text inserts bring the total to three segments. + assert_eq!(package.loro_update_segments.len(), 3); + let snapshot_id = package.compact_update_segments_if_needed(&doc, 1)?; + assert!(snapshot_id.is_some()); + assert!(package.loro_update_segments.is_empty()); + + let latest_doc = package.load_loro_doc()?; + assert_eq!(body_text(&latest_doc).to_string(), "\none two"); + let revision_doc = package.load_revision_loro_doc(initial_revision)?; + assert_eq!(body_text(&revision_doc).to_string(), "\n"); + Ok(()) + } + + #[test] + fn package_rebuilds_search_units_from_loro_body_flow() -> io::Result<()> { + let doc = new_loro_document("Search").map_err(loro_test_error)?; + let text = body_text(&doc); + text.insert(1, "Alpha\nBeta").map_err(loro_test_error)?; + doc.commit(); + + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Search")?; + package.rebuild_search_units_from_loro(&doc)?; + + assert_eq!(package.search_units.len(), 2); + assert_eq!(package.search_units[0].body, "Alpha"); + assert_eq!(package.search_units[1].body, "Beta"); + assert_eq!( + package.manifest.search_cache_frontier.as_deref(), + Some(package.manifest.latest_frontier.as_slice()) + ); + assert!(!package.search_units[0].paragraph_start_cursor.is_empty()); + assert!(!package.search_units[0].paragraph_end_cursor.is_empty()); + Ok(()) + } + + #[test] + fn table_cells_use_column_ids_and_project_by_column_order() -> io::Result<()> { + let source = crate::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![InputBlock::Table(InputTableBlock { + rows: vec![InputTableRow { + id: RowId(1), + cells: vec![ + InputTableCell { + id: CellId::from_coordinate(RowId(1), ColumnId(1)), + row_id: RowId(1), + column_id: ColumnId(1), + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "left".to_string(), + styles: RunStyles::default(), + }], + })], + row_span: 1, + col_span: 1, + }, + InputTableCell { + id: CellId::from_coordinate(RowId(1), ColumnId(2)), + row_id: RowId(1), + column_id: ColumnId(2), + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "right".to_string(), + styles: RunStyles::default(), + }], + })], + row_span: 1, + col_span: 1, + }, + ], + }], + columns: vec![ + InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::Auto, + }, + InputTableColumn { + id: ColumnId(2), + width: InputTableColumnWidth::Auto, + }, + ], + style: InputTableStyle { header_row: false }, + })], + ); + let doc = document_to_loro(&source, "Table schema")?; + let table_owner = first_table_owner(&doc)?; + let table = test_child_map(&table_owner, "table")?; + let column_ids = test_ordered_ids(&table, "column_order")?; + assert_eq!(column_ids.len(), 2); + + let cells_by_id = test_child_map(&table, "cells_by_id")?; + let mut seen_column_ids = Vec::new(); + for cell_id in cells_by_id.keys().map(|key| key.to_string()) { + let cell = test_child_map(&cells_by_id, &cell_id)?; + assert!(cell.get("column_index").is_none()); + seen_column_ids.push(test_map_string(&cell, "column_id")?); + } + assert!(seen_column_ids.contains(&column_ids[0])); + assert!(seen_column_ids.contains(&column_ids[1])); + + let column_order = test_child_movable_list(&table, "column_order")?; + column_order.mov(0, 1).map_err(loro_test_error)?; + doc.commit(); + + let projected = document_from_loro(&doc)?; + let projected_table = projected + .blocks + .iter() + .find_map(|block| match block { + Block::Table(table) => Some(table), + _ => None, + }) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "projected table is missing"))?; + let cell_texts = projected_table.rows[0] + .cells + .iter() + .map(|cell| match &cell.blocks[0] { + TableCellBlock::Paragraph(paragraph) => paragraph.text.as_str(), + TableCellBlock::Table(_) => "", + }) + .collect::>(); + assert_eq!(cell_texts, vec!["right", "left"]); + Ok(()) + } + + #[test] + fn nested_tables_use_stable_list_refs_and_project_by_anchor_cursor() -> io::Result<()> { + let nested_table = InputTableBlock { + // Distinct ids from the outer table: row/column ids are globally unique + // durable ids, so the inner and outer cells must not collide on a shared + // coordinate (which would collide their global cell text flows). + rows: vec![InputTableRow { + id: RowId(2), + cells: vec![InputTableCell { + id: CellId::from_coordinate(RowId(2), ColumnId(2)), + row_id: RowId(2), + column_id: ColumnId(2), + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "inner".to_string(), + styles: RunStyles::default(), + }], + })], + row_span: 1, + col_span: 1, + }], + }], + columns: vec![InputTableColumn { + id: ColumnId(2), + width: InputTableColumnWidth::Auto, + }], + style: InputTableStyle { header_row: false }, + }; + let source = crate::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![InputBlock::Table(InputTableBlock { + rows: vec![InputTableRow { + id: RowId(1), + cells: vec![InputTableCell { + id: CellId::from_coordinate(RowId(1), ColumnId(1)), + row_id: RowId(1), + column_id: ColumnId(1), + blocks: vec![ + InputTableCellBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "outer".to_string(), + styles: RunStyles::default(), + }], + }), + InputTableCellBlock::Table(nested_table), + ], + row_span: 1, + col_span: 1, + }], + }], + columns: vec![InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::Auto, + }], + style: InputTableStyle { header_row: false }, + })], + ); + let doc = document_to_loro(&source, "Nested table")?; + let table = test_child_map(&first_table_owner(&doc)?, "table")?; + let cell = first_cell_map(&table)?; + assert_eq!(test_ordered_ids(&cell, "nested_table_ids")?.len(), 1); + assert!(test_child_map(&cell, "nested_tables_by_id").is_ok()); + assert!(!cell.keys().any(|key| key.starts_with("nested_table."))); + + let projected = document_from_loro(&doc)?; + let projected_table = projected + .blocks + .iter() + .find_map(|block| match block { + Block::Table(table) => Some(table), + _ => None, + }) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "projected table is missing"))?; + let blocks = &projected_table.rows[0].cells[0].blocks; + assert!(matches!(&blocks[0], TableCellBlock::Paragraph(paragraph) if paragraph.text == "outer")); + assert!( + matches!(&blocks[1], TableCellBlock::Table(table) if matches!(&table.rows[0].cells[0].blocks[0], TableCellBlock::Paragraph(paragraph) if paragraph.text == "inner")) + ); + Ok(()) + } + + #[test] + fn package_search_units_come_from_projection_objects_and_tables() -> io::Result<()> { + let source = crate::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![ + InputBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "Body".to_string(), + styles: RunStyles::default(), + }], + }), + InputBlock::Image(InputImageBlock { + asset_id: AssetId(7), + alt_text: "diagram alt".to_string(), + caption: Some(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "caption text".to_string(), + styles: RunStyles::default(), + }], + }), + sizing: InputImageSizing::FitWidth, + alignment: InputBlockAlignment::Left, + external_url: None, + }), + InputBlock::Equation(InputEquationBlock { + source: "x^2".to_string(), + syntax: InputEquationSyntax::Latex, + display: InputEquationDisplay::Display, + }), + InputBlock::Table(InputTableBlock { + rows: vec![InputTableRow { + id: RowId(1), + cells: vec![InputTableCell { + id: CellId::from_coordinate(RowId(1), ColumnId(1)), + row_id: RowId(1), + column_id: ColumnId(1), + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "cell text".to_string(), + styles: RunStyles::default(), + }], + })], + row_span: 1, + col_span: 1, + }], + }], + columns: vec![InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::Auto, + }], + style: InputTableStyle { header_row: false }, + }), + ], + ); + let doc = document_to_loro(&source, "Search projection")?; + let image = first_block_owner_by_kind(&doc, "image")?; + let caption_flow_id = test_map_string(&image, "caption_flow_id")?; + let root = doc.get_map(crate::ROOT); + let flows = test_child_map(&root, crate::FLOWS_BY_ID)?; + let caption_flow = test_child_map(&flows, &caption_flow_id)?; + let caption_text = test_child_text(&caption_flow, crate::FLOW_TEXT_KEY)?; + caption_text + .insert(1, "caption text") + .map_err(loro_test_error)?; + doc.commit(); + let package = DocumentPackage::from_loro_snapshot(&doc, "Search projection")?; + let units = package.current_search_units(); + + assert!( + units + .iter() + .any(|unit| unit.unit_kind == "paragraph" && unit.body == "Body") + ); + assert!( + units + .iter() + .any(|unit| unit.unit_kind == "image_alt" && unit.body == "diagram alt") + ); + assert!( + units + .iter() + .any(|unit| unit.unit_kind == "image_caption" && unit.body == "caption text") + ); + assert!( + units + .iter() + .any(|unit| unit.unit_kind == "equation" && unit.body == "x^2") + ); + assert!( + units + .iter() + .any(|unit| unit.unit_kind == "table_cell" && unit.body == "cell text") + ); + assert!( + units + .iter() + .all(|unit| !unit.body.contains(crate::OBJECT_REPLACEMENT)) + ); + let body_unit = units + .iter() + .find(|unit| unit.unit_kind == "paragraph" && unit.body == "Body") + .expect("body paragraph search unit should exist"); + assert!(!body_unit.paragraph_start_cursor.is_empty()); + assert!(!body_unit.paragraph_end_cursor.is_empty()); + assert_eq!( + package.manifest.search_cache_frontier.as_deref(), + Some(package.manifest.latest_frontier.as_slice()) + ); + Ok(()) + } + + #[test] + fn public_db8_roundtrips_structured_loro_objects_and_assets() -> io::Result<()> { + let asset_id = AssetId(42); + let asset_bytes = b"not really a png".to_vec(); + let mut source = crate::document_from_input_blocks( + crate::flowstate_document_theme(), + vec![ + InputBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "before".to_string(), + styles: RunStyles::default(), + }], + }), + InputBlock::Image(InputImageBlock { + asset_id, + alt_text: "diagram alt".to_string(), + caption: Some(InputParagraph { + style: crate::ParagraphStyle::Custom(1), + runs: vec![InputRun { + text: "caption".to_string(), + styles: RunStyles::default(), + }], + }), + sizing: InputImageSizing::Fixed { + width_px: 320, + height_px: Some(180), + }, + alignment: InputBlockAlignment::Center, + external_url: None, + }), + InputBlock::Equation(InputEquationBlock { + source: "x^2".to_string(), + syntax: InputEquationSyntax::Latex, + display: InputEquationDisplay::InlineLikeParagraph, + }), + InputBlock::Table(InputTableBlock { + rows: vec![InputTableRow { + id: RowId(1), + cells: vec![InputTableCell { + id: CellId::from_coordinate(RowId(1), ColumnId(1)), + row_id: RowId(1), + column_id: ColumnId(1), + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: crate::ParagraphStyle::Normal, + runs: vec![InputRun { + text: "cell".to_string(), + styles: RunStyles::default(), + }], + })], + row_span: 1, + col_span: 1, + }], + }], + columns: vec![InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::FixedPx(144), + }], + style: InputTableStyle { header_row: true }, + }), + ], + ); + source.assets.assets.insert( + asset_id, + AssetRecord { + id: asset_id, + mime_type: "image/png".into(), + original_name: Some("diagram.png".into()), + content_hash: AssetRecord::stable_content_hash(&asset_bytes), + bytes: Arc::new(asset_bytes.clone()), + }, + ); + + let doc = document_to_loro(&source, "Structured").map_err(loro_test_error)?; + let bytes = + DocumentPackage::from_loro_snapshot_with_assets(&doc, "Structured", crate::loro_import::assets_from_document(&source))?.to_bytes()?; + let loaded = read_db8_bytes(&bytes)?; + assert_eq!(crate::paragraph_text(&loaded, 0), "before"); + assert_eq!( + loaded + .assets + .assets + .get(&asset_id) + .map(|asset| asset.bytes.as_ref().clone()), + Some(asset_bytes) + ); + + assert!(matches!( + &loaded.blocks[1], + Block::Image(image) + if image.asset_id == asset_id + && image.alt_text.as_ref() == "diagram alt" + && image.caption.as_ref().is_some_and(|caption| caption.style == crate::ParagraphStyle::Custom(1)) + )); + assert!(matches!( + &loaded.blocks[2], + Block::Equation(equation) + if equation.source.as_ref() == "x^2" && equation.display == crate::EquationDisplay::InlineLikeParagraph + )); + assert!(matches!( + &loaded.blocks[3], + Block::Table(table) + if table.style.header_row + && matches!( + table.columns.as_slice(), + [crate::TableColumn { + width: crate::TableColumnWidth::FixedPx(144), + .. + }] + ) + && matches!(&table.rows[0].cells[0].blocks[0], TableCellBlock::Paragraph(paragraph) if paragraph.text == "cell") + )); + Ok(()) + } + + #[test] + fn package_serializes_empty_schema_migration_log() -> io::Result<()> { + let doc = new_loro_document("Migrations").map_err(loro_test_error)?; + let bytes = DocumentPackage::from_loro_snapshot(&doc, "Migrations")?.to_bytes()?; + let package = DocumentPackage::from_bytes(&bytes)?; + assert!(package.manifest.schema_migrations.is_empty()); + Ok(()) + } + + #[test] + fn package_rejects_inconsistent_schema_migration_record() -> io::Result<()> { + let doc = new_loro_document("Migrations").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Migrations")?; + package + .manifest + .schema_migrations + .push(SchemaMigrationRecord { + id: 1, + from_version: 2, + to_version: 1, + applied_at_unix_secs: 0, + description: "backwards".to_string(), + }); + + let error = package + .validate() + .expect_err("a backwards migration record must be rejected"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + Ok(()) + } + + #[test] + fn package_integrity_index_covers_durable_chunks() -> io::Result<()> { + let doc = new_loro_document("Integrity").map_err(loro_test_error)?; + let bytes = b"asset bytes".to_vec(); + let asset = AssetChunk { + asset_id: 7, + content_hash: blake3_hash(&bytes), + mime_type: "application/octet-stream".to_string(), + byte_length: bytes.len() as u64, + bytes, + metadata: Vec::new(), + }; + let package = DocumentPackage::from_loro_snapshot_with_assets(&doc, "Integrity", vec![asset])?; + + assert!( + package + .integrity_index + .iter() + .any(|entry| entry.chunk_kind == CHUNK_ASSET && entry.id == 7) + ); + assert_eq!( + package + .integrity_index + .iter() + .filter(|entry| entry.chunk_kind == CHUNK_LORO_SNAPSHOT) + .count(), + package.loro_snapshots.len(), + ); + + let roundtrip = DocumentPackage::from_bytes(&package.to_bytes()?)?; + assert_eq!(roundtrip.integrity_index.len(), package.integrity_index.len()); + Ok(()) + } + + #[test] + fn package_rejects_tampered_integrity_index() -> io::Result<()> { + let doc = new_loro_document("Integrity").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Integrity")?; + let entry = package + .integrity_index + .first_mut() + .expect("at least one integrity entry"); + entry.checksum = [0; 32]; + + let error = package + .validate() + .expect_err("a tampered integrity entry must be rejected"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + Ok(()) + } + + fn first_table_owner(doc: &LoroDoc) -> io::Result { + first_block_owner_by_kind(doc, "table") + } + + fn first_block_owner_by_kind(doc: &LoroDoc, kind: &str) -> io::Result { + let root = doc.get_map(crate::ROOT); + let blocks = test_child_map(&root, crate::BLOCKS_BY_ID)?; + for block_id in blocks.keys().map(|key| key.to_string()) { + let block = test_child_map(&blocks, &block_id)?; + if test_map_string_opt(&block, "kind").as_deref() == Some(kind) { + return Ok(block); + } + } + Err(io::Error::new(io::ErrorKind::InvalidData, format!("{kind} block is missing"))) + } + + fn first_cell_map(table: &LoroMap) -> io::Result { + let cells_by_id = test_child_map(table, "cells_by_id")?; + let cell_id = cells_by_id + .keys() + .next() + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "table cell is missing"))? + .to_string(); + test_child_map(&cells_by_id, &cell_id) + } + + fn test_child_map(parent: &LoroMap, key: &str) -> io::Result { + match parent.get(key) { + Some(ValueOrContainer::Container(container)) => container + .into_map() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, format!("`{key}` is not a map"))), + _ => Err(io::Error::new(io::ErrorKind::InvalidData, format!("missing map `{key}`"))), + } + } + + fn test_child_movable_list(parent: &LoroMap, key: &str) -> io::Result { + match parent.get(key) { + Some(ValueOrContainer::Container(Container::MovableList(list))) => Ok(list), + _ => Err(io::Error::new(io::ErrorKind::InvalidData, format!("missing movable list `{key}`"))), + } + } + + fn test_child_text(parent: &LoroMap, key: &str) -> io::Result { + match parent.get(key) { + Some(ValueOrContainer::Container(container)) => container + .into_text() + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, format!("`{key}` is not text"))), + _ => Err(io::Error::new(io::ErrorKind::InvalidData, format!("missing text `{key}`"))), + } + } + + fn test_ordered_ids(parent: &LoroMap, key: &str) -> io::Result> { + Ok( + test_child_movable_list(parent, key)? + .to_vec() + .into_iter() + .filter_map(|value| match value { + LoroValue::String(value) => Some(value.to_string()), + _ => None, + }) + .collect(), + ) + } + + fn test_map_string(map: &LoroMap, key: &str) -> io::Result { + test_map_string_opt(map, key).ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("missing string `{key}`"))) + } + + fn test_map_string_opt(map: &LoroMap, key: &str) -> Option { + map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + }) + } + + fn loro_test_error(error: impl std::error::Error + Send + Sync + 'static) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error) + } + + /// §act-four M2: the preview header materializes the leading blocks and + /// matches the full document there; and — riding the journal delta — it stays + /// fresh for an edited-but-unflushed document, exactly the "preview without a + /// solid full cache" case. Read is O(header): only the manifest + header + /// chunk decode. + #[test] + fn preview_header_matches_leading_blocks_and_stays_fresh_across_edits() -> io::Result<()> { + use crate::{DocumentTheme, ParagraphStyle, document_from_input, import_document_projection}; + + let paras: Vec = (0..60) + .map(|i| InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text: format!("paragraph {i:03} of the document body"), + styles: RunStyles::default(), + }], + }) + .collect(); + let projection = document_from_input(DocumentTheme::default(), paras); + let imported = import_document_projection(projection, "Header Test").map_err(loro_test_error)?; + let doc = imported.doc; + let dir = tempfile::tempdir()?; + let path = dir.path().join("header.db8"); + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Header Test")?; + package.write(&path)?; + + // Checkpoint header == leading blocks of the full materialization. + let header = DocumentPackage::read_preview_header(&path)?.expect("fresh header at checkpoint"); + assert!( + !header.paragraphs.is_empty() && header.paragraphs.len() <= PREVIEW_HEADER_BLOCKS, + "header trims to the leading blocks" + ); + let full = document_from_loro(&doc).map_err(loro_test_error)?; + for ix in 0..header.paragraphs.len() { + assert_eq!( + crate::paragraph_text(&header, ix), + crate::paragraph_text(&full, ix), + "header paragraph {ix} matches the full doc" + ); + } + + // Edit the document, persist as a journal delta (no full checkpoint). The + // full projection cache is nulled — but the header rides the delta and + // stays fresh. + let text = body_text(&doc); + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + text.insert(1, "EDIT ").map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(loro::ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + assert!( + package.manifest.projection_cache_frontier.is_none(), + "an edit nulls the full projection cache" + ); + let edited_projection = document_from_loro(&doc).map_err(loro_test_error)?; + package.refresh_preview_header(&edited_projection)?; + package.append_latest_update_to_path(&path)?; + + // The full cache is absent, but the header preview is fresh and correct. + assert!( + DocumentPackage::read_cached_projection(&path)?.is_none(), + "no fresh full cache after the edit" + ); + let header2 = DocumentPackage::read_preview_header(&path)?.expect("header fresh after edit via the journal delta"); + assert_eq!(header2.frontier, package.manifest.latest_frontier, "header is at the latest frontier"); + assert_eq!( + crate::paragraph_text(&header2, 0), + "EDIT paragraph 000 of the document body", + "header reflects the edit" + ); + Ok(()) + } + + /// 2026-07-07 .db8 first-edit-freeze regressions: (a) appending a segment + /// must be O(delta) — it must not re-validate the WHOLE package (whose full + /// validation previously imported the entire Loro snapshot per keystroke); + /// (b) reading a stacked journal must decode only the LAST generation. + #[test] + fn append_update_segment_validates_only_the_delta() -> io::Result<()> { + let doc = new_loro_document("Delta validate").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Delta validate")?; + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc).insert(1, "x").map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + + // Corrupt HISTORY (the stored first segment's checksum): a delta-scoped + // append must still succeed, while a full validation must reject it. + package.loro_update_segments[0].checksum = [0_u8; 32]; + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc).insert(1, "y").map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package + .append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update) + .expect("append validates only the delta, not all history"); + assert!(package.validate().is_err(), "full validation still catches the historical corruption"); + Ok(()) + } + + #[test] + fn journal_read_decodes_only_the_latest_generation() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("generations.db8"); + let doc = new_loro_document("Generations").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Generations")?; + package.write(&path)?; + // Stack several full generations by rewriting through the journal append + // path (each `write` under the compaction threshold appends a generation). + for round in 0..3 { + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + body_text(&doc) + .insert(1, &format!("g{round}")) + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + package.write(&path)?; + } + let bytes = fs::read(&path)?; + let payloads = committed_journal_payloads(&bytes)?; + let generations = payloads + .iter() + .filter(|payload| payload.starts_with(PACKAGE_MAGIC)) + .count(); + assert!(generations >= 2, "test setup must stack multiple generations, got {generations}"); + let read_back = DocumentPackage::read(&path)?; + assert_eq!(read_back.manifest.latest_frontier, package.manifest.latest_frontier); + assert_eq!(read_back.loro_update_segments.len(), package.loro_update_segments.len()); + Ok(()) + } + + #[test] + fn shallow_snapshot_chunk_round_trips_and_decodes_shallow() -> io::Result<()> { + let doc = new_loro_document("Shallow").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "early history") + .map_err(loro_test_error)?; + doc.commit(); + let mid_frontier = doc.state_frontiers(); + text + .insert(text.len_unicode(), " and the recent window") + .map_err(loro_test_error)?; + doc.commit(); + + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow")?; + package.record_shallow_snapshot(&doc, &mid_frontier)?; + let bytes = package.to_bytes()?; + + let reloaded = DocumentPackage::from_bytes(&bytes)?; + let shallow = reloaded + .latest_shallow_snapshot() + .expect("shallow chunk survives the round trip"); + assert_eq!(reloaded.manifest.shallow_snapshot_frontier.as_deref(), Some(shallow.frontier.as_slice())); + // The accelerator never displaces the full snapshot (invariant S1). + assert!(!reloaded.loro_snapshots.is_empty()); + + let probe = LoroDoc::new(); + probe.import(&shallow.bytes).map_err(loro_test_error)?; + assert!(probe.is_shallow(), "decoded chunk must be a shallow doc"); + assert_eq!(body_text(&probe).to_string(), body_text(&doc).to_string()); + assert_eq!( + encode_frontiers(&probe.shallow_since_frontiers()), + shallow.root_frontier, + "recorded root must match the decoded doc's shallow root" + ); + assert_eq!(encode_frontiers(&probe.oplog_frontiers()), shallow.frontier); + Ok(()) + } + + #[test] + fn clear_shallow_snapshot_round_trips_clean() -> io::Result<()> { + let doc = new_loro_document("Shallow clear").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "content") + .map_err(loro_test_error)?; + doc.commit(); + let root = doc.state_frontiers(); + + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow clear")?; + package.record_shallow_snapshot(&doc, &root)?; + package.clear_shallow_snapshot(); + let reloaded = DocumentPackage::from_bytes(&package.to_bytes()?)?; + assert!(reloaded.latest_shallow_snapshot().is_none()); + assert!(reloaded.manifest.shallow_snapshot_frontier.is_none()); + Ok(()) + } + + #[test] + fn shallow_stamp_without_chunk_fails_validate() -> io::Result<()> { + let doc = new_loro_document("Shallow stamp").map_err(loro_test_error)?; + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow stamp")?; + package.manifest.shallow_snapshot_frontier = Some(package.manifest.latest_frontier.clone()); + let error = package + .validate() + .expect_err("stamp without a chunk must fail"); + assert!(error.to_string().contains("shallow snapshot")); + Ok(()) + } + + #[test] + fn shallow_load_equals_full_load_with_trailing_segments() -> io::Result<()> { + let doc = new_loro_document("Shallow equivalence").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "history before the root") + .map_err(loro_test_error)?; + doc.commit(); + let root = doc.state_frontiers(); + text + .insert(text.len_unicode(), " | window before checkpoint") + .map_err(loro_test_error)?; + doc.commit(); + + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow equivalence")?; + package.compact_to_snapshot(&doc)?; + package.record_shallow_snapshot(&doc, &root)?; + + // Post-checkpoint segment appended AFTER the shallow chunk was minted — + // the shallow load must replay it on top of the shallow state. + let from_frontier = doc.state_frontiers(); + let from_vv = doc.state_vv(); + text + .insert(text.len_unicode(), " | post-checkpoint edits") + .map_err(loro_test_error)?; + doc.commit(); + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + + let reloaded = DocumentPackage::from_bytes(&package.to_bytes()?)?; + let shallow_doc = reloaded + .load_loro_doc_shallow()? + .expect("shallow chunk + trailing segment must reconstruct the tip"); + assert!(shallow_doc.is_shallow()); + let full_doc = reloaded.load_loro_doc()?; + assert_eq!(body_text(&shallow_doc).to_string(), body_text(&full_doc).to_string()); + assert_eq!( + encode_frontiers(&shallow_doc.oplog_frontiers()), + encode_frontiers(&full_doc.oplog_frontiers()) + ); + let shallow_projection = crate::document_from_loro(&shallow_doc).map_err(loro_test_error)?; + let full_projection = crate::document_from_loro(&full_doc).map_err(loro_test_error)?; + assert_eq!(shallow_projection.ids.paragraph_ids, full_projection.ids.paragraph_ids); + assert_eq!(shallow_projection.paragraphs.len(), full_projection.paragraphs.len()); + + // A stale stamp (frontier no longer reconstructing the tip) must demote + // to the full path, not open a wrong version. + let mut stale = reloaded.clone(); + stale.manifest.latest_frontier = vec![9, 9, 9]; + assert!(stale.load_loro_doc_shallow()?.is_none()); + Ok(()) + } + + #[test] + fn compaction_refuses_a_shallow_live_doc() -> io::Result<()> { + let doc = new_loro_document("Shallow compact guard").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "history") + .map_err(loro_test_error)?; + doc.commit(); + let root = doc.state_frontiers(); + text + .insert(text.len_unicode(), " tip") + .map_err(loro_test_error)?; + doc.commit(); + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow compact guard")?; + package.record_shallow_snapshot(&doc, &root)?; + let shallow_doc = package + .load_loro_doc_shallow()? + .expect("shallow chunk reconstructs the tip"); + + let error = package + .compact_to_snapshot(&shallow_doc) + .expect_err("compacting a shallow live doc must refuse (S1)"); + assert!(error.to_string().contains("shallow")); + // The inline backstop silently skips instead of erroring. + assert!( + package + .compact_update_segments_if_needed(&shallow_doc, 1)? + .is_none() + ); + // The package is untouched: full snapshot still present, still loadable. + package.load_loro_doc()?; + Ok(()) + } + + #[test] + fn shallow_session_checkpoint_preserves_full_history() -> io::Result<()> { + let doc = new_loro_document("Shallow checkpoint").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "ancient history") + .map_err(loro_test_error)?; + doc.commit(); + let ancient_frontier = doc.state_frontiers(); + text + .insert(text.len_unicode(), " | window") + .map_err(loro_test_error)?; + doc.commit(); + + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow checkpoint")?; + package.compact_to_snapshot(&doc)?; + package.record_shallow_snapshot(&doc, &ancient_frontier)?; + + // A shallow SESSION: open from the shallow chunk, edit, checkpoint. + let live = package + .load_loro_doc_shallow()? + .expect("shallow chunk reconstructs the tip"); + assert!(live.is_shallow()); + let from_frontier = live.state_frontiers(); + let from_vv = live.state_vv(); + let live_text = body_text(&live); + live_text + .insert(live_text.len_unicode(), " | shallow session edit") + .map_err(loro_test_error)?; + live.commit(); + let update = live + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &live.state_frontiers(), &live.state_vv(), update)?; + + package.compact_to_snapshot_any(&live)?; + + let reloaded = DocumentPackage::from_bytes(&package.to_bytes()?)?; + // The full chunk carries COMPLETE history: not shallow, and the ancient + // pre-root frontier is still reachable. + let full = reloaded.load_loro_doc()?; + assert!(!full.is_shallow(), "compaction from a shallow session must persist FULL history"); + assert_eq!(body_text(&full).to_string(), body_text(&live).to_string()); + full.checkout(&ancient_frontier).map_err(loro_test_error)?; + full.checkout_to_latest(); + // The shallow accelerator was re-minted at the new checkpoint tip. + let shallow = reloaded + .latest_shallow_snapshot() + .expect("checkpoint re-mints the shallow accelerator"); + assert_eq!(shallow.frontier, reloaded.manifest.latest_frontier); + let next_live = reloaded + .load_loro_doc_shallow()? + .expect("next open takes the shallow path again"); + assert_eq!(body_text(&next_live).to_string(), body_text(&live).to_string()); + Ok(()) + } + + #[test] + fn pre_root_revision_loads_from_a_shallow_session_package() -> io::Result<()> { + // G3: time-travel below the shallow root is served from the package's + // retained full history, never the live doc. + let doc = new_loro_document("Shallow revision").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "ancient state") + .map_err(loro_test_error)?; + doc.commit(); + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow revision")?; + let (ancient_revision, _) = package.compact_to_named_snapshot(&doc, "Ancient", "pre-root", None, None)?; + + text + .insert(text.len_unicode(), " | later work") + .map_err(loro_test_error)?; + doc.commit(); + let root = doc.state_frontiers(); + text + .insert(text.len_unicode(), " | window") + .map_err(loro_test_error)?; + doc.commit(); + let from_frontier = package_tip_frontiers(&package)?; + let from_vv = package_tip_vv(&package)?; + let update = doc + .export(ExportMode::updates(&from_vv)) + .map_err(loro_test_error)?; + package.append_update_segment(&from_frontier, &from_vv, &doc.state_frontiers(), &doc.state_vv(), update)?; + package.record_shallow_snapshot(&doc, &root)?; + + // Shallow session checkpoints; the ancient revision must still load. + let live = package.load_loro_doc_shallow()?.expect("shallow session"); + package.compact_to_snapshot_any(&live)?; + let revision_doc = package.load_revision_loro_doc(ancient_revision)?; + assert_eq!(body_text(&revision_doc).to_string(), "\nancient state"); + Ok(()) + } + + fn package_tip_frontiers(package: &DocumentPackage) -> io::Result { + decode_frontiers(&package.manifest.latest_frontier) + } + + fn package_tip_vv(package: &DocumentPackage) -> io::Result { + VersionVector::decode(&package.manifest.latest_version_vector).map_err(loro_test_error) + } + + #[test] + fn deferred_snapshot_hash_still_catches_corruption_at_decode() -> io::Result<()> { + let doc = new_loro_document("Deferred hash").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "content to corrupt") + .map_err(loro_test_error)?; + doc.commit(); + let package = DocumentPackage::from_loro_snapshot(&doc, "Deferred hash")?; + let mut bytes = package.to_bytes()?; + + // Flip one byte INSIDE the latest snapshot chunk's encoded payload. The + // read defers full-snapshot hashing, so from_bytes must still succeed... + let payload = encode_chunk(package.latest_snapshot().expect("snapshot"), "Loro snapshot")?; + let payload_at = bytes + .windows(payload.len()) + .position(|window| window == payload) + .expect("latest snapshot payload present in the package bytes"); + bytes[payload_at + payload.len() / 2] ^= 0xFF; + let reloaded = DocumentPackage::from_bytes(&bytes)?; + // ...but every consumer of those bytes must catch it. + let load_error = reloaded + .load_loro_doc() + .expect_err("corrupt snapshot must fail at decode"); + assert!(load_error.to_string().contains("checksum"), "{load_error}"); + let write_error = reloaded + .to_bytes() + .expect_err("corrupt snapshot must never be re-written as valid"); + assert!(write_error.to_string().contains("checksum"), "{write_error}"); + Ok(()) + } + + #[test] + fn shallow_chunk_survives_journal_tail_repair() -> io::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("shallow-tail.db8"); + let doc = new_loro_document("Shallow tail").map_err(loro_test_error)?; + let text = body_text(&doc); + text + .insert(text.len_unicode(), "durable") + .map_err(loro_test_error)?; + doc.commit(); + let root = doc.state_frontiers(); + let mut package = DocumentPackage::from_loro_snapshot(&doc, "Shallow tail")?; + package.record_shallow_snapshot(&doc, &root)?; + package.write(&path)?; + let committed_len = fs::metadata(&path)?.len(); + + let mut file = OpenOptions::new().append(true).open(&path)?; + std::io::Write::write_all(&mut file, b"torn shallow tail")?; + drop(file); + + let repaired = DocumentPackage::read(&path)?; + repaired.validate()?; + assert!(repaired.latest_shallow_snapshot().is_some()); + assert_eq!(fs::metadata(&path)?.len(), committed_len); + Ok(()) + } +} diff --git a/crates/flowstate-document/src/package_search.rs b/crates/flowstate-document/src/package_search.rs new file mode 100644 index 00000000..03811772 --- /dev/null +++ b/crates/flowstate-document/src/package_search.rs @@ -0,0 +1,501 @@ +use std::io; + +use gpui_flowtext::{DocumentTheme, InputBlock, InputParagraph, ParagraphStyle, RunSemanticStyle}; +use loro::{Container, LoroDoc, LoroMap, LoroText, LoroValue, ValueOrContainer, cursor::Side}; + +use crate::{ + BLOCKS_BY_ID, FLOW_TEXT_KEY, FLOWS_BY_ID, OBJECT_REPLACEMENT, ROOT, ROOT_BODY_FLOW_ID, flowstate_document_theme, package::SearchUnitChunk, +}; + +pub(crate) fn search_units_from_loro(doc: &LoroDoc, document_id: u128, frontier: &[u8]) -> io::Result> { + let input_blocks = crate::loro_projection::input_blocks_from_loro(doc)?; + search_units_from_input_blocks(doc, &input_blocks, document_id, frontier) +} + +/// §act-five P9: build search units from PRE-MATERIALIZED input blocks, so a +/// checkpoint that already materialized the projection (for the projection cache) +/// does not re-run the full `document_from_loro` a SECOND time just for search — +/// the two independent materializations pegged the off-gate package thread. +pub(crate) fn search_units_from_input_blocks( + doc: &LoroDoc, + input_blocks: &[InputBlock], + document_id: u128, + frontier: &[u8], +) -> io::Result> { + let probe0 = std::time::Instant::now(); + let body = crate::loro_schema::body_text(doc); + let body_paragraph_ranges = body_paragraph_cursor_ranges(&body); + let probe_ranges = probe0.elapsed(); + // §A13.4.2 (vendor patch #26): resolve EVERY paragraph range's cursors in + // one batched state acquisition — per-call `get_cursor` overhead dominated + // the search reindex (~21k calls on a 5k-paragraph doc). + let cursor_queries: Vec<(usize, loro::cursor::Side)> = body_paragraph_ranges + .iter() + .flat_map(|range| [(range.start, loro::cursor::Side::Left), (range.end, loro::cursor::Side::Right)]) + .collect(); + let probe = std::env::var_os("FLOWSTATE_OPEN_PROBE").is_some(); + let probe_t = std::time::Instant::now(); + let resolved = { + use loro::ContainerTrait as _; + body.to_handler().get_cursors_batch(&cursor_queries) + }; + let probe_cursors = probe_t.elapsed(); + let body_paragraph_cursors: Vec<(Vec, Vec)> = resolved + .chunks(2) + .map(|pair| { + let start = pair[0] + .as_ref() + .map(loro::cursor::Cursor::encode) + .unwrap_or_default(); + let end = pair + .get(1) + .and_then(|c| c.as_ref()) + .map(loro::cursor::Cursor::encode) + .unwrap_or_default(); + (start, end) + }) + .collect(); + let mut builder = SearchUnitBuilder { + document_id, + frontier, + units: Vec::new(), + next_unit_ix: 0, + heading_path: Vec::new(), + body_paragraph_cursors, + body_paragraph_ix: 0, + theme: flowstate_document_theme(), + }; + let probe_t = std::time::Instant::now(); + for block in input_blocks { + builder.push_block(block, &body); + } + let probe_blocks = probe_t.elapsed(); + let probe_t = std::time::Instant::now(); + builder.push_loro_object_units(doc)?; + if probe { + eprintln!( + "[flowstate-search-probe] ranges={probe_ranges:?} batch={probe_cursors:?} blocks={probe_blocks:?} objects={:?} units={}", + probe_t.elapsed(), + builder.units.len() + ); + } + Ok(builder.units) +} + +#[derive(Clone, Copy, Debug)] +struct BodyParagraphRange { + start: usize, + end: usize, +} + +struct SearchUnitBuilder<'a> { + document_id: u128, + frontier: &'a [u8], + units: Vec, + next_unit_ix: usize, + heading_path: Vec, + /// §A13.4.2: pre-resolved (start, end) cursor encodings per body + /// paragraph, from ONE batched resolution. + body_paragraph_cursors: Vec<(Vec, Vec)>, + body_paragraph_ix: usize, + theme: DocumentTheme, +} + +impl SearchUnitBuilder<'_> { + fn push_block(&mut self, block: &InputBlock, body: &LoroText) { + match block { + InputBlock::Paragraph(paragraph) => self.push_body_paragraph(paragraph, body), + InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_) => {}, + } + } + + fn push_body_paragraph(&mut self, paragraph: &InputParagraph, _body: &LoroText) { + let text = input_paragraph_text(paragraph); + let normalized = normalized_search_text(&text); + if !normalized.is_empty() + && let Some(level) = heading_level(&self.theme, paragraph.style) + { + self.update_heading_path(level, normalized.clone()); + } + let cursors = self + .body_paragraph_cursors + .get(self.body_paragraph_ix) + .cloned(); + self.body_paragraph_ix += 1; + self.push_text_unit( + paragraph_unit_kind(paragraph), + &text, + SearchUnitRefs { + flow_id: Some(ROOT_BODY_FLOW_ID.to_string()), + cursors: cursors.clone(), + paragraph_cursors: cursors, + ..SearchUnitRefs::default() + }, + ); + } + + fn push_loro_object_units(&mut self, doc: &LoroDoc) -> io::Result<()> { + let root = doc.get_map(ROOT); + let Some(blocks) = child_map(&root, BLOCKS_BY_ID) else { + return Ok(()); + }; + let Some(flows) = child_map(&root, FLOWS_BY_ID) else { + return Ok(()); + }; + for block_id in map_keys(&blocks) { + let Some(block) = child_map(&blocks, &block_id) else { + continue; + }; + match map_string_opt(&block, "kind").as_deref() { + Some("image") => self.push_image_units(&flows, &block_id, &block)?, + Some("equation") => self.push_equation_units(&flows, &block_id, &block)?, + Some("table") => self.push_table_units(&flows, &block_id, None, &block)?, + _ => {}, + } + } + Ok(()) + } + + fn push_image_units(&mut self, flows: &LoroMap, block_id: &str, block: &LoroMap) -> io::Result<()> { + if let Some(flow_id) = map_string_opt(block, "alt_text_flow_id") { + self.push_flow_text_unit(flows, "image_alt", &flow_id, SearchUnitRefs::for_block(block_id))?; + } + if let Some(flow_id) = map_string_opt(block, "caption_flow_id") { + self.push_flow_text_unit(flows, "image_caption", &flow_id, SearchUnitRefs::for_block(block_id))?; + } + Ok(()) + } + + fn push_equation_units(&mut self, flows: &LoroMap, block_id: &str, block: &LoroMap) -> io::Result<()> { + if let Some(flow_id) = map_string_opt(block, "source_flow_id") { + self.push_flow_text_unit(flows, "equation", &flow_id, SearchUnitRefs::for_block(block_id))?; + } + Ok(()) + } + + fn push_table_units(&mut self, flows: &LoroMap, block_id: &str, _parent_cell_id: Option<&str>, owner: &LoroMap) -> io::Result<()> { + let Some(table) = child_map(owner, "table") else { + return Ok(()); + }; + let table_id = map_string_opt(owner, "id").unwrap_or_else(|| block_id.to_string()); + let Some(cells) = child_map(&table, "cells_by_id") else { + return Ok(()); + }; + for cell_id in map_keys(&cells) { + let Some(cell) = child_map(&cells, &cell_id) else { + continue; + }; + if let Some(flow_id) = map_string_opt(&cell, "flow_id") { + self.push_flow_text_unit( + flows, + "table_cell", + &flow_id, + SearchUnitRefs { + block_id: Some(block_id.to_string()), + table_id: Some(table_id.clone()), + cell_id: Some(cell_id.clone()), + ..SearchUnitRefs::default() + }, + )?; + } + if let Some(nested_tables) = child_map(&cell, "nested_tables_by_id") { + for nested_id in ordered_ids(&cell, "nested_table_ids") { + if let Some(nested) = child_map(&nested_tables, &nested_id) { + self.push_table_units(flows, block_id, Some(&cell_id), &nested)?; + } + } + } + } + Ok(()) + } + + fn push_flow_text_unit(&mut self, flows: &LoroMap, unit_kind: &str, flow_id: &str, mut refs: SearchUnitRefs) -> io::Result<()> { + let Some(flow) = child_map(flows, flow_id) else { + return Ok(()); + }; + let Some(text) = child_text(&flow, FLOW_TEXT_KEY) else { + return Ok(()); + }; + let body = searchable_flow_text(&text); + refs.flow_id = Some(flow_id.to_string()); + refs.cursors = text_cursor_fields(&text); + self.push_text_unit(unit_kind, &body, refs); + Ok(()) + } + + fn push_text_unit(&mut self, unit_kind: &str, text: &str, refs: SearchUnitRefs) { + let body = normalized_search_text(text); + if body.is_empty() { + return; + } + let (unit_start_cursor, unit_end_cursor) = refs.cursors.unwrap_or_default(); + let (paragraph_start_cursor, paragraph_end_cursor) = refs.paragraph_cursors.unwrap_or_default(); + let heading = self.heading_path.last().cloned().unwrap_or_default(); + let unit_id = stable_search_unit_id(self.document_id, self.next_unit_ix, self.frontier, unit_kind, &body); + self.next_unit_ix += 1; + self.units.push(SearchUnitChunk { + frontier: self.frontier.to_vec(), + unit_id, + unit_kind: unit_kind.to_string(), + flow_id: refs.flow_id, + block_id: refs.block_id, + table_id: refs.table_id, + cell_id: refs.cell_id, + heading_path: self.heading_path.clone(), + heading, + body: body.clone(), + insert_text: body, + unit_start_cursor, + unit_end_cursor, + paragraph_start_cursor, + paragraph_end_cursor, + }); + } + + fn update_heading_path(&mut self, level: usize, heading: String) { + if self.heading_path.len() <= level { + self.heading_path.resize(level + 1, String::new()); + } else { + self.heading_path.truncate(level + 1); + } + self.heading_path[level] = heading; + } +} + +#[derive(Default)] +struct SearchUnitRefs { + flow_id: Option, + block_id: Option, + table_id: Option, + cell_id: Option, + cursors: Option<(Vec, Vec)>, + paragraph_cursors: Option<(Vec, Vec)>, +} + +impl SearchUnitRefs { + fn for_block(block_id: &str) -> Self { + Self { + block_id: Some(block_id.to_string()), + ..Self::default() + } + } +} + +fn body_paragraph_cursor_ranges(text: &LoroText) -> Vec { + let mut ranges = Vec::new(); + let mut rendered_blocks = 0_usize; + let mut start = 0_usize; + let mut has_text = false; + let mut seen_sentinel = false; + let mut unicode_pos = 0_usize; + + // §A13.4.2: raw chunk iteration — the former `streaming_to_delta` walk + // paid rich-text mark segmentation for a pass that only reads TEXT. + let mut walk = |insert: &str| { + for ch in insert.chars() { + match ch { + '\n' => { + if seen_sentinel { + ranges.push(BodyParagraphRange { start, end: unicode_pos }); + rendered_blocks += 1; + } else { + seen_sentinel = true; + } + start = unicode_pos + 1; + has_text = false; + }, + OBJECT_REPLACEMENT => { + if has_text { + ranges.push(BodyParagraphRange { start, end: unicode_pos }); + rendered_blocks += 1; + has_text = false; + } + rendered_blocks += 1; + start = unicode_pos + 1; + }, + _ => has_text = true, + } + unicode_pos += 1; + } + }; + text.iter(|chunk| { + walk(chunk); + true + }); + + if has_text || rendered_blocks == 0 && seen_sentinel { + ranges.push(BodyParagraphRange { + start, + end: text.len_unicode(), + }); + } + ranges +} + +fn text_cursor_fields(text: &LoroText) -> Option<(Vec, Vec)> { + let len = text.len_unicode(); + if len == 0 { + return None; + } + let start = usize::from(text.to_string().starts_with('\n') && len > 1); + Some(( + text + .get_cursor(start, Side::Left) + .map(|cursor| cursor.encode()) + .unwrap_or_default(), + text + .get_cursor(len, Side::Right) + .map(|cursor| cursor.encode()) + .unwrap_or_default(), + )) +} + +fn input_paragraph_text(paragraph: &InputParagraph) -> String { + paragraph.runs.iter().map(|run| run.text.as_str()).collect() +} + +fn normalized_search_text(text: &str) -> String { + text + .chars() + .filter(|ch| *ch != OBJECT_REPLACEMENT) + .collect::() + .trim() + .to_string() +} + +fn searchable_flow_text(text: &LoroText) -> String { + let text = text.to_string(); + let searchable = text + .strip_prefix('\n') + .unwrap_or(&text) + .replace(OBJECT_REPLACEMENT, " "); + let Some(without_first_newline) = searchable.strip_prefix('\n') else { + return searchable; + }; + let content = without_first_newline.trim_start_matches('\n'); + let mut normalized = String::with_capacity(content.len() + 1); + normalized.push('\n'); + normalized.push_str(content); + normalized +} + +fn child_map(parent: &LoroMap, key: &str) -> Option { + parent.get(key).and_then(|value| match value { + ValueOrContainer::Container(container) => container.into_map().ok(), + ValueOrContainer::Value(_) => None, + }) +} + +fn child_text(parent: &LoroMap, key: &str) -> Option { + parent.get(key).and_then(|value| match value { + ValueOrContainer::Container(container) => container.into_text().ok(), + ValueOrContainer::Value(_) => None, + }) +} + +fn map_keys(map: &LoroMap) -> Vec { + let mut keys = map.keys().map(|key| key.to_string()).collect::>(); + keys.sort(); + keys +} + +fn map_string_opt(map: &LoroMap, key: &str) -> Option { + map.get(key).and_then(|value| match value { + ValueOrContainer::Value(LoroValue::String(value)) => Some(value.to_string()), + _ => None, + }) +} + +fn ordered_ids(map: &LoroMap, key: &str) -> Vec { + let Some(ValueOrContainer::Container(Container::MovableList(list))) = map.get(key) else { + return Vec::new(); + }; + (0..list.len()) + .filter_map(|ix| match list.get(ix) { + Some(ValueOrContainer::Value(LoroValue::String(value))) => Some(value.to_string()), + _ => None, + }) + .collect() +} + +fn paragraph_unit_kind(paragraph: &InputParagraph) -> &'static str { + if paragraph + .runs + .iter() + .any(|run| matches!(run.styles.semantic, RunSemanticStyle::Custom(1))) + { + return "cite"; + } + match paragraph.style { + ParagraphStyle::Custom(0) => "pocket", + ParagraphStyle::Custom(1) => "hat", + ParagraphStyle::Custom(2) => "block", + ParagraphStyle::Custom(3) => "tag", + ParagraphStyle::Custom(4) => "analytic", + ParagraphStyle::Custom(6) => "undertag", + ParagraphStyle::Normal | ParagraphStyle::Custom(_) => "paragraph", + } +} + +fn heading_level(theme: &DocumentTheme, style: ParagraphStyle) -> Option { + let ParagraphStyle::Custom(slot) = style else { + return None; + }; + theme + .custom_paragraph_styles + .get(&(slot & 0x7f)) + .and_then(|style| style.section_level) + .map(usize::from) +} + +fn stable_search_unit_id(document_id: u128, unit_ix: usize, frontier: &[u8], unit_kind: &str, body: &str) -> u128 { + let mut hasher = blake3::Hasher::new(); + hasher.update(&document_id.to_le_bytes()); + hasher.update(&(unit_ix as u64).to_le_bytes()); + hasher.update(frontier); + hasher.update(unit_kind.as_bytes()); + hasher.update(body.as_bytes()); + let digest = hasher.finalize(); + let mut bytes = [0_u8; 16]; + bytes.copy_from_slice(&digest.as_bytes()[..16]); + u128::from_le_bytes(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::loro_schema::new_loro_document; + + #[test] + fn body_paragraph_ranges_match_projection_object_boundaries() -> io::Result<()> { + let doc = new_loro_document("ranges").map_err(loro_test_error)?; + let body = crate::loro_schema::body_text(&doc); + body.insert(1, "before").map_err(loro_test_error)?; + body + .insert(body.len_unicode(), &OBJECT_REPLACEMENT.to_string()) + .map_err(loro_test_error)?; + body + .insert(body.len_unicode(), "after") + .map_err(loro_test_error)?; + + let ranges = body_paragraph_cursor_ranges(&body); + assert_eq!(ranges.len(), 2); + assert_eq!(ranges[0].start..ranges[0].end, 1..7); + assert_eq!(ranges[1].start..ranges[1].end, 8..13); + Ok(()) + } + + #[test] + fn searchable_flow_text_preserves_one_leading_sentinel_newline() -> io::Result<()> { + let doc = new_loro_document("text").map_err(loro_test_error)?; + let body = crate::loro_schema::body_text(&doc); + body.insert(1, "\n\nalpha").map_err(loro_test_error)?; + + assert_eq!(searchable_flow_text(&body), "\nalpha"); + Ok(()) + } + + fn loro_test_error(error: impl std::error::Error + Send + Sync + 'static) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error) + } +} diff --git a/crates/flowstate-document/src/projection_defects.rs b/crates/flowstate-document/src/projection_defects.rs new file mode 100644 index 00000000..2d654c81 --- /dev/null +++ b/crates/flowstate-document/src/projection_defects.rs @@ -0,0 +1,184 @@ +//! §5 projection defect reporting. +//! +//! Projecting a Loro document must never silently normalize malformed canonical +//! state: dropped blocks, fabricated identities, and coerced ids are data +//! corruption laundering. Instead, the projector records one +//! [`ProjectionDefect`] per anomaly while still producing a deterministic +//! projection (quarantined content is appended in stable order, fabricated +//! identities are deterministic for that projection only). The CRDT runtime +//! consumes the defects and applies idempotent canonical repair mutations with a +//! dedicated commit origin, so peers converge on the repaired state. + +/// One malformed-canonical-state anomaly discovered while projecting a Loro +/// document. Each variant carries enough context for the runtime to apply (or +/// refuse) a canonical repair keyed by the stable location of the defect. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProjectionDefect { + /// FS-002: an object block whose `anchor_cursor` is missing, undecodable, or + /// no longer resolves to a live U+FFFC placeholder in its flow. The block is + /// projected in a quarantine position instead of vanishing. + UnresolvedObjectAnchor { + block_key: String, + flow_id: String, + anchor_cursor: Option>, + }, + /// FS-003: two object blocks resolved their anchors to the same placeholder + /// position; only one can own it. The displaced block is projected in a + /// quarantine position instead of being overwritten out of existence. + CollidingObjectAnchors { + flow_id: String, + anchor_unicode: usize, + kept_block_key: String, + displaced_block_key: String, + }, + /// FS-004: a paragraph boundary with no durable paragraph metadata record. + /// The projection keeps a deterministic fabricated placeholder id for this + /// projection only; the runtime must write a real durable record. + /// `boundary_unicode` is `None` when no live boundary newline exists (e.g. an + /// entirely empty/uninitialized flow). + MissingParagraphMetadata { + flow_id: String, + boundary_unicode: Option, + fabricated_id: u128, + }, + /// FS-005: a paragraph boundary with no durable paragraph *block* record. + /// Mirrors [`Self::MissingParagraphMetadata`] for the block registry. + MissingParagraphBlock { + flow_id: String, + boundary_unicode: Option, + fabricated_id: u128, + }, + /// FS-011: an image block whose `asset_id` is missing or unparseable. The + /// projection uses a placeholder `AssetId(0)` and reports the defect so the + /// runtime can recover the id (e.g. from the block's `content_hash`). + InvalidAssetId { block_key: String, raw_asset_id: Option }, + /// adjustmentplan:224: a paragraph boundary newline without a + /// paragraph-style mark. The projection defaults it to `Normal` and the + /// runtime schedules the canonical mark repair. + MissingParagraphStyleMark { flow_id: String, boundary_unicode: usize }, + /// FS-036 backstop: a U+FFFC placeholder character with no object block + /// anchored to it. Projecting skips the character; the runtime deletes the + /// orphan character canonically once no block claims it. + OrphanObjectPlaceholder { flow_id: String, unicode_pos: usize }, + /// FS-010: a table grid that is not a well-formed full rectangle — a missing + /// `(row, column)` cell (the concurrent add-row × add-column gap), a duplicate + /// coordinate, an out-of-bounds cell span, or a cell orphaned from the + /// `row_order` / `column_order`. The projector normalizes the grid + /// deterministically (synthesize / clamp / drop) so every peer reads the same + /// full grid, and the runtime applies the matching idempotent canonical repair + /// (ensure the coordinate cell / clamp the span / delete the orphan). The + /// `row_id` / `column_id` are the durable u128 coordinate ids (`None` when the + /// defect is not tied to a single coordinate). + TableTopology { + table_block_key: String, + row_id: Option, + column_id: Option, + kind: TableTopologyKind, + }, +} + +/// The specific table-grid anomaly carried by [`ProjectionDefect::TableTopology`] +/// (§P2b / FS-010). Mirrors `table_topology::TableTopologyDefect` but stays a +/// flat, id-type-free enum so `projection_defects` does not depend on the id +/// newtypes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableTopologyKind { + MissingCell, + DuplicateCoordinate, + InvalidSpan, + OrphanCell, +} + +impl TableTopologyKind { + /// Short stable discriminator used in the defect's stable key. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::MissingCell => "missing_cell", + Self::DuplicateCoordinate => "duplicate_coordinate", + Self::InvalidSpan => "invalid_span", + Self::OrphanCell => "orphan_cell", + } + } +} + +impl ProjectionDefect { + /// Short stable class name for telemetry counters. + #[must_use] + pub fn class(&self) -> &'static str { + match self { + Self::UnresolvedObjectAnchor { .. } => "unresolved_object_anchor", + Self::CollidingObjectAnchors { .. } => "colliding_object_anchors", + Self::MissingParagraphMetadata { .. } => "missing_paragraph_metadata", + Self::MissingParagraphBlock { .. } => "missing_paragraph_block", + Self::InvalidAssetId { .. } => "invalid_asset_id", + Self::MissingParagraphStyleMark { .. } => "missing_paragraph_style_mark", + Self::OrphanObjectPlaceholder { .. } => "orphan_object_placeholder", + Self::TableTopology { .. } => "table_topology", + } + } + + /// Stable key identifying the canonical location of the defect. The runtime + /// caps repair attempts per key, so a defect that persists across repair + /// passes is quarantined instead of spinning. + #[must_use] + pub fn stable_key(&self) -> String { + match self { + Self::UnresolvedObjectAnchor { block_key, flow_id, .. } => { + format!("{}:{flow_id}:{block_key}", self.class()) + }, + Self::CollidingObjectAnchors { + flow_id, + displaced_block_key, + .. + } => format!("{}:{flow_id}:{displaced_block_key}", self.class()), + Self::MissingParagraphMetadata { + flow_id, boundary_unicode, .. + } + | Self::MissingParagraphBlock { + flow_id, boundary_unicode, .. + } => match boundary_unicode { + Some(boundary) => format!("{}:{flow_id}:{boundary}", self.class()), + None => format!("{}:{flow_id}:none", self.class()), + }, + Self::InvalidAssetId { block_key, .. } => format!("{}:{block_key}", self.class()), + Self::MissingParagraphStyleMark { flow_id, boundary_unicode } => { + format!("{}:{flow_id}:{boundary_unicode}", self.class()) + }, + Self::OrphanObjectPlaceholder { flow_id, unicode_pos } => { + format!("{}:{flow_id}:{unicode_pos}", self.class()) + }, + Self::TableTopology { + table_block_key, + row_id, + column_id, + kind, + } => { + let row = row_id.map_or_else(|| "none".to_string(), |id| id.to_string()); + let column = column_id.map_or_else(|| "none".to_string(), |id| id.to_string()); + format!("{}:{table_block_key}:{}:{row}:{column}", self.class(), kind.as_str()) + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stable_keys_are_stable_per_canonical_location() { + let defect = ProjectionDefect::MissingParagraphMetadata { + flow_id: "body".to_string(), + boundary_unicode: Some(6), + fabricated_id: 1, + }; + let same_location_other_fabrication = ProjectionDefect::MissingParagraphMetadata { + flow_id: "body".to_string(), + boundary_unicode: Some(6), + fabricated_id: 2, + }; + assert_eq!(defect.stable_key(), same_location_other_fabrication.stable_key()); + assert_eq!(defect.stable_key(), "missing_paragraph_metadata:body:6"); + } +} diff --git a/crates/flowstate-document/src/streaming_delta.rs b/crates/flowstate-document/src/streaming_delta.rs new file mode 100644 index 00000000..ad7f4bbe --- /dev/null +++ b/crates/flowstate-document/src/streaming_delta.rs @@ -0,0 +1,120 @@ +//! §act-eleven A11.10 (perf-heaven T7.7): streaming body-delta +//! materialization. `LoroText::to_delta()` builds the whole body TWICE — the +//! handler materializes a `Vec` (a map per segment: `"insert"` +//! string + `"attributes"` map), then the `loro` facade copies every string +//! and attribute map AGAIN into `Vec`. [`streaming_to_delta`] +//! builds the `Vec` directly from the vendored streaming span +//! walker (`TextHandler::for_each_richtext_span`), skipping the `LoroValue` +//! intermediate entirely: one string copy per segment, one attribute map per +//! style CHANGE (not per span). +//! +//! Equivalence is pinned two ways: the vendored value functions are expressed +//! on the same walkers (so the T1 verify + corpus + fuzz oracles pin the walk +//! itself), and the tests below assert `streaming_to_delta == to_delta` +//! segment-for-segment on styled/unmarked/fragmented fixtures — including the +//! decoded-snapshot `LazyLoad::Src` fast path. + +use loro::{ContainerTrait as _, LoroText, TextDelta}; +use loro_internal::delta::StyleMeta; + +/// Drop-in replacement for `text.to_delta()` (insert-only segments, adjacent +/// spans merged on visible-attribute equality — the exact merge law +/// `get_richtext_value` applies). +pub fn streaming_to_delta(text: &LoroText) -> Vec { + let handler = text.to_handler(); + let mut out: Vec = Vec::new(); + let mut last_meta: Option = None; + handler.for_each_richtext_span(&mut |chunk, meta| { + if let Some(last) = last_meta.as_ref() + && last.visible_eq(meta) + && let Some(TextDelta::Insert { insert, .. }) = out.last_mut() + { + insert.push_str(chunk); + return; + } + let visible = meta.to_visible_attributes(); + out.push(TextDelta::Insert { + insert: chunk.to_string(), + attributes: (!visible.is_empty()).then_some(visible), + }); + last_meta = Some(meta.clone()); + }); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::loro_schema::{self, MARK_DIRECT_UNDERLINE, MARK_HIGHLIGHT_STYLE, MARK_RUN_SEMANTIC_STYLE, MARK_STRIKETHROUGH}; + use loro::{LoroDoc, LoroValue}; + + /// A delta-shape-diverse fixture: several paragraphs, overlapping marks of + /// every run-style key, an UNMARK inside a marked range (null-valued style — + /// the visible-equality edge), fragmented same-style chunks (out-of-order + /// inserts), and multi-byte text. + fn styled_doc() -> (LoroDoc, LoroText) { + let doc = LoroDoc::new(); + loro_schema::configure_text_styles(&doc); + let text = doc.get_text("streaming-fixture"); + text + .insert(0, "Plain lead-in with 宽字符 content.\nSecond paragraph body.\nThird row.") + .expect("insert"); + // Fragment the chunk tree: a later insert in the MIDDLE of the first run + // splits it into multiple same-style chunks that the delta must re-merge. + text.insert(5, " spliced").expect("mid insert"); + text + .mark(0..12, MARK_RUN_SEMANTIC_STYLE, LoroValue::I64(2)) + .expect("mark semantic"); + text + .mark(8..20, MARK_HIGHLIGHT_STYLE, LoroValue::I64(1)) + .expect("mark highlight"); + text + .mark(30..44, MARK_DIRECT_UNDERLINE, LoroValue::Bool(true)) + .expect("mark underline"); + text + .mark(38..50, MARK_STRIKETHROUGH, LoroValue::Bool(true)) + .expect("mark strike"); + // Unmark a subrange INSIDE the semantic run: writes a null-valued style + // key, which the delta's attribute maps must treat as absent. + text.unmark(4..9, MARK_RUN_SEMANTIC_STYLE).expect("unmark"); + doc.commit(); + (doc, text) + } + + #[test] + fn streaming_matches_to_delta_on_built_state() { + let (_doc, text) = styled_doc(); + // to_delta first: forces/uses the built state; streaming then walks the + // same Dst tree. + let baseline = text.to_delta(); + let streamed = streaming_to_delta(&text); + assert!(!baseline.is_empty(), "fixture produced an empty delta — net is vacuous"); + assert_eq!(streamed, baseline, "streaming delta diverged from to_delta on the built state"); + } + + #[test] + fn streaming_matches_to_delta_on_decoded_snapshot_src_path() { + let (doc, _text) = styled_doc(); + let snapshot = doc + .export(loro::ExportMode::Snapshot) + .expect("snapshot export"); + // Streaming FIRST on the freshly decoded doc: the richtext state is still + // `LazyLoad::Src`, so this exercises `for_each_span_from_src`. + let reloaded = LoroDoc::new(); + reloaded.import(&snapshot).expect("snapshot import"); + let streamed = streaming_to_delta(&reloaded.get_text("streaming-fixture")); + // Baseline from a SEPARATE decode so neither call can warm the other. + let baseline_doc = LoroDoc::new(); + baseline_doc.import(&snapshot).expect("snapshot import"); + let baseline = baseline_doc.get_text("streaming-fixture").to_delta(); + assert!(!baseline.is_empty(), "fixture produced an empty delta — net is vacuous"); + assert_eq!(streamed, baseline, "streaming delta diverged from to_delta on the Src fast path"); + } + + #[test] + fn streaming_on_empty_text_is_empty() { + let doc = LoroDoc::new(); + let text = doc.get_text("empty"); + assert_eq!(streaming_to_delta(&text), Vec::::new()); + } +} diff --git a/crates/flowstate-document/src/table_topology.rs b/crates/flowstate-document/src/table_topology.rs new file mode 100644 index 00000000..5cbcc7fd --- /dev/null +++ b/crates/flowstate-document/src/table_topology.rs @@ -0,0 +1,406 @@ +//! Pure table-topology normalization (§P2b, FS-010). +//! +//! Turns the raw set of durable-id cell records read from canonical Loro +//! (`row_order` / `column_order` / `cells_by_id`) into a full, well-formed grid +//! plus a list of [`TableTopologyDefect`]s describing what had to be repaired. +//! +//! This module is deliberately **pure**: it depends only on `std`, the durable +//! id types ([`RowId`], [`ColumnId`], [`CellId`]) and their canonical +//! [`CellId::from_coordinate`] mix. It has no Loro, GPUI, or I/O dependency, so +//! the identical normalization runs on both sides of the refactor — the +//! read-side projector (which builds [`RawCellRecord`]s from Loro) and the +//! runtime repair transaction (which turns the reported defects into convergent +//! canonical mutations). Every output is a deterministic function of the inputs, +//! so two peers holding the same canonical state produce byte-identical grids +//! and defect lists. + +use std::collections::hash_map::Entry; + +use gpui_flowtext::{CellId, ColumnId, RowId}; +// §perf: Fx hashing for these u128 durable-id keyed maps/sets. The keys are trusted +// canonical ids (not attacker-controlled) and the map is documented as never iterated, +// so switching from SipHash is faster and does not affect determinism. +use rustc_hash::{FxHashMap, FxHashSet}; + +/// A single cell record as read from canonical `cells_by_id` (§P2b). +/// +/// The projector builds one of these per cell container actually present in +/// Loro; [`normalize`] reconciles them against the ordered `row_order` / +/// `column_order` id lists. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RawCellRecord { + /// Durable id of the row this cell claims to occupy. + pub row_id: RowId, + /// Durable id of the column this cell claims to occupy. + pub column_id: ColumnId, + /// Durable id of the cell container itself, as read (never recomputed). + pub cell_id: CellId, + /// Row span as stored, before clamping to the grid edge. + pub row_span: u16, + /// Column span as stored, before clamping to the grid edge. + pub col_span: u16, +} + +/// A normalized cell occupying one `(row, column)` coordinate of the full grid. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NormalizedCell { + /// Durable id of the row this cell occupies. + pub row_id: RowId, + /// Durable id of the column this cell occupies. + pub column_id: ColumnId, + /// Durable id of the cell container. Left as read for a present cell; set via + /// [`CellId::from_coordinate`] for a synthesized one. + pub cell_id: CellId, + /// Row span, clamped so it never runs past the last row. + pub row_span: u16, + /// Column span, clamped so it never runs past the last column. + pub col_span: u16, + /// True when this cell was synthesized to fill a missing coordinate (had no + /// [`RawCellRecord`]). + pub synthesized: bool, +} + +/// A well-formedness anomaly found while normalizing a table grid (§P2b, +/// FS-010). +/// +/// Each variant names the `(row, column)` coordinate it concerns so the runtime +/// repair transaction can key an idempotent, convergent canonical mutation on +/// the stable durable ids. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TableTopologyDefect { + /// No [`RawCellRecord`] existed for this coordinate; a cell was synthesized. + MissingCell { row_id: RowId, column_id: ColumnId }, + /// Two or more raw records claimed this coordinate; one was kept + /// deterministically and the rest dropped. + DuplicateCoordinate { row_id: RowId, column_id: ColumnId }, + /// A present cell's `row_span` / `col_span` had to be clamped to the grid + /// edge. + InvalidSpan { row_id: RowId, column_id: ColumnId }, + /// A raw record named a `row_id` or `column_id` absent from the order lists; + /// it was dropped. + OrphanCell { row_id: RowId, column_id: ColumnId }, +} + +/// The result of [`normalize`]: a full grid plus the defects that were repaired. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NormalizedTable { + /// Row-major full grid: exactly one [`NormalizedCell`] for every + /// `(row_id, column_id)` coordinate, ordered by `row_ids` then `column_ids` + /// (the caller-supplied order). Always `row_ids.len() * column_ids.len()` + /// long. + pub cells: Vec, + /// Defects found, in the deterministic order documented on [`normalize`]. + pub defects: Vec, +} + +/// Normalize a raw table grid into a full, well-formed grid plus a defect list +/// (§P2b, FS-010). +/// +/// `row_ids` / `column_ids` are the ordered durable ids from `row_order` / +/// `column_order`; `raw` is every cell record read from `cells_by_id`. +/// +/// Rules — all deterministic, so two peers with the same inputs produce +/// byte-identical output: +/// * **Missing:** for every `(row, column)` coordinate with no raw record, +/// synthesize a [`NormalizedCell`] with +/// `cell_id = CellId::from_coordinate(row, column)`, `row_span` = 1, +/// `col_span` = 1, `synthesized` = true, and record a `MissingCell` defect. +/// * **Orphan:** a raw record whose `row_id` or `column_id` is absent from the +/// order lists is dropped and reported as an `OrphanCell` defect. +/// * **Duplicate:** when two or more raw records share a coordinate, keep +/// exactly one deterministically — the smallest `cell_id.0`, tie-broken by the +/// smallest `(row_span, col_span)` — and record `DuplicateCoordinate` once. +/// * **Span clamp:** `row_span` / `col_span` are clamped to at least 1 and at +/// most the number of rows / columns remaining from this cell's position, so a +/// span never runs past the grid edge. If a clamp changed a value, record an +/// `InvalidSpan` defect. A synthesized cell always has spans 1/1 and is never +/// `InvalidSpan`. +/// +/// A present-and-kept cell's `cell_id` is left exactly as read (never +/// recomputed); only synthesized cells get [`CellId::from_coordinate`]. The +/// returned `cells` is always exactly `row_ids.len() * column_ids.len()` long. +/// +/// Defect ordering is fully deterministic: first every `OrphanCell` in `raw` +/// input order, then a row-major pass over the grid appending, per coordinate, +/// either `MissingCell` (absent) or — for a present coordinate — +/// `DuplicateCoordinate` then `InvalidSpan`. +#[must_use] +pub fn normalize(row_ids: &[RowId], column_ids: &[ColumnId], raw: &[RawCellRecord]) -> NormalizedTable { + let row_set: FxHashSet = row_ids.iter().map(|row| row.0).collect(); + let column_set: FxHashSet = column_ids.iter().map(|column| column.0).collect(); + + let mut defects: Vec = Vec::new(); + + // Single pass over `raw`: orphans are reported here (in input order, before + // any grid defect); surviving records are resolved into `chosen`. The map is + // only ever queried by coordinate below, never iterated, so output order is + // driven purely by the ordered id lists and stays deterministic regardless of + // `HashMap` iteration order. + let mut chosen: FxHashMap<(u128, u128), RawCellRecord> = FxHashMap::default(); + let mut duplicate_coords: FxHashSet<(u128, u128)> = FxHashSet::default(); + for record in raw { + if !row_set.contains(&record.row_id.0) || !column_set.contains(&record.column_id.0) { + defects.push(TableTopologyDefect::OrphanCell { + row_id: record.row_id, + column_id: record.column_id, + }); + continue; + } + let key = (record.row_id.0, record.column_id.0); + match chosen.entry(key) { + Entry::Vacant(slot) => { + slot.insert(*record); + }, + Entry::Occupied(mut slot) => { + duplicate_coords.insert(key); + // Keep the global minimum survivor key, so the choice is independent of + // the order duplicates appear in `raw`. + if survivor_key(record) < survivor_key(slot.get()) { + slot.insert(*record); + } + }, + } + } + + // Row-major grid pass: fill every coordinate and append its per-coordinate + // defects in grid order. + let mut cells: Vec = Vec::with_capacity(row_ids.len().saturating_mul(column_ids.len())); + for (row_index, row_id) in row_ids.iter().enumerate() { + let rows_remaining = row_ids.len() - row_index; + for (column_index, column_id) in column_ids.iter().enumerate() { + let columns_remaining = column_ids.len() - column_index; + let key = (row_id.0, column_id.0); + match chosen.get(&key) { + None => { + cells.push(NormalizedCell { + row_id: *row_id, + column_id: *column_id, + cell_id: CellId::from_coordinate(*row_id, *column_id), + row_span: 1, + col_span: 1, + synthesized: true, + }); + defects.push(TableTopologyDefect::MissingCell { + row_id: *row_id, + column_id: *column_id, + }); + }, + Some(record) => { + if duplicate_coords.contains(&key) { + defects.push(TableTopologyDefect::DuplicateCoordinate { + row_id: *row_id, + column_id: *column_id, + }); + } + let row_span = clamp_span(record.row_span, rows_remaining); + let col_span = clamp_span(record.col_span, columns_remaining); + if row_span != record.row_span || col_span != record.col_span { + defects.push(TableTopologyDefect::InvalidSpan { + row_id: *row_id, + column_id: *column_id, + }); + } + cells.push(NormalizedCell { + row_id: *row_id, + column_id: *column_id, + cell_id: record.cell_id, + row_span, + col_span, + synthesized: false, + }); + }, + } + } + } + + NormalizedTable { cells, defects } +} + +/// Deterministic survivor ordering among duplicate records at one coordinate: +/// smallest `cell_id`, then smallest `(row_span, col_span)`. +fn survivor_key(record: &RawCellRecord) -> (u128, u16, u16) { + (record.cell_id.0, record.row_span, record.col_span) +} + +/// Clamp a span into `1..=max_remaining` so it never runs past the grid edge. +/// `max_remaining` is the count of rows/columns from this cell's position to the +/// last one and is always at least 1. +fn clamp_span(span: u16, max_remaining: usize) -> u16 { + let ceiling = u16::try_from(max_remaining).unwrap_or(u16::MAX); + span.clamp(1, ceiling) +} + +#[cfg(test)] +mod tests { + use super::{RawCellRecord, TableTopologyDefect, normalize}; + use gpui_flowtext::{CellId, ColumnId, RowId}; + use std::collections::HashSet; + + #[test] + fn full_grid_synthesis() { + let r0 = RowId(10); + let r1 = RowId(20); + let c0 = ColumnId(30); + let c1 = ColumnId(40); + let present_00 = CellId(1000); + let present_11 = CellId(2000); + let raw = [ + RawCellRecord { + row_id: r0, + column_id: c0, + cell_id: present_00, + row_span: 1, + col_span: 1, + }, + RawCellRecord { + row_id: r1, + column_id: c1, + cell_id: present_11, + row_span: 1, + col_span: 1, + }, + ]; + + let table = normalize(&[r0, r1], &[c0, c1], &raw); + + // Full 2x2 grid, row-major: (r0,c0),(r0,c1),(r1,c0),(r1,c1). + assert_eq!(table.cells.len(), 4); + assert_eq!(table.cells[0].cell_id, present_00); + assert!(!table.cells[0].synthesized); + assert_eq!(table.cells[3].cell_id, present_11); + assert!(!table.cells[3].synthesized); + + // The two absent coordinates are synthesized with from_coordinate ids + 1/1. + let synth_01 = table.cells[1]; + assert!(synth_01.synthesized); + assert_eq!(synth_01.cell_id, CellId::from_coordinate(r0, c1)); + assert_eq!((synth_01.row_span, synth_01.col_span), (1, 1)); + let synth_10 = table.cells[2]; + assert!(synth_10.synthesized); + assert_eq!(synth_10.cell_id, CellId::from_coordinate(r1, c0)); + assert_eq!((synth_10.row_span, synth_10.col_span), (1, 1)); + + assert_eq!( + table.defects, + vec![ + TableTopologyDefect::MissingCell { row_id: r0, column_id: c1 }, + TableTopologyDefect::MissingCell { row_id: r1, column_id: c0 }, + ] + ); + } + + #[test] + fn span_clamp() { + let r0 = RowId(1); + let r1 = RowId(2); + let c0 = ColumnId(1); + let raw = [ + RawCellRecord { + row_id: r0, + column_id: c0, + cell_id: CellId(100), + row_span: 9, + col_span: 1, + }, + RawCellRecord { + row_id: r1, + column_id: c0, + cell_id: CellId(200), + row_span: 1, + col_span: 1, + }, + ]; + + let table = normalize(&[r0, r1], &[c0], &raw); + + assert_eq!(table.cells.len(), 2); + // Row 0's span of 9 clamps to the two rows remaining from position 0. + assert_eq!(table.cells[0].row_span, 2); + assert_eq!(table.cells[0].col_span, 1); + assert_eq!(table.defects, vec![TableTopologyDefect::InvalidSpan { row_id: r0, column_id: c0 }]); + } + + #[test] + fn orphan_drop() { + let r0 = RowId(1); + let c0 = ColumnId(1); + let ghost_row = RowId(999); + let raw = [RawCellRecord { + row_id: ghost_row, + column_id: c0, + cell_id: CellId(7), + row_span: 1, + col_span: 1, + }]; + + let table = normalize(&[r0], &[c0], &raw); + + // The orphan is dropped; the grid is still full via synthesis. + assert_eq!(table.cells.len(), 1); + assert!(table.cells[0].synthesized); + assert_eq!(table.cells[0].cell_id, CellId::from_coordinate(r0, c0)); + assert_eq!( + table.defects, + vec![ + TableTopologyDefect::OrphanCell { + row_id: ghost_row, + column_id: c0, + }, + TableTopologyDefect::MissingCell { row_id: r0, column_id: c0 }, + ] + ); + } + + #[test] + fn duplicate_determinism() { + let r0 = RowId(1); + let c0 = ColumnId(1); + let raw = [ + RawCellRecord { + row_id: r0, + column_id: c0, + cell_id: CellId(5000), + row_span: 1, + col_span: 1, + }, + RawCellRecord { + row_id: r0, + column_id: c0, + cell_id: CellId(3000), + row_span: 1, + col_span: 1, + }, + ]; + + let first = normalize(&[r0], &[c0], &raw); + let second = normalize(&[r0], &[c0], &raw); + assert_eq!(first, second); + + assert_eq!(first.cells.len(), 1); + // The smallest cell_id.0 survives. + assert_eq!(first.cells[0].cell_id, CellId(3000)); + assert!(!first.cells[0].synthesized); + assert_eq!( + first.defects, + vec![TableTopologyDefect::DuplicateCoordinate { row_id: r0, column_id: c0 }] + ); + } + + #[test] + fn stable_cell_uid_uniqueness() { + let rows: Vec = (1..=10).map(RowId).collect(); + let columns: Vec = (1..=10).map(ColumnId).collect(); + + let mut ids: HashSet = HashSet::new(); + for row in &rows { + for column in &columns { + let id = CellId::from_coordinate(*row, *column); + // Injective across the 10x10 distinct coordinates. + assert!(ids.insert(id), "distinct coordinates must map to distinct cell ids"); + // Stable: recomputing the same coordinate yields the identical id. + assert_eq!(id, CellId::from_coordinate(*row, *column)); + } + } + assert_eq!(ids.len(), 100); + } +} diff --git a/crates/flowstate-docx/Cargo.toml b/crates/flowstate-docx/Cargo.toml index 6179e6a5..2acce0b8 100644 --- a/crates/flowstate-docx/Cargo.toml +++ b/crates/flowstate-docx/Cargo.toml @@ -11,10 +11,18 @@ workspace = true docx-rs = { workspace = true } docxide-pdf = { workspace = true } flowstate-document = { workspace = true } +flowstate-fidelity.workspace = true gpui = { version = "0.2.2", default-features = false } hotpath = { workspace = true } +image = { version = "0.25.10", default-features = false, features = ["png", "jpeg", "gif", "bmp", "tiff", "webp"] } +imagesize = { workspace = true } lopdf = { workspace = true } +memchr = "2.8.2" quick-xml = { workspace = true } +# rdocx-oxml links quick-xml 0.37; the fragment-parallel document.xml parse +# drives its public `from_xml(reader)` entry points, so the Reader type must +# come from the same quick-xml build. +quick-xml-oxml = { package = "quick-xml", version = "0.37" } rdocx = { workspace = true } rdocx-opc = { workspace = true } rdocx-oxml = { workspace = true } diff --git a/crates/flowstate-docx/src/cleaner.rs b/crates/flowstate-docx/src/cleaner.rs index 9bbe1eaa..9cb66d78 100644 --- a/crates/flowstate-docx/src/cleaner.rs +++ b/crates/flowstate-docx/src/cleaner.rs @@ -1,4 +1,4 @@ -use std::{fs, io::Cursor, path::Path}; +use std::{fs, io::Cursor, path::Path, sync::Arc}; use rdocx_opc::OpcPackage; @@ -20,7 +20,17 @@ pub enum CleanAction { #[derive(Clone, Debug)] pub struct CleanedDocx { pub bytes: Vec, - pub main_document_xml: Option>, + // §perf-heaven T8.4: `Arc<[u8]>` (was `Vec`) — an uncompressed copy of + // document.xml is deliberately kept (parsing it repeatedly from the zip would + // re-inflate it), but shared ownership makes any downstream reuse a cheap + // refcount bump instead of another ~doc.xml-sized allocation. + pub main_document_xml: Option>, + // §act-nine A9.2: the in-memory package the cleaner already opened (unzip #1), + // carried so the structured walk's image resolution reuses it instead of + // re-inflating the zip from `bytes` (was unzip #3). `bytes` stays alongside it + // because rdocx's `from_bytes` takes packaged bytes (that inflate stays for + // now). `None` when the bytes are not a readable OPC package. + pub package: Option>, pub report: DocxCleanReport, } @@ -39,6 +49,9 @@ pub struct DocxCleanStats { pub tab_values_normalized: usize, pub section_values_normalized: usize, pub style_type_values_normalized: usize, + /// Fractional numeric attributes (e.g. Google-Docs `w:line="316.06"`) rounded to + /// integers so the strict downstream OOXML integer parse does not reject them. + pub numeric_values_normalized: usize, pub styles_normalized: usize, pub styles_removed: usize, pub paragraphs_restyled: usize, @@ -48,64 +61,120 @@ pub struct DocxCleanStats { #[hotpath::measure] pub fn clean_docx_path(path: impl AsRef) -> std::io::Result { - clean_docx_vec(fs::read(path)?) + let bytes = fs::read(path)?; + let (recompressed, main_document_xml, package, stats) = normalize_docx_formatting_values(&bytes)?; + // Reuse the caller-owned Vec untouched when nothing was normalized. + Ok(cleaned_docx(recompressed.unwrap_or(bytes), main_document_xml, package, stats)) } #[hotpath::measure] pub fn clean_docx_bytes(bytes: &[u8]) -> std::io::Result { - clean_docx_vec(bytes.to_vec()) + let (recompressed, main_document_xml, package, stats) = normalize_docx_formatting_values(bytes)?; + // §act-nine A9.2: copy the borrowed input only when normalization did NOT + // rewrite the package (the recompressed output replaces it entirely) — the + // old up-front `to_vec` copied the whole package even when it was about to be + // superseded by the re-deflated bytes. + Ok(cleaned_docx( + recompressed.unwrap_or_else(|| bytes.to_vec()), + main_document_xml, + package, + stats, + )) } #[hotpath::measure] -fn clean_docx_vec(bytes: Vec) -> std::io::Result { - let (bytes, main_document_xml, stats) = normalize_docx_formatting_values(bytes)?; - Ok(CleanedDocx { +fn cleaned_docx(bytes: Vec, main_document_xml: Option>, package: Option>, stats: DocxCleanStats) -> CleanedDocx { + CleanedDocx { bytes, main_document_xml, + package, report: DocxCleanReport { stats, actions: CLEANING_RULES, }, - }) + } } +/// `(recompressed package bytes when normalization changed anything, uncompressed +/// document.xml, the opened in-memory package, clean stats)`. +type NormalizedDocx = (Option>, Option>, Option>, DocxCleanStats); + #[hotpath::measure] -fn normalize_docx_formatting_values(bytes: Vec) -> std::io::Result<(Vec, Option>, DocxCleanStats)> { - let mut package = match OpcPackage::from_reader(Cursor::new(bytes.as_slice())) { +fn normalize_docx_formatting_values(bytes: &[u8]) -> std::io::Result { + let mut package = match OpcPackage::from_reader(Cursor::new(bytes)) { Ok(package) => package, - Err(_) => return Ok((bytes, None, DocxCleanStats::default())), + Err(_) => return Ok((None, None, None, DocxCleanStats::default())), }; let main_document_part = package.main_document_part(); let mut stats = DocxCleanStats::default(); - for (part_name, part) in &mut package.parts { - if !part_might_contain_word_xml(part_name, part) { - continue; - } - let Ok(xml) = std::str::from_utf8(part) else { - continue; - }; - let (normalized, part_stats) = normalize_formatting_values_in_xml(xml); - if let Some(normalized) = normalized { + // §act-twelve A12.3.1: the qualifying parts (document.xml + headers/ + // footers/footnotes, ≤ ~7) normalize INDEPENDENTLY — run them on scoped + // threads (the interpreter's established overlap pattern) so the cost is + // the largest part, not the sum (~100ms serial → ~max-part on the impact + // doc). Results apply serially afterwards; determinism is unaffected + // (per-part transform, order-independent stats merge... stats merge below + // follows the collection order for exactness). + let normalized_parts: Vec<(String, Option, DocxCleanStats)> = { + let jobs: Vec<(&String, &str)> = package + .parts + .iter() + .filter(|(part_name, part)| part_might_contain_word_xml(part_name, part)) + .filter_map(|(part_name, part)| std::str::from_utf8(part).ok().map(|xml| (part_name, xml))) + .collect(); + let mut results = std::thread::scope(|scope| { + // Spawn ALL before joining any (a lazy chain would serialize the parts). + #[allow(clippy::needless_collect, reason = "spawn-all-then-join-all; lazy iteration would serialize the burst")] + let handles: Vec<_> = jobs + .into_iter() + .map(|(part_name, xml)| { + scope.spawn(move || { + let (normalized, part_stats) = normalize_formatting_values_in_xml(xml); + (part_name.clone(), normalized, part_stats) + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("docx part normalization thread")) + .collect::>() + }); + // Deterministic apply order regardless of HashMap iteration order. + results.sort_by(|a, b| a.0.cmp(&b.0)); + results + }; + for (part_name, normalized, part_stats) in normalized_parts { + if let Some(normalized) = normalized + && let Some(part) = package.parts.get_mut(&part_name) + { *part = normalized.into_bytes(); stats.merge(part_stats); } } - let main_document_xml = main_document_part + let main_document_xml: Option> = main_document_part .as_deref() .and_then(|part_name| package.get_part(part_name)) - .map(<[u8]>::to_vec); - - if !stats.has_changes() { - return Ok((bytes, main_document_xml, stats)); + .map(Arc::from); + + if std::env::var_os("FLOWSTATE_CLEAN_DEBUG").is_some() { + eprintln!("[clean-debug] has_changes={} stats={stats:?}", stats.has_changes()); + if let Some(xml) = main_document_xml.as_deref() { + let crlf = xml.windows(2).filter(|w| *w == [13u8, 10u8]).count(); + let cr_entity = xml.windows(5).filter(|w| *w == *b" ").count(); + eprintln!("[clean-debug] doc.xml crlf={crlf} cr_entity={cr_entity} len={}", xml.len()); + } } - - let mut output = Cursor::new(Vec::new()); - package - .write_to(&mut output) - .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; - Ok((output.into_inner(), main_document_xml, stats)) + let recompressed = if stats.has_changes() { + let mut output = Cursor::new(Vec::new()); + package + .write_to(&mut output) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + Some(output.into_inner()) + } else { + None + }; + Ok((recompressed, main_document_xml, Some(Arc::new(package)), stats)) } #[hotpath::measure] @@ -233,6 +302,20 @@ fn normalize_formatting_tag(tag: &str) -> (Option, DocxCleanStats) { &mut stats.border_values_normalized, ); }, + // Fractional-twip integers (Google Docs exports write `w:line`/`w:before`/ + // `w:after`/indents/sizes as floats like "316.06"): round to an integer so the + // strict downstream OOXML integer parse (`rdocx-oxml`) does not reject them. + "spacing" => { + for attr in ["line", "before", "after"] { + normalize_numeric_attr(tag, &mut normalized, attr, &mut stats.numeric_values_normalized); + } + }, + "ind" => { + for attr in ["left", "right", "start", "end", "firstLine", "hanging"] { + normalize_numeric_attr(tag, &mut normalized, attr, &mut stats.numeric_values_normalized); + } + }, + "sz" | "szCs" => normalize_numeric_attr(tag, &mut normalized, "val", &mut stats.numeric_values_normalized), _ => {}, } @@ -252,6 +335,36 @@ fn tag_local_name(tag: &str) -> Option<&str> { (!name.is_empty()).then_some(name) } +/// Round a fractional numeric attribute (e.g. `w:line="316.0615539550781"` from a +/// Google Docs export) to the nearest integer, in place, so the strict OOXML integer +/// parse downstream does not fail with `invalid digit`. No-op when the value is +/// already a valid integer or is not numeric at all. Chains through `normalized_tag` +/// like [`normalize_attr`] so multiple attrs on one tag compose. +fn normalize_numeric_attr(original_tag: &str, normalized_tag: &mut Option, attr_name: &str, count: &mut usize) { + let tag = normalized_tag.as_deref().unwrap_or(original_tag); + let Some((value_start, value_end)) = attr_value_range(tag, attr_name) else { + return; + }; + let value = &tag[value_start..value_end]; + if value.parse::().is_ok() { + return; // already a valid integer + } + let Ok(number) = value.parse::() else { + return; // non-numeric (e.g. "auto") — leave for the enum/value normalizers + }; + #[allow( + clippy::cast_possible_truncation, + reason = "twip/half-point attributes are small; a rounded fractional value cannot overflow i64" + )] + let rounded = number.round() as i64; + let mut normalized = String::with_capacity(tag.len()); + normalized.push_str(&tag[..value_start]); + normalized.push_str(&rounded.to_string()); + normalized.push_str(&tag[value_end..]); + *normalized_tag = Some(normalized); + *count += 1; +} + #[hotpath::measure] fn normalize_attr( original_tag: &str, @@ -321,14 +434,13 @@ fn attr_value_range(tag: &str, target_attr_name: &str) -> Option<(usize, usize)> #[hotpath::measure] fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { - !needle.is_empty() - && haystack - .windows(needle.len()) - .any(|window| window == needle) + // §perf: memchr's SIMD substring search is O(n) vs the O(n·m) `windows` scan. + // The empty-needle guard preserves the prior `false` result (memmem::find + // returns Some(0) for an empty needle). + !needle.is_empty() && memchr::memmem::find(haystack, needle).is_some() } impl DocxCleanStats { - #[hotpath::measure] const fn has_changes(self) -> bool { self.underline_values_normalized + self.highlight_values_normalized @@ -337,10 +449,10 @@ impl DocxCleanStats { + self.tab_values_normalized + self.section_values_normalized + self.style_type_values_normalized + + self.numeric_values_normalized > 0 } - #[hotpath::measure] const fn merge(&mut self, other: Self) { self.underline_values_normalized += other.underline_values_normalized; self.highlight_values_normalized += other.highlight_values_normalized; @@ -349,6 +461,7 @@ impl DocxCleanStats { self.tab_values_normalized += other.tab_values_normalized; self.section_values_normalized += other.section_values_normalized; self.style_type_values_normalized += other.style_type_values_normalized; + self.numeric_values_normalized += other.numeric_values_normalized; } } diff --git a/crates/flowstate-docx/src/exporter.rs b/crates/flowstate-docx/src/exporter.rs index 93839ecf..a1b6fce4 100644 --- a/crates/flowstate-docx/src/exporter.rs +++ b/crates/flowstate-docx/src/exporter.rs @@ -1,7 +1,13 @@ mod blocks; mod formatting; +mod omml_export; mod package; +mod sections; mod styles; +mod xml_postprocess; + +#[cfg(test)] +mod fidelity_tests; use std::{ io::{self, Cursor}, @@ -9,12 +15,46 @@ use std::{ }; use docx_rs::Docx; -use flowstate_document::Document; +use flowstate_document::{Block, DocumentProjection}; +use flowstate_fidelity::FidelityClass; + +use self::{ + blocks::add_block, formatting::docx_fonts, package::write_recompressed_docx, sections::SectionContext, styles::add_flowstate_styles, + xml_postprocess::SideChannel, +}; -use self::{blocks::add_block, formatting::docx_fonts, package::write_recompressed_docx, styles::add_flowstate_styles}; +/// A non-fatal export degradation. Currently records image assets that could not +/// be embedded and were exported as descriptive text instead (FS-129). +pub type ExportWarning = String; + +#[hotpath::measure] +pub fn write_docx(path: impl AsRef, document: &DocumentProjection) -> io::Result<()> { + write_docx_with_report(path, document).map(|_warnings| ()) +} +/// Like [`write_docx`] but returns the structured warnings gathered while +/// building the package (e.g. un-embeddable image assets). #[hotpath::measure] -pub fn write_docx(path: impl AsRef, document: &Document) -> io::Result<()> { +pub fn write_docx_with_report(path: impl AsRef, document: &DocumentProjection) -> io::Result> { + // Import/export fidelity: record the shape of the projection entering DOCX + // export. The closure only runs when tracing is enabled, so counting is free + // when off. + flowstate_fidelity::event(FidelityClass::ImportExport, "write-docx", || { + let (mut paragraphs, mut tables, mut images, mut equations) = (0_usize, 0_usize, 0_usize, 0_usize); + for block in document.blocks.iter() { + match block { + Block::Paragraph(_) => paragraphs += 1, + Block::Table(_) => tables += 1, + Block::Image(_) => images += 1, + Block::Equation(_) => equations += 1, + } + } + format!( + "paragraphs={paragraphs} tables={tables} images={images} equations={equations} sections={}", + document.sections.len() + ) + }); + let path = path.as_ref(); if let Some(parent) = path .parent() @@ -22,16 +62,33 @@ pub fn write_docx(path: impl AsRef, document: &Document) -> io::Result<()> { std::fs::create_dir_all(parent)?; } + + // FS-126: resolve page sections before emitting blocks so section boundaries + // and the document-level page properties can be placed correctly. + let context = SectionContext::resolve(document); let mut docx = add_flowstate_styles(Docx::new().default_fonts(docx_fonts(&document.theme)), &document.theme); - for block in document.blocks.iter() { - docx = add_block(docx, document, block, &document.theme); + docx = context.apply_document_section(docx); + + let mut side = SideChannel::default(); + // §perf-heaven T8.6: paragraph byte ranges are derived from the block tree by + // paragraph RANK (`paragraph_byte_range`, which accounts for the `\n` block + // separators), so track the running paragraph index alongside the block index. + let mut paragraph_ix = 0usize; + for (block_ix, block) in document.blocks.iter().enumerate() { + let boundary = context.boundary_section_property(block_ix); + docx = add_block(docx, document, paragraph_ix, block, &document.theme, &context, &mut side, boundary); + if matches!(block, Block::Paragraph(_)) { + paragraph_ix += 1; + } } + let mut uncompressed_package = Cursor::new(Vec::new()); docx .build() .pack(&mut uncompressed_package) .map_err(|error| io::Error::other(format!("failed to write docx package: {error}")))?; - write_recompressed_docx(path, uncompressed_package.into_inner()) + write_recompressed_docx(path, uncompressed_package.into_inner(), &side)?; + Ok(side.into_warnings()) } #[hotpath::measure] diff --git a/crates/flowstate-docx/src/exporter/blocks.rs b/crates/flowstate-docx/src/exporter/blocks.rs index ec2698eb..fafbb492 100644 --- a/crates/flowstate-docx/src/exporter/blocks.rs +++ b/crates/flowstate-docx/src/exporter/blocks.rs @@ -1,53 +1,87 @@ +use std::borrow::Cow; + use docx_rs::{ - AlignmentType, BreakType, Docx, Paragraph as DocxParagraph, Run, Shading, Table as DocxTable, TableCell as DocxTableCell, - TableRow as DocxTableRow, + AlignmentType, BreakType, Docx, Paragraph as DocxParagraph, Pic, Run, SectionProperty, Shading, Table as DocxTable, + TableCell as DocxTableCell, TableLayoutType, TableRow as DocxTableRow, VMergeType, VertAlignType, WidthType, }; use flowstate_document::{ - Block, Document, DocumentTheme, EquationBlock, HighlightStyle, ImageBlock, Paragraph, ParagraphStyle, RunSemanticStyle, RunStyles, - SOFT_LINE_BREAK, TableBlock, TableCellBlock, TableCellParagraph, document_text_slice, + Block, BlockAlignment, DocumentProjection, DocumentTheme, EquationBlock, EquationDisplay, HighlightStyle, ImageBlock, ImageSizing, Paragraph, + ParagraphStyle, RunSemanticStyle, RunStyles, SOFT_LINE_BREAK, TableBlock, TableCell, TableCellBlock, TableCellParagraph, TableColumnWidth, + VertAlign, document_text_slice, }; +use flowstate_fidelity::FidelityClass; use super::{ formatting::{apply_run_text_format, color_hex, docx_fonts}, + omml_export::latex_to_omml, + sections::{SectionContext, px_to_twips, twips_to_emu}, styles::{apply_paragraph_style, apply_semantic_run_text_border}, + xml_postprocess::SideChannel, }; #[hotpath::measure] -pub(super) fn add_block(docx: Docx, document: &Document, block: &Block, theme: &DocumentTheme) -> Docx { +pub(super) fn add_block( + docx: Docx, + document: &DocumentProjection, + paragraph_ix: usize, + block: &Block, + theme: &DocumentTheme, + context: &SectionContext, + side: &mut SideChannel, + boundary: Option, +) -> Docx { match block { - Block::Paragraph(paragraph) => docx.add_paragraph(export_document_paragraph(document, paragraph, theme)), - Block::Table(table) => docx.add_table(export_table(table, theme)), - Block::Image(image) => docx.add_paragraph(placeholder_paragraph_for_image(document, image, theme)), - Block::Equation(equation) => docx.add_paragraph(placeholder_paragraph_for_equation(equation, theme)), + Block::Paragraph(paragraph) => { + let mut docx_paragraph = export_document_paragraph(document, paragraph_ix, paragraph, theme); + // FS-126: a non-final section terminates in the last paragraph before its + // boundary, carrying that section's page properties in its `w:sectPr`. + if let Some(section_property) = boundary { + docx_paragraph = docx_paragraph.section_property(section_property); + } + docx.add_paragraph(docx_paragraph) + }, + Block::Table(table) => docx.add_table(export_table(table, theme, context)), + Block::Image(image) => docx.add_paragraph(export_image(document, image, theme, context, side)), + Block::Equation(equation) => docx.add_paragraph(omml_paragraph_for_equation(equation, theme, side)), } } #[hotpath::measure] -fn export_document_paragraph(document: &Document, paragraph: &Paragraph, theme: &DocumentTheme) -> DocxParagraph { - let text = document_text_slice(document, paragraph.byte_range.clone()); - export_paragraph_with_text(paragraph, &text, theme) +fn export_document_paragraph(document: &DocumentProjection, paragraph_ix: usize, paragraph: &Paragraph, theme: &DocumentTheme) -> DocxParagraph { + // §perf-heaven T8.6: `byte_range` is no longer a `Paragraph` field; the text + // span is derived on demand via `paragraph_byte_range` (block tree `O(log N)` + // `paragraph_start`, which includes the inter-block `\n` separators). + let text = document_text_slice(document, flowstate_document::paragraph_byte_range(document, paragraph_ix)); + export_paragraph_with_text(paragraph, &text, theme, false) } #[hotpath::measure] -fn export_table_cell_paragraph(paragraph: &TableCellParagraph, theme: &DocumentTheme) -> DocxParagraph { - export_paragraph_with_text(¶graph.paragraph, ¶graph.text, theme) +fn export_table_cell_paragraph(paragraph: &TableCellParagraph, theme: &DocumentTheme, force_bold: bool) -> DocxParagraph { + export_paragraph_with_text(¶graph.paragraph, ¶graph.text, theme, force_bold) } #[hotpath::measure] -fn export_paragraph_with_text(paragraph: &Paragraph, text: &str, theme: &DocumentTheme) -> DocxParagraph { +fn export_paragraph_with_text(paragraph: &Paragraph, text: &str, theme: &DocumentTheme, force_bold: bool) -> DocxParagraph { let mut out = apply_paragraph_style(DocxParagraph::new(), paragraph.style, theme); let mut byte = 0usize; for run in ¶graph.runs { let start = byte.min(text.len()); let end = (byte + run.len).min(text.len()).max(start); - out = add_text_run(out, &text[start..end], run.styles, paragraph.style, theme); + out = add_text_run(out, &text[start..end], run.styles, paragraph.style, theme, force_bold); byte = end; } if paragraph.runs.is_empty() && text.is_empty() { out = out.add_run(Run::new()); } if matches!(paragraph.style, flowstate_document::PARAGRAPH_POCKET | flowstate_document::PARAGRAPH_HAT) { - out = out.add_run(Run::new().add_break(BreakType::Page)); + // §act6 (round-trip fidelity): start the header on a new page via the + // `w:pageBreakBefore` PARAGRAPH PROPERTY, not a trailing page-break RUN. The + // run form reimports as body text — the importer maps EVERY `` (page + // included, structured.rs) to `SOFT_LINE_BREAK` — so it injected a spurious + // U+2028 after every pocket/hat header, breaking export->reimport idempotency + // on ~80% of the corpus. A property does not reimport as text, and a header + // beginning a new page is the correct layout regardless. + out = out.page_break_before(true); } out } @@ -59,6 +93,7 @@ fn add_text_run( styles: RunStyles, paragraph_style: ParagraphStyle, theme: &DocumentTheme, + force_bold: bool, ) -> DocxParagraph { let mut first = true; for segment in text.split(SOFT_LINE_BREAK) { @@ -68,19 +103,33 @@ fn add_text_run( styles, paragraph_style, theme, + force_bold, )); } first = false; if !segment.is_empty() { - let segment = segment.replace('\u{f8ff}', "¶"); - paragraph = paragraph.add_run(apply_run_style(Run::new().add_text(segment), styles, paragraph_style, theme)); + // §perf: `replace` always allocates a new String even when U+F8FF is + // absent (the common case). Only allocate the substituted copy when the + // sentinel is actually present; otherwise borrow the original slice. + let segment = if segment.contains('\u{f8ff}') { + Cow::Owned(segment.replace('\u{f8ff}', "¶")) + } else { + Cow::Borrowed(segment) + }; + paragraph = paragraph.add_run(apply_run_style( + Run::new().add_text(segment.as_ref()), + styles, + paragraph_style, + theme, + force_bold, + )); } } paragraph } #[hotpath::measure] -fn apply_run_style(run: Run, styles: RunStyles, paragraph_style: ParagraphStyle, theme: &DocumentTheme) -> Run { +fn apply_run_style(run: Run, styles: RunStyles, paragraph_style: ParagraphStyle, theme: &DocumentTheme, force_bold: bool) -> Run { let mut run = run.fonts(docx_fonts(theme)); run = match styles.semantic { flowstate_document::SEMANTIC_CITE => apply_semantic_run_text_border(run.style("Style13ptBold"), theme, 1), @@ -106,6 +155,24 @@ fn apply_run_style(run: Run, styles: RunStyles, paragraph_style: ParagraphStyle, if styles.strikethrough { run = run.strike(); } + // Superscript/subscript (orthogonal to the semantic style). docx-rs exposes + // this only on `RunProperty`, so set it there directly; `clone` keeps the + // partial move off `run` and is negligible on the export path. + match styles.vert_align { + VertAlign::Baseline => {}, + VertAlign::Superscript => { + run.run_property = run + .run_property + .clone() + .vert_align(VertAlignType::SuperScript); + }, + VertAlign::Subscript => { + run.run_property = run + .run_property + .clone() + .vert_align(VertAlignType::SubScript); + }, + } if let Some(highlight) = styles.highlight { run = run.shading( Shading::new().fill(color_hex(match highlight { @@ -117,39 +184,492 @@ fn apply_run_style(run: Run, styles: RunStyles, paragraph_style: ParagraphStyle, })), ); } + // FS-124: bold the header row as a visible fallback for the `w:tblHeader` + // repeat-on-page semantics injected during XML post-processing. + if force_bold { + run = run.bold(); + } run } +// -- Tables (FS-123 spans, FS-124 grid/header) ------------------------------- + #[hotpath::measure] -fn export_table(table: &TableBlock, theme: &DocumentTheme) -> DocxTable { - DocxTable::new( - table - .rows - .iter() - .map(|row| { - DocxTableRow::new( - row - .cells - .iter() - .map(|cell| { - let mut out = DocxTableCell::new(); - for block in &cell.blocks { - out = match block { - TableCellBlock::Paragraph(paragraph) => out.add_paragraph(export_table_cell_paragraph(paragraph, theme)), - TableCellBlock::Table(table) => out.add_table(export_table(table, theme)), - }; - } - out - }) - .collect(), - ) - }) - .collect(), - ) +fn export_table(table: &TableBlock, theme: &DocumentTheme, context: &SectionContext) -> DocxTable { + // FS-124 fidelity: the `w:tblHeader` marker is emitted only on the first row, + // so a header-flagged table with no rows loses its repeat-on-page header + // semantics. + flowstate_fidelity::check( + !table.style.header_row || !table.rows.is_empty(), + FidelityClass::ImportExport, + "export-dropped-header", + || "table has header_row set but no rows; w:tblHeader marker not emitted".to_string(), + ); + let content_width = context.content_width_twips.max(1); + let column_widths: Vec = table + .columns + .iter() + .map(|column| column.width.clone()) + .collect(); + let mut grid = compute_grid_twips(&column_widths, content_width); + if grid.is_empty() { + // No canonical column widths: derive an equal-width grid from the widest row + // so column spans and vertical merges still resolve. + let derived = derived_column_count(&table.rows); + if derived > 0 { + let each = (content_width / derived as i64).max(1); + grid = vec![each; derived]; + } + } + let column_count = grid.len(); + + // Track ongoing vertical merges per origin column so continuation cells can be + // synthesized in following rows (docx-rs never generates them). + let mut active: Vec> = vec![None; column_count]; + let mut rows: Vec = Vec::with_capacity(table.rows.len()); + + for (row_ix, row) in table.rows.iter().enumerate() { + let is_header = table.style.header_row && row_ix == 0; + let cells = export_row_cells(row, &grid, column_count, is_header, theme, context, &mut active); + let mut docx_row = DocxTableRow::new(cells); + if is_header { + // The only per-row `trPr` child docx-rs 0.4.20 exposes; the post-process + // pass rewrites this marker to `` (FS-124). + docx_row = docx_row.cant_split(); + } + rows.push(docx_row); + } + + let mut docx_table = DocxTable::new(rows).layout(TableLayoutType::Fixed); + if !grid.is_empty() { + docx_table = docx_table.set_grid(grid.iter().map(|width| *width as usize).collect()); + } + let total: i64 = grid.iter().sum(); + let table_width = if total > 0 { total } else { content_width }; + docx_table.width(table_width.max(1) as usize, WidthType::Dxa) +} + +#[derive(Clone, Copy)] +struct ActiveMerge { + rows_left: u16, + col_span: u16, +} + +#[hotpath::measure] +fn export_row_cells( + row: &flowstate_document::TableRow, + grid: &[i64], + column_count: usize, + is_header: bool, + theme: &DocumentTheme, + context: &SectionContext, + active: &mut [Option], +) -> Vec { + let mut cells: Vec = Vec::new(); + let mut source = row.cells.iter(); + let mut column = 0usize; + + while column < column_count { + if let Some(mut merge) = active[column] { + // Continuation cell for a vertical merge that started in an earlier row. + let span = (merge.col_span.max(1) as usize).min(column_count - column); + let width = grid_slice_width(grid, column, span); + let mut cell = DocxTableCell::new() + .add_paragraph(empty_cell_paragraph()) + .vertical_merge(VMergeType::Continue) + .width(width, WidthType::Dxa); + if span > 1 { + cell = cell.grid_span(span); + } + cells.push(cell); + merge.rows_left -= 1; + active[column] = if merge.rows_left == 0 { None } else { Some(merge) }; + column += span.max(1); + } else if let Some(source_cell) = source.next() { + let col_span = (source_cell.col_span.max(1) as usize).min(column_count - column); + let row_span = source_cell.row_span.max(1); + let width = grid_slice_width(grid, column, col_span); + let mut cell = export_table_cell(source_cell, theme, context, is_header).width(width, WidthType::Dxa); + if col_span > 1 { + cell = cell.grid_span(col_span); + } + if row_span > 1 { + cell = cell.vertical_merge(VMergeType::Restart); + active[column] = Some(ActiveMerge { + rows_left: row_span - 1, + col_span: col_span as u16, + }); + } + cells.push(cell); + column += col_span.max(1); + } else { + // Row declared fewer cells than the grid; pad so column topology stays valid. + let width = grid_slice_width(grid, column, 1); + cells.push( + DocxTableCell::new() + .add_paragraph(empty_cell_paragraph()) + .width(width, WidthType::Dxa), + ); + column += 1; + } + } + + // Repair malformed topology: append any surplus source cells rather than + // dropping their content (they lose span alignment but remain visible). + for source_cell in source { + let col_span = source_cell.col_span.max(1) as usize; + let mut cell = export_table_cell(source_cell, theme, context, is_header); + if col_span > 1 { + cell = cell.grid_span(col_span); + } + cells.push(cell); + } + + cells +} + +fn derived_column_count(rows: &[flowstate_document::TableRow]) -> usize { + rows + .iter() + .map(|row| { + row + .cells + .iter() + .map(|cell| usize::from(cell.col_span.max(1))) + .sum::() + }) + .max() + .unwrap_or(0) +} + +fn grid_slice_width(grid: &[i64], start: usize, span: usize) -> usize { + let end = (start + span).min(grid.len()); + let sum: i64 = grid + .get(start..end) + .map(|slice| slice.iter().sum()) + .unwrap_or(0); + sum.max(1) as usize +} + +#[hotpath::measure] +fn export_table_cell(cell: &TableCell, theme: &DocumentTheme, context: &SectionContext, is_header: bool) -> DocxTableCell { + let mut out = DocxTableCell::new(); + let mut emitted = false; + let mut last_was_table = false; + for block in &cell.blocks { + out = match block { + TableCellBlock::Paragraph(paragraph) => { + last_was_table = false; + emitted = true; + out.add_paragraph(export_table_cell_paragraph(paragraph, theme, is_header)) + }, + TableCellBlock::Table(table) => { + last_was_table = true; + emitted = true; + out.add_table(export_table(table, theme, context)) + }, + }; + } + // A cell must contain at least one paragraph and cannot end with a table. + if !emitted || last_was_table { + out = out.add_paragraph(empty_cell_paragraph()); + } + out } +fn empty_cell_paragraph() -> DocxParagraph { + DocxParagraph::new() +} + +/// Map canonical column widths to a DOCX grid in twips. Fixed pixels convert +/// directly; fractional and auto columns share the content width remaining after +/// fixed columns (auto counts as one fractional unit). +fn compute_grid_twips(widths: &[TableColumnWidth], content_width: i64) -> Vec { + if widths.is_empty() { + return Vec::new(); + } + let fixed_total: i64 = widths + .iter() + .filter_map(|width| match width { + TableColumnWidth::FixedPx(px) => Some(px_to_twips(*px)), + _ => None, + }) + .sum(); + let weight_sum: i64 = widths + .iter() + .map(|width| match width { + TableColumnWidth::Fraction(weight) => i64::from(*weight), + TableColumnWidth::Auto => 1, + TableColumnWidth::FixedPx(_) => 0, + }) + .sum(); + let remaining = (content_width - fixed_total).max(0); + widths + .iter() + .map(|width| match width { + TableColumnWidth::FixedPx(px) => px_to_twips(*px).max(1), + TableColumnWidth::Fraction(weight) => proportional_width(remaining, i64::from(*weight), weight_sum), + TableColumnWidth::Auto => proportional_width(remaining, 1, weight_sum), + }) + .collect() +} + +fn proportional_width(remaining: i64, weight: i64, weight_sum: i64) -> i64 { + if weight_sum <= 0 { + return 1; + } + ((i128::from(remaining) * i128::from(weight)) / i128::from(weight_sum)).max(1) as i64 +} + +// -- Images (FS-127 alt, FS-128 sizing, FS-129 transcode) -------------------- + #[hotpath::measure] -fn placeholder_paragraph_for_image(document: &Document, image: &ImageBlock, theme: &DocumentTheme) -> DocxParagraph { +fn export_image( + document: &DocumentProjection, + image: &ImageBlock, + theme: &DocumentTheme, + context: &SectionContext, + side: &mut SideChannel, +) -> DocxParagraph { + // FS-127 fidelity: an image caption has no OOXML representation in this + // exporter, so a present caption is silently dropped on export. + flowstate_fidelity::check(image.caption.is_none(), FidelityClass::ImportExport, "export-dropped-caption", || { + format!("image asset {:?} caption is not written to docx export", image.asset_id) + }); + if let Some(asset) = document.assets.assets.get(&image.asset_id) + && !asset.is_loading_placeholder() + { + match prepare_embeddable_image(asset.bytes.as_ref()) { + EmbedResult::Ready(bytes) => { + let (width_px, height_px) = image_dimensions(&bytes, image); + let mut pic = Pic::new_with_dimensions(bytes, width_px, height_px); + if let Some((width_emu, height_emu)) = fit_width_emu(image, context, width_px, height_px) { + pic = pic.size(width_emu, height_emu); + } + // FS-127: alt text is written onto `wp:docPr` (descr + title) during the + // post-process pass; embeddable images keep no bracketed text fallback. + let alt = image_alt_text(document, image); + side.push_image(alt.clone(), alt); + return aligned_paragraph(image.alignment).add_run(Run::new().fonts(docx_fonts(theme)).add_image(pic)); + }, + EmbedResult::Fallback(reason) => side.push_warning(reason), + } + } + // §A11.9: a genuinely-LINKED image (external URL, no embeddable asset bytes) + // exports as an inline drawing whose blip carries `r:link` to an external + // `TargetMode="External"` relationship. docx-rs has no `r:link` API, so a + // sentinel run is emitted here and the post-process pass swaps it for the + // full drawing (the equation-injection seam); the package writer adds the + // relationship. NOT `push_image` — that ledger pairs with the docx-rs-emitted + // `wp:docPr` literals in document order, and no such literal exists here (the + // injected drawing carries its own `docPr` with the alt text baked in). + if let Some(url) = image.external_url.as_deref().filter(|url| !url.is_empty()) { + let (width_px, height_px) = match image.sizing { + ImageSizing::Fixed { width_px, height_px } => (width_px.max(1), height_px.unwrap_or(width_px).max(1)), + // No bytes to sniff intrinsic dimensions from; match the embedded path's + // unknown-size default. + ImageSizing::Intrinsic | ImageSizing::FitWidth => (640, 480), + }; + let (width_emu, height_emu) = fit_width_emu(image, context, width_px, height_px).unwrap_or((px_to_emu(width_px), px_to_emu(height_px))); + let alt = image_alt_text(document, image); + let sentinel = side.push_linked_image(url, alt.as_deref(), width_emu, height_emu); + return aligned_paragraph(image.alignment).add_run(Run::new().add_text(sentinel)); + } + // FS-127 fidelity: reaching the text fallback means no `` (and thus + // no `wp:docPr descr`) is emitted, so an image carrying alt text loses its + // accessible descr sentinel on export. + if flowstate_fidelity::enabled() { + let has_alt = image_alt_text(document, image).is_some(); + flowstate_fidelity::check(!has_alt, FidelityClass::ImportExport, "export-dropped-alt", || { + format!( + "image asset {:?} with alt text fell back to text; docPr descr not emitted", + image.asset_id + ) + }); + } + image_text_fallback(document, image, theme) +} + +enum EmbedResult { + Ready(Vec), + Fallback(String), +} + +/// Prepare image bytes for embedding. PNG/JPEG embed as-is (docx-rs stores PNG +/// media and transcodes on build); other decodable raster formats (GIF/BMP/ +/// TIFF/WebP) are transcoded to PNG here so decode failures degrade to a visible +/// fallback instead of being silently dropped. SVG and undecodable data fall +/// back with a structured warning (FS-129). +fn prepare_embeddable_image(bytes: &[u8]) -> EmbedResult { + if is_png(bytes) || is_jpeg(bytes) { + return EmbedResult::Ready(bytes.to_vec()); + } + // §perf-heaven T8.8: Windows metafiles (EMF/WMF) are VALID docx image parts + // but the `image` crate can't decode them — they used to fall back to + // `[Picture N]` body text (the dominant docx round-trip residual, +chars). + // Embed the original bytes as-is; docx-rs writes them to `word/media/imageN.png` + // and the recompress pass (`package::write_recompressed_docx`) sniffs the magic + // and gives the part its real `.emf`/`.wmf` extension + content-type + rel — so + // the image round-trips LOSSLESSLY as an object instead of corrupting the text. + if is_metafile(bytes).is_some() { + return EmbedResult::Ready(bytes.to_vec()); + } + // §act-ten A9.6: some real-world docx media parts are HEADERLESS raw DIBs + // (a bare BITMAPINFOHEADER, e.g. 44-byte 1x1 tracking pixels) — valid Word + // media, but no `BM` magic, so `image` can't sniff them and they degraded to + // the bracketed alt-text fallback (2 of the 4 corpus roundtrip residuals). + // Prepend the 14-byte BITMAPFILEHEADER to make a decodable BMP and transcode. + let dib_shimmed; + let bytes = match shim_headerless_dib(bytes) { + Some(bmp) => { + dib_shimmed = bmp; + dib_shimmed.as_slice() + }, + None => bytes, + }; + match image::load_from_memory(bytes) { + Ok(decoded) => { + let mut buffer = std::io::Cursor::new(Vec::new()); + match decoded.write_to(&mut buffer, image::ImageFormat::Png) { + Ok(()) => EmbedResult::Ready(buffer.into_inner()), + Err(error) => EmbedResult::Fallback(format!("failed to transcode image to PNG for DOCX embedding: {error}")), + } + }, + Err(error) => EmbedResult::Fallback(format!( + "unsupported image format for DOCX embedding ({error}); exported as descriptive text" + )), + } +} + +/// §act-ten A9.6: detect a headerless raw DIB (a bare `BITMAPINFOHEADER` — or +/// the ancient `BITMAPCOREHEADER` — with no `BM` file header) and wrap it into +/// a decodable BMP by prepending the 14-byte `BITMAPFILEHEADER`. Word accepts +/// such raw DIBs as media parts; the `image` crate needs the file header. +fn shim_headerless_dib(bytes: &[u8]) -> Option> { + if bytes.len() < 12 || bytes.starts_with(b"BM") { + return None; + } + let header_size = u32::from_le_bytes(bytes[0..4].try_into().ok()?); + // 40 = BITMAPINFOHEADER, 108/124 = V4/V5, 12 = BITMAPCOREHEADER. + if !matches!(header_size, 12 | 40 | 108 | 124) || bytes.len() <= header_size as usize { + return None; + } + // Plausibility: positive dimensions bounded to Word-realistic sizes, so a + // random binary that happens to start with 0x28 doesn't get mis-shimmed. + let (width, height, bit_count, palette_len) = if header_size == 12 { + let width = u16::from_le_bytes(bytes[4..6].try_into().ok()?) as i64; + let height = u16::from_le_bytes(bytes[6..8].try_into().ok()?) as i64; + let bit_count = u16::from_le_bytes(bytes[10..12].try_into().ok()?); + let palette = if bit_count <= 8 { (1usize << bit_count) * 3 } else { 0 }; + (width, height, bit_count, palette) + } else { + if bytes.len() < 40 { + return None; + } + let width = i32::from_le_bytes(bytes[4..8].try_into().ok()?) as i64; + let height = (i32::from_le_bytes(bytes[8..12].try_into().ok()?) as i64).abs(); + let bit_count = u16::from_le_bytes(bytes[14..16].try_into().ok()?); + let colors_used = u32::from_le_bytes(bytes[32..36].try_into().ok()?) as usize; + let palette = if bit_count <= 8 { + (if colors_used == 0 { 1usize << bit_count } else { colors_used }) * 4 + } else { + 0 + }; + (width, height, bit_count, palette) + }; + if width <= 0 || height <= 0 || width > 30_000 || height > 30_000 { + return None; + } + if !matches!(bit_count, 1 | 4 | 8 | 16 | 24 | 32) { + return None; + } + let pixel_offset = 14usize + header_size as usize + palette_len; + let mut bmp = Vec::with_capacity(14 + bytes.len()); + bmp.extend_from_slice(b"BM"); + bmp.extend_from_slice(&(14u32 + bytes.len() as u32).to_le_bytes()); + bmp.extend_from_slice(&[0, 0, 0, 0]); + bmp.extend_from_slice(&(pixel_offset as u32).to_le_bytes()); + bmp.extend_from_slice(bytes); + Some(bmp) +} + +/// The docx media extension for a Windows metafile, by magic bytes: EMF (the +/// signature `" EMF"` at offset 40) or WMF (placeable `0xD7CDC69A`, or a standard +/// `META_HEADER` `0x0001`/`0x0002` with mtype `0x0009`). Used by both the export +/// pass-through and the recompress rename. §perf-heaven T8.8. +pub(crate) fn is_metafile(bytes: &[u8]) -> Option<&'static str> { + if bytes.len() >= 44 && bytes[0..4] == [0x01, 0x00, 0x00, 0x00] && bytes[40..44] == *b" EMF" { + return Some("emf"); + } + if bytes.starts_with(&[0xD7, 0xCD, 0xC6, 0x9A]) || bytes.starts_with(&[0x01, 0x00, 0x09, 0x00]) || bytes.starts_with(&[0x02, 0x00, 0x09, 0x00]) + { + return Some("wmf"); + } + None +} + +/// Pixels to EMU at the `DrawingML` convention of 9,525 EMU per 96-dpi pixel. +fn px_to_emu(px: u32) -> u32 { + px.saturating_mul(9_525) +} + +fn is_png(bytes: &[u8]) -> bool { + bytes.starts_with(&[137, 80, 78, 71, 13, 10, 26, 10]) +} + +fn is_jpeg(bytes: &[u8]) -> bool { + bytes.starts_with(&[0xFF, 0xD8, 0xFF]) +} + +/// Intrinsic (or explicitly fixed) pixel dimensions used to seed the `DrawingML` +/// extent. Fit-width overrides the extent afterwards via [`fit_width_emu`]. +fn image_dimensions(bytes: &[u8], image: &ImageBlock) -> (u32, u32) { + if let ImageSizing::Fixed { width_px, height_px } = image.sizing { + return (width_px.max(1), height_px.unwrap_or(width_px).max(1)); + } + imagesize::blob_size(bytes) + .map(|size| (size.width.max(1) as u32, size.height.max(1) as u32)) + .unwrap_or((640, 480)) +} + +/// For [`ImageSizing::FitWidth`], scale the intrinsic aspect ratio to the section +/// content width and return the EMU extent to apply via [`Pic::size`]. +fn fit_width_emu(image: &ImageBlock, context: &SectionContext, width_px: u32, height_px: u32) -> Option<(u32, u32)> { + if !matches!(image.sizing, ImageSizing::FitWidth) { + return None; + } + let width_emu = twips_to_emu(context.content_width_twips.max(1)).max(1); + let height_emu = if width_px == 0 { + width_emu + } else { + ((u64::from(width_emu) * u64::from(height_px.max(1))) / u64::from(width_px.max(1))) as u32 + }; + Some((width_emu, height_emu.max(1))) +} + +fn image_alt_text(document: &DocumentProjection, image: &ImageBlock) -> Option { + let alt = image.alt_text.trim(); + if !alt.is_empty() { + return Some(alt.to_string()); + } + document + .assets + .assets + .get(&image.asset_id) + .and_then(|asset| asset.original_name.as_ref()) + .map(|name| name.trim().to_string()) + .filter(|name| !name.is_empty()) +} + +fn aligned_paragraph(alignment: BlockAlignment) -> DocxParagraph { + match alignment { + BlockAlignment::Left => DocxParagraph::new(), + BlockAlignment::Center => DocxParagraph::new().align(AlignmentType::Center), + BlockAlignment::Right => DocxParagraph::new().align(AlignmentType::Right), + } +} + +fn image_text_fallback(document: &DocumentProjection, image: &ImageBlock, theme: &DocumentTheme) -> DocxParagraph { let mut text = image.alt_text.to_string(); if text.trim().is_empty() && let Some(asset) = document.assets.assets.get(&image.asset_id) @@ -160,7 +680,7 @@ fn placeholder_paragraph_for_image(document: &Document, image: &ImageBlock, them if text.trim().is_empty() { text = "Image".to_string(); } - DocxParagraph::new().add_run( + aligned_paragraph(image.alignment).add_run( Run::new() .fonts(docx_fonts(theme)) .italic() @@ -168,12 +688,38 @@ fn placeholder_paragraph_for_image(document: &Document, image: &ImageBlock, them ) } +// -- Equations (FS-125 OMML) ------------------------------------------------- + #[hotpath::measure] -fn placeholder_paragraph_for_equation(equation: &EquationBlock, theme: &DocumentTheme) -> DocxParagraph { +fn omml_paragraph_for_equation(equation: &EquationBlock, theme: &DocumentTheme, side: &mut SideChannel) -> DocxParagraph { + let display = matches!(equation.display, EquationDisplay::Display); + if let Some(omml) = latex_to_omml(&equation.source, display) { + // Emit a placeholder run; the post-process pass swaps the enclosing run for + // the OMML fragment (docx-rs 0.4.20 cannot express `m:oMath`). + let sentinel = side.push_equation(omml); + let paragraph = if display { + DocxParagraph::new().align(AlignmentType::Center) + } else { + DocxParagraph::new() + }; + return paragraph.add_run(Run::new().add_text(sentinel)); + } + // FS-125 fidelity: the OMML conversion failed, so the equation degrades to the + // documented bracketed text fallback. Record the degradation and assert the + // fallback still carries the source (an equation must never vanish entirely). + let fallback = format!("[Equation: {}]", equation.source); + if flowstate_fidelity::enabled() { + flowstate_fidelity::event(FidelityClass::ImportExport, "equation-omml-fallback", || { + format!("latex not convertible to OMML; using text fallback: {:?}", equation.source) + }); + flowstate_fidelity::check(!fallback.is_empty(), FidelityClass::ImportExport, "export-dropped-omml", || { + format!("equation emitted neither OMML nor a text fallback: {:?}", equation.source) + }); + } DocxParagraph::new().align(AlignmentType::Center).add_run( Run::new() .fonts(docx_fonts(theme)) .italic() - .add_text(format!("[Equation: {}]", equation.source)), + .add_text(fallback), ) } diff --git a/crates/flowstate-docx/src/exporter/fidelity_tests.rs b/crates/flowstate-docx/src/exporter/fidelity_tests.rs new file mode 100644 index 00000000..9118364a --- /dev/null +++ b/crates/flowstate-docx/src/exporter/fidelity_tests.rs @@ -0,0 +1,500 @@ +//! Structural DOCX/PDF export fidelity fixtures (FS-122..FS-130). +//! +//! Each test builds a [`DocumentProjection`] exercising one finding, writes a +//! real `.docx`, unzips `word/document.xml`, and asserts on OOXML *structure* +//! (element presence) rather than exact bytes. A PDF smoke test drives the +//! shared `write_pdf` path. These run in-crate so they can use the exporter's +//! own `zip` / `image` dependencies to read entries and synthesize assets. +#![allow( + clippy::default_trait_access, + reason = "section-attr fixture types are not nameable from this crate; `Default::default()` is the only way to build them" +)] +#![allow( + clippy::case_sensitive_file_extension_comparisons, + reason = "docx-rs writes media parts as deterministically lowercase .png; the fixtures assert that exact output" +)] + +use std::{ + io::Read, + path::PathBuf, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, +}; + +use flowstate_document::{ + AssetId, AssetRecord, CellId, ColumnId, DocumentProjection, DocumentSection, InputBlock, InputBlockAlignment, InputEquationBlock, + InputEquationDisplay, InputEquationSyntax, InputImageBlock, InputImageSizing, InputParagraph, InputRun, InputTableBlock, InputTableCell, + InputTableCellBlock, InputTableColumn, InputTableColumnWidth, InputTableRow, InputTableStyle, ParagraphStyle, RowId, RunStyles, SectionId, + SectionKind, document_from_input_blocks, flowstate_document_theme, +}; + +use crate::{write_docx, write_pdf}; + +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn temp_path(extension: &str) -> PathBuf { + let sequence = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!("flowstate-fidelity-{}-{sequence}.{extension}", std::process::id())) +} + +fn read_zip_entry(path: &std::path::Path, name: &str) -> String { + let file = std::fs::File::open(path).expect("open exported docx"); + let mut archive = zip::ZipArchive::new(file).expect("read docx zip"); + let mut entry = archive + .by_name(name) + .unwrap_or_else(|_| panic!("missing zip entry {name}")); + let mut text = String::new(); + entry.read_to_string(&mut text).expect("read zip entry"); + text +} + +fn zip_entry_names(path: &std::path::Path) -> Vec { + let file = std::fs::File::open(path).expect("open exported docx"); + let mut archive = zip::ZipArchive::new(file).expect("read docx zip"); + (0..archive.len()) + .filter_map(|index| { + archive + .by_index(index) + .ok() + .map(|entry| entry.name().to_string()) + }) + .collect() +} + +fn paragraph_block(text: &str) -> InputBlock { + InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text: text.to_string(), + styles: RunStyles::default(), + }], + }) +} + +fn text_cell(row_id: RowId, column_id: ColumnId, text: &str, row_span: u16, col_span: u16) -> InputTableCell { + InputTableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text: text.to_string(), + styles: RunStyles::default(), + }], + })], + row_span, + col_span, + } +} + +fn encode_image(format: image::ImageFormat) -> Vec { + let buffer = image::RgbImage::from_pixel(4, 3, image::Rgb([200u8, 60, 60])); + let mut cursor = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgb8(buffer) + .write_to(&mut cursor, format) + .expect("encode fixture image"); + cursor.into_inner() +} + +fn insert_asset(document: &mut DocumentProjection, id: AssetId, mime: &'static str, name: &'static str, bytes: Vec) { + let content_hash = AssetRecord::stable_content_hash(&bytes); + document.assets.assets.insert( + id, + AssetRecord { + id, + mime_type: mime.into(), + original_name: Some(name.into()), + content_hash, + bytes: Arc::new(bytes), + }, + ); +} + +/// §act-eleven C7: a headerless raw DIB media part (a bare BITMAPINFOHEADER, +/// the 44-byte tracking-pixel shape — 2 of the 4 old corpus residuals) must +/// export as an EMBEDDED image via the `BM`-header shim, not degrade to the +/// bracketed alt-text fallback. +#[test] +fn headerless_dib_asset_exports_as_an_image_not_bracket_text() { + // A 1x1 32-bit headerless DIB: BITMAPINFOHEADER (40 bytes) + one BGRA pixel. + let mut dib = Vec::new(); + dib.extend_from_slice(&40u32.to_le_bytes()); // biSize + dib.extend_from_slice(&1i32.to_le_bytes()); // biWidth + dib.extend_from_slice(&1i32.to_le_bytes()); // biHeight + dib.extend_from_slice(&1u16.to_le_bytes()); // biPlanes + dib.extend_from_slice(&32u16.to_le_bytes()); // biBitCount + dib.extend_from_slice(&[0u8; 24]); // compression..colors-important + dib.extend_from_slice(&[10u8, 20, 30, 255]); // one pixel + assert_eq!(dib.len(), 44, "fixture must be the 44-byte field shape"); + + let mut document = document_from_input_blocks( + flowstate_document_theme(), + vec![ + InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text: "before".into(), + styles: RunStyles::default(), + }], + }), + InputBlock::Image(InputImageBlock { + asset_id: AssetId(77), + alt_text: "https://pixel.example/track".into(), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Left, + external_url: None, + }), + ], + ); + insert_asset(&mut document, AssetId(77), "image/bmp", "pixel.bmp", dib); + + let path = std::env::temp_dir().join(format!("flowstate-dib-{}.docx", std::process::id())); + write_docx(&path, &document).expect("write docx"); + let bytes = std::fs::read(&path).expect("read exported docx"); + let _ = std::fs::remove_file(&path); + + let (imported, _report) = crate::import_docx_bytes_to_loro(&bytes, "dib-shim").expect("reimport"); + let (projection, defects) = flowstate_document::document_from_loro_with_defects(&imported.doc).expect("project"); + assert!(defects.is_empty(), "reimport produced defects: {defects:?}"); + let images = projection + .blocks + .iter() + .filter(|block| matches!(block, flowstate_document::Block::Image(_))) + .count(); + assert_eq!(images, 1, "the DIB image must survive as an IMAGE block"); + assert!( + !projection + .text + .to_string() + .contains("[https://pixel.example/track]"), + "the bracketed alt-text fallback fired — the DIB shim regressed" + ); +} + +/// §act-eleven A11.9(c): an image block carrying an external URL (and no +/// embeddable asset) exports as a LINKED drawing (`a:blip r:link` + a +/// `TargetMode="External"` relationship) and survives reimport with the SAME +/// URL — no bracketed `[alt]` text fabricated anywhere in the cycle. +#[test] +fn external_url_image_roundtrips_as_linked_drawing() { + let url = "https://example.com/linked.png?season=2026&side=aff"; + let document = document_from_input_blocks( + flowstate_document_theme(), + vec![ + paragraph_block("before"), + InputBlock::Image(InputImageBlock { + asset_id: AssetId(0xA119), + alt_text: "linked alt".to_string(), + caption: None, + sizing: InputImageSizing::Fixed { + width_px: 96, + height_px: Some(72), + }, + alignment: InputBlockAlignment::Left, + external_url: Some(url.to_string()), + }), + paragraph_block("after"), + ], + ); + + let path = temp_path("docx"); + let warnings = crate::write_docx_with_report(&path, &document).expect("write docx"); + let xml = read_zip_entry(&path, "word/document.xml"); + let rels = read_zip_entry(&path, "word/_rels/document.xml.rels"); + let bytes = std::fs::read(&path).expect("read exported docx"); + let _ = std::fs::remove_file(&path); + + assert!(warnings.is_empty(), "a linked image must not warn: {warnings:?}"); + assert!(xml.contains("r:link=\"rIdFsLink1\""), "a:blip r:link missing: {xml}"); + assert!(!xml.contains("FLOWSTATE_LINKIMG"), "linked-image sentinel leaked: {xml}"); + assert!(rels.contains("TargetMode=\"External\""), "external relationship mode missing: {rels}"); + assert!( + rels.contains("Id=\"rIdFsLink1\"") && rels.contains("Target=\"https://example.com/linked.png?season=2026&side=aff\""), + "external relationship target missing/unescaped: {rels}" + ); + + let (imported, _report) = crate::import_docx_bytes_to_loro(&bytes, "linked-roundtrip").expect("reimport"); + let (projection, defects) = flowstate_document::document_from_loro_with_defects(&imported.doc).expect("project"); + assert!(defects.is_empty(), "reimport produced defects: {defects:?}"); + let images: Vec<_> = projection + .blocks + .iter() + .filter_map(|block| match block { + flowstate_document::Block::Image(image) => Some(image.clone()), + _ => None, + }) + .collect(); + assert_eq!(images.len(), 1, "the linked image must survive as ONE image block"); + assert_eq!( + images[0] + .external_url + .as_ref() + .map(|url| -> &str { url.as_ref() }), + Some(url), + "external URL must roundtrip unchanged" + ); + assert_eq!(images[0].alt_text, "linked alt", "alt text must roundtrip via the injected docPr descr"); + assert_eq!( + images[0].sizing, + flowstate_document::ImageSizing::Fixed { + width_px: 96, + height_px: Some(72) + }, + "the injected wp:extent must roundtrip the fixed sizing" + ); + assert!( + !projection.text.to_string().contains("[linked alt]"), + "the bracketed alt-text fallback fired — the linked-image export regressed" + ); +} + +#[test] +fn table_spans_grid_and_header_are_emitted() { + // FS-123 spans + FS-124 grid/header: a 2-column table with a spanning header + // cell, a row-spanned cell, and a fractional grid. + let table = InputBlock::Table(InputTableBlock { + rows: vec![ + InputTableRow { + id: RowId(1), + cells: vec![text_cell(RowId(1), ColumnId(1), "Header", 1, 2)], + }, + InputTableRow { + id: RowId(2), + cells: vec![text_cell(RowId(2), ColumnId(1), "A", 2, 1), text_cell(RowId(2), ColumnId(2), "B", 1, 1)], + }, + InputTableRow { + id: RowId(3), + cells: vec![text_cell(RowId(3), ColumnId(1), "C", 1, 1)], + }, + ], + columns: vec![ + InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::Fraction(1), + }, + InputTableColumn { + id: ColumnId(2), + width: InputTableColumnWidth::Fraction(1), + }, + ], + style: InputTableStyle { header_row: true }, + }); + let document = document_from_input_blocks(flowstate_document_theme(), vec![paragraph_block("Intro"), table]); + let path = temp_path("docx"); + write_docx(&path, &document).expect("write docx"); + let xml = read_zip_entry(&path, "word/document.xml"); + let _ = std::fs::remove_file(&path); + + assert!(xml.contains(""), "table grid missing: {xml}"); + assert!(xml.contains(""), "tblHeader missing"); + assert!(!xml.contains(""), "display equation should use oMathPara"); + assert!(xml.contains(""), "fraction OMML missing"); + assert!(!xml.contains("@@FLOWSTATE_OMML"), "equation sentinel leaked into output"); + assert!(!xml.contains("[Equation:"), "equation fell back to text"); +} + +#[test] +fn unconvertible_equation_falls_back_to_text() { + let document = document_from_input_blocks( + flowstate_document_theme(), + vec![InputBlock::Equation(InputEquationBlock { + // Unbalanced braces are treated as unconvertible. + source: "\\frac{1}{2".to_string(), + syntax: InputEquationSyntax::Latex, + display: InputEquationDisplay::Display, + })], + ); + let path = temp_path("docx"); + write_docx(&path, &document).expect("write docx"); + let xml = read_zip_entry(&path, "word/document.xml"); + let _ = std::fs::remove_file(&path); + assert!(xml.contains("[Equation:"), "expected bracketed fallback for unconvertible source"); +} + +#[test] +fn section_page_attributes_are_emitted() { + // FS-126: a single custom section maps onto the document-level sectPr. + let mut document = document_from_input_blocks(flowstate_document_theme(), vec![paragraph_block("Body")]); + let first_paragraph = document.ids.paragraph_ids[0]; + let mut section = DocumentSection { + id: SectionId(1), + parent_id: None, + kind: SectionKind::Custom(0), + heading_paragraph: None, + start_paragraph: first_paragraph, + end_paragraph_exclusive: None, + page: Some(Default::default()), + }; + if let Some(page) = section.page.as_mut() { + page.page_size.width_twips = 15_840; + page.page_size.height_twips = 12_240; + page.margins.left_twips = 720; + page.margins.right_twips = 720; + page.margins.top_twips = 1_000; + page.margins.bottom_twips = 1_100; + } + document.sections = Arc::new(vec![section]); + + let path = temp_path("docx"); + write_docx(&path, &document).expect("write docx"); + let xml = read_zip_entry(&path, "word/document.xml"); + let _ = std::fs::remove_file(&path); + + assert!(xml.contains("= 2, "expected boundary + document sectPr: {xml}"); + assert!(xml.contains("w:w=\"12240\""), "first section page width missing"); + assert!(xml.contains("w:w=\"15840\""), "final section page width missing"); +} + +#[test] +fn pdf_smoke_test_produces_pdf() { + // FS-130: the shared write_pdf path must still produce a PDF. + let document = document_from_input_blocks(flowstate_document_theme(), vec![paragraph_block("PDF smoke test paragraph.")]); + let path = temp_path("pdf"); + write_pdf(&path, &document).expect("write pdf"); + let bytes = std::fs::read(&path).expect("read pdf"); + let _ = std::fs::remove_file(&path); + assert!(bytes.starts_with(b"%PDF"), "output is not a PDF"); +} diff --git a/crates/flowstate-docx/src/exporter/omml_export.rs b/crates/flowstate-docx/src/exporter/omml_export.rs new file mode 100644 index 00000000..9d1cbec9 --- /dev/null +++ b/crates/flowstate-docx/src/exporter/omml_export.rs @@ -0,0 +1,847 @@ +//! Focused LaTeX -> Office Math (OMML) conversion for DOCX equation export +//! (FS-125). +//! +//! [`crate::interpreter::omml`] converts OMML -> LaTeX on import; this module is +//! its export mirror. It covers the same constructs (runs/text, fractions, +//! sub/superscripts, radicals, delimiters, n-ary operators, functions, accents, +//! overbars, limits and matrices) plus a Greek/operator symbol table. Anything +//! the parser cannot make sense of (empty source, unbalanced braces, a missing +//! `\right`/`\end`) yields `None`, and the caller keeps the bracketed +//! `[Equation: source]` text fallback. +//! +//! The produced string is raw OMML injected verbatim into `word/document.xml` +//! by [`crate::exporter::xml_postprocess`]; that pass also declares `xmlns:m` on +//! `w:document`. `Display` equations are wrapped in `m:oMathPara`; inline-like +//! equations produce a bare `m:oMath`. + +/// Convert LaTeX `source` to an OMML fragment. `display` selects +/// `m:oMathPara`-wrapped output. Returns `None` for unconvertible source. +pub(super) fn latex_to_omml(source: &str, display: bool) -> Option { + let trimmed = source.trim(); + if trimmed.is_empty() { + return None; + } + let tokens = tokenize(trimmed); + let mut parser = Parser { + tokens, + pos: 0, + failed: false, + }; + let nodes = parser.parse_sequence(false); + if parser.failed { + return None; + } + let body = render_nodes(&nodes); + if body.trim().is_empty() { + return None; + } + let omath = format!("{body}"); + if display { + Some(format!("{omath}")) + } else { + Some(omath) + } +} + +// -- Tokenizer --------------------------------------------------------------- + +#[derive(Clone)] +enum Token { + /// `\word` (letters) or `\` (e.g. `\{`, `\,`). + Command(String), + LBrace, + RBrace, + Sup, + Sub, + Amp, + /// `\\` row separator (matrices). + RowBreak, + Char(char), +} + +fn tokenize(source: &str) -> Vec { + let mut tokens = Vec::new(); + let mut chars = source.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.peek().copied() { + Some('\\') => { + chars.next(); + tokens.push(Token::RowBreak); + }, + Some(next) if next.is_ascii_alphabetic() => { + let mut word = String::new(); + while let Some(&peeked) = chars.peek() { + if peeked.is_ascii_alphabetic() { + word.push(peeked); + chars.next(); + } else { + break; + } + } + tokens.push(Token::Command(word)); + }, + Some(next) => { + chars.next(); + tokens.push(Token::Command(next.to_string())); + }, + None => {}, + }, + '{' => tokens.push(Token::LBrace), + '}' => tokens.push(Token::RBrace), + '^' => tokens.push(Token::Sup), + '_' => tokens.push(Token::Sub), + '&' => tokens.push(Token::Amp), + c if c.is_whitespace() => {}, + c => tokens.push(Token::Char(c)), + } + } + tokens +} + +// -- AST --------------------------------------------------------------------- + +enum Node { + /// Literal math text (already symbol-mapped; XML-escaped at render time). + Text(String), + /// A run rendered on its own (function names); never merged with neighbours. + Ident(String), + Group(Vec), + Sub(Box, Vec), + Sup(Box, Vec), + SubSup(Box, Vec, Vec), + Frac(Vec, Vec), + /// `Sqrt(optional degree, radicand)`. + Sqrt(Option>, Vec), + /// `Delim(open, close, inner)` with resolved delimiter characters. + Delim(String, String, Vec), + /// `Nary(operator, sub, sup, operand)`. + Nary(String, Vec, Vec, Vec), + /// `Accent(combining char, base)`. + Accent(String, Vec), + Bar(Vec), + /// `LimLow(base, lower limit)`. + LimLow(Vec, Vec), + /// `Matrix(optional delimiters, rows of cells)`. + Matrix(Option<(String, String)>, Vec>>), +} + +// -- Parser ------------------------------------------------------------------ + +struct Parser { + tokens: Vec, + pos: usize, + failed: bool, +} + +impl Parser { + fn peek(&self) -> Option { + self.tokens.get(self.pos).cloned() + } + + fn bump(&mut self) { + self.pos += 1; + } + + /// Parse a run of atoms until end-of-input or a closing brace. When + /// `stop_at_rbrace` is set the terminating `}` is consumed; reaching the end + /// without it (or an unmatched `}` at top level) marks the parse failed. + fn parse_sequence(&mut self, stop_at_rbrace: bool) -> Vec { + let mut nodes = Vec::new(); + loop { + match self.peek() { + None => { + if stop_at_rbrace { + self.failed = true; + } + break; + }, + Some(Token::RBrace) => { + self.bump(); + if !stop_at_rbrace { + self.failed = true; + } + break; + }, + _ => { + let Some(atom) = self.parse_atom() else { + break; + }; + let node = self.apply_scripts(atom); + nodes.push(node); + }, + } + } + nodes + } + + /// Parse a single atom without trailing scripts. Returns `None` at a stopping + /// token (`}`) or end-of-input. + fn parse_atom(&mut self) -> Option { + match self.peek()? { + Token::RBrace => None, + Token::LBrace => { + self.bump(); + Some(Node::Group(self.parse_sequence(true))) + }, + Token::Char(c) => { + self.bump(); + Some(Node::Text(c.to_string())) + }, + // Stray separators / scripts without a base degrade to nothing. + Token::Amp | Token::RowBreak | Token::Sup | Token::Sub => { + self.bump(); + Some(Node::Text(String::new())) + }, + Token::Command(name) => { + self.bump(); + Some(self.parse_command(&name)) + }, + } + } + + fn apply_scripts(&mut self, base: Node) -> Node { + let mut sub: Option> = None; + let mut sup: Option> = None; + loop { + match self.peek() { + Some(Token::Sub) if sub.is_none() => { + self.bump(); + sub = Some(self.parse_arg()); + }, + Some(Token::Sup) if sup.is_none() => { + self.bump(); + sup = Some(self.parse_arg()); + }, + _ => break, + } + } + match (sub, sup) { + (None, None) => base, + (Some(sub), None) => Node::Sub(Box::new(base), sub), + (None, Some(sup)) => Node::Sup(Box::new(base), sup), + (Some(sub), Some(sup)) => Node::SubSup(Box::new(base), sub, sup), + } + } + + /// Parse one command argument: a braced group or a single atom. + fn parse_arg(&mut self) -> Vec { + match self.peek() { + Some(Token::LBrace) => { + self.bump(); + self.parse_sequence(true) + }, + Some(_) => self.parse_atom().map(|node| vec![node]).unwrap_or_default(), + None => Vec::new(), + } + } + + fn parse_command(&mut self, name: &str) -> Node { + match name { + "frac" | "dfrac" | "tfrac" | "cfrac" => { + let num = self.parse_arg(); + let den = self.parse_arg(); + Node::Frac(num, den) + }, + "sqrt" => { + let degree = self.parse_optional_bracket(); + let radicand = self.parse_arg(); + Node::Sqrt(degree, radicand) + }, + "overline" => Node::Bar(self.parse_arg()), + "left" => { + let open = self.parse_delimiter(); + let inner = self.parse_until_right(); + let close = self.parse_delimiter(); + Node::Delim(open, close, inner) + }, + // A stray `\right` outside `\left` degrades to nothing. + "right" => Node::Text(String::new()), + "begin" => self.parse_environment(), + "end" => { + self.skip_group(); + Node::Text(String::new()) + }, + "lim" => { + if matches!(self.peek(), Some(Token::Sub)) { + self.bump(); + Node::LimLow(vec![Node::Ident("lim".to_string())], self.parse_arg()) + } else { + Node::Ident("lim".to_string()) + } + }, + "text" | "mathrm" | "mathbf" | "mathit" | "mathsf" | "mathtt" | "mathbb" | "mathcal" | "mathfrak" | "boldsymbol" | "operatorname" => { + Node::Group(self.parse_arg()) + }, + _ => { + if let Some(chr) = accent_char(name) { + return Node::Accent(chr.to_string(), self.parse_arg()); + } + if let Some(op) = nary_operator(name) { + let (sub, sup) = self.parse_nary_scripts(); + return Node::Nary(op.to_string(), sub, sup, Vec::new()); + } + if is_function(name) { + return Node::Ident(name.to_string()); + } + if let Some(symbol) = command_symbol(name) { + return Node::Text(symbol.to_string()); + } + if let Some(literal) = literal_command(name) { + return Node::Text(literal.to_string()); + } + if is_spacing(name) { + return Node::Text(" ".to_string()); + } + // Unknown command: drop it (best-effort) rather than fail the whole + // equation, so a single unsupported macro does not force text fallback. + Node::Text(String::new()) + }, + } + } + + /// Parse the `_..` / `^..` limits that immediately follow an n-ary operator. + fn parse_nary_scripts(&mut self) -> (Vec, Vec) { + let mut sub = Vec::new(); + let mut sup = Vec::new(); + loop { + match self.peek() { + Some(Token::Sub) if sub.is_empty() => { + self.bump(); + sub = self.parse_arg(); + }, + Some(Token::Sup) if sup.is_empty() => { + self.bump(); + sup = self.parse_arg(); + }, + _ => break, + } + } + (sub, sup) + } + + /// Parse an optional `[..]` degree group (for `\sqrt[n]{x}`). + fn parse_optional_bracket(&mut self) -> Option> { + if !matches!(self.peek(), Some(Token::Char('['))) { + return None; + } + self.bump(); + let mut nodes = Vec::new(); + loop { + match self.peek() { + None => break, + Some(Token::Char(']')) => { + self.bump(); + break; + }, + _ => { + let Some(atom) = self.parse_atom() else { + break; + }; + nodes.push(self.apply_scripts(atom)); + }, + } + } + Some(nodes) + } + + /// Read the delimiter following `\left` or `\right`. + fn parse_delimiter(&mut self) -> String { + match self.peek() { + Some(Token::Char(c)) => { + self.bump(); + delimiter_from_char(c) + }, + Some(Token::Command(name)) => { + self.bump(); + delimiter_from_command(&name) + }, + _ => String::new(), + } + } + + fn parse_until_right(&mut self) -> Vec { + let mut nodes = Vec::new(); + loop { + match self.peek() { + None => { + self.failed = true; + break; + }, + Some(Token::Command(name)) if name == "right" => { + self.bump(); + break; + }, + Some(Token::RBrace) => { + self.failed = true; + self.bump(); + break; + }, + _ => { + let Some(atom) = self.parse_atom() else { + break; + }; + nodes.push(self.apply_scripts(atom)); + }, + } + } + nodes + } + + fn parse_environment(&mut self) -> Node { + let env = self.read_environment_name(); + let delimiters = match env.as_str() { + "pmatrix" => Some(("(".to_string(), ")".to_string())), + "bmatrix" => Some(("[".to_string(), "]".to_string())), + "Bmatrix" => Some(("{".to_string(), "}".to_string())), + "vmatrix" => Some(("|".to_string(), "|".to_string())), + "Vmatrix" => Some(("\u{2016}".to_string(), "\u{2016}".to_string())), + _ => None, + }; + let rows = self.parse_matrix_body(); + Node::Matrix(delimiters, rows) + } + + fn read_environment_name(&mut self) -> String { + let mut name = String::new(); + if matches!(self.peek(), Some(Token::LBrace)) { + self.bump(); + } + loop { + match self.peek() { + Some(Token::RBrace) => { + self.bump(); + break; + }, + Some(Token::Char(c)) => { + name.push(c); + self.bump(); + }, + Some(Token::Command(part)) => { + name.push_str(&part); + self.bump(); + }, + _ => break, + } + } + name + } + + fn parse_matrix_body(&mut self) -> Vec>> { + let mut rows: Vec>> = Vec::new(); + let mut row: Vec> = Vec::new(); + let mut cell: Vec = Vec::new(); + loop { + match self.peek() { + None => { + self.failed = true; + break; + }, + Some(Token::Command(name)) if name == "end" => { + self.bump(); + self.skip_group(); + break; + }, + Some(Token::Amp) => { + self.bump(); + row.push(std::mem::take(&mut cell)); + }, + Some(Token::RowBreak) => { + self.bump(); + row.push(std::mem::take(&mut cell)); + rows.push(std::mem::take(&mut row)); + }, + _ => { + let Some(atom) = self.parse_atom() else { + break; + }; + cell.push(self.apply_scripts(atom)); + }, + } + } + if !cell.is_empty() || !row.is_empty() { + row.push(cell); + rows.push(row); + } + rows + } + + fn skip_group(&mut self) { + if !matches!(self.peek(), Some(Token::LBrace)) { + return; + } + self.bump(); + let mut depth = 1_usize; + while depth > 0 { + match self.peek() { + None => break, + Some(Token::LBrace) => { + depth += 1; + self.bump(); + }, + Some(Token::RBrace) => { + depth -= 1; + self.bump(); + }, + _ => self.bump(), + } + } + } +} + +// -- Renderer ---------------------------------------------------------------- + +fn render_nodes(nodes: &[Node]) -> String { + let mut out = String::new(); + let mut text = String::new(); + for node in nodes { + if let Node::Text(value) = node { + text.push_str(value); + } else { + flush_text(&mut out, &mut text); + out.push_str(&render_node(node)); + } + } + flush_text(&mut out, &mut text); + out +} + +fn flush_text(out: &mut String, text: &mut String) { + if !text.is_empty() { + out.push_str(""); + out.push_str(&escape_text(text)); + out.push_str(""); + text.clear(); + } +} + +fn render_node(node: &Node) -> String { + match node { + Node::Text(value) => format!("{}", escape_text(value)), + Node::Ident(value) => format!("{}", escape_text(value)), + Node::Group(inner) => render_nodes(inner), + Node::Sub(base, sub) => format!("{}{}", render_node(base), render_nodes(sub)), + Node::Sup(base, sup) => format!("{}{}", render_node(base), render_nodes(sup)), + Node::SubSup(base, sub, sup) => format!( + "{}{}{}", + render_node(base), + render_nodes(sub), + render_nodes(sup) + ), + Node::Frac(num, den) => format!("{}{}", render_nodes(num), render_nodes(den)), + Node::Sqrt(Some(degree), radicand) => format!( + "{}{}", + render_nodes(degree), + render_nodes(radicand) + ), + Node::Sqrt(None, radicand) => format!( + "{}", + render_nodes(radicand) + ), + Node::Delim(open, close, inner) => render_delimited(open, close, &render_nodes(inner)), + Node::Nary(operator, sub, sup, operand) => render_nary(operator, sub, sup, operand), + Node::Accent(chr, base) => format!( + "{}", + escape_attr(chr), + render_nodes(base) + ), + Node::Bar(inner) => format!( + "{}", + render_nodes(inner) + ), + Node::LimLow(base, limit) => format!( + "{}{}", + render_nodes(base), + render_nodes(limit) + ), + Node::Matrix(delimiters, rows) => { + let mut matrix = String::from(""); + for row in rows { + matrix.push_str(""); + for cell in row { + matrix.push_str(""); + matrix.push_str(&render_nodes(cell)); + matrix.push_str(""); + } + matrix.push_str(""); + } + matrix.push_str(""); + match delimiters { + Some((open, close)) => render_delimited(open, close, &matrix), + None => matrix, + } + }, + } +} + +fn render_delimited(open: &str, close: &str, inner: &str) -> String { + format!( + "{}", + escape_attr(open), + escape_attr(close), + inner + ) +} + +fn render_nary(operator: &str, sub: &[Node], sup: &[Node], operand: &[Node]) -> String { + format!( + "{}{}{}", + escape_attr(operator), + if sub.is_empty() { "1" } else { "0" }, + if sup.is_empty() { "1" } else { "0" }, + render_nodes(sub), + render_nodes(sup), + render_nodes(operand) + ) +} + +// -- Symbol / delimiter tables ----------------------------------------------- + +fn escape_text(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +fn escape_attr(value: &str) -> String { + escape_text(value).replace('"', """) +} + +fn is_spacing(name: &str) -> bool { + matches!( + name, + "," | ";" | ":" | "!" | " " | "quad" | "qquad" | "thinspace" | "medspace" | "thickspace" + ) +} + +fn literal_command(name: &str) -> Option<&'static str> { + Some(match name { + "{" => "{", + "}" => "}", + "%" => "%", + "$" => "$", + "#" => "#", + "&" => "&", + "_" => "_", + "|" => "\u{2016}", + "backslash" => "\\", + _ => return None, + }) +} + +fn is_function(name: &str) -> bool { + matches!( + name, + "sin" + | "cos" + | "tan" + | "cot" + | "sec" + | "csc" + | "arcsin" + | "arccos" + | "arctan" + | "sinh" + | "cosh" + | "tanh" + | "coth" + | "log" + | "ln" + | "lg" + | "exp" + | "det" + | "dim" + | "gcd" + | "hom" + | "ker" + | "deg" + | "arg" + | "max" + | "min" + | "sup" + | "inf" + | "limsup" + | "liminf" + | "Pr" + ) +} + +fn accent_char(name: &str) -> Option<&'static str> { + Some(match name { + "hat" | "widehat" => "\u{0302}", + "tilde" | "widetilde" => "\u{0303}", + "bar" => "\u{0304}", + "vec" => "\u{20D7}", + "dot" => "\u{0307}", + "ddot" => "\u{0308}", + "dddot" => "\u{20DB}", + "check" => "\u{030C}", + "breve" => "\u{0306}", + "acute" => "\u{0301}", + "grave" => "\u{0300}", + "mathring" => "\u{030A}", + _ => return None, + }) +} + +fn nary_operator(name: &str) -> Option<&'static str> { + Some(match name { + "sum" => "\u{2211}", + "prod" => "\u{220F}", + "coprod" => "\u{2210}", + "int" => "\u{222B}", + "iint" => "\u{222C}", + "iiint" => "\u{222D}", + "oint" => "\u{222E}", + "bigcup" => "\u{22C3}", + "bigcap" => "\u{22C2}", + "bigvee" => "\u{22C1}", + "bigwedge" => "\u{22C0}", + "bigoplus" => "\u{2A01}", + "bigotimes" => "\u{2A02}", + "bigodot" => "\u{2A00}", + "biguplus" => "\u{2A04}", + "bigsqcup" => "\u{2A06}", + _ => return None, + }) +} + +fn delimiter_from_char(c: char) -> String { + match c { + '(' => "(".to_string(), + ')' => ")".to_string(), + '[' => "[".to_string(), + ']' => "]".to_string(), + '|' => "|".to_string(), + '/' => "/".to_string(), + '.' => String::new(), + '<' => "\u{27E8}".to_string(), + '>' => "\u{27E9}".to_string(), + other => other.to_string(), + } +} + +fn delimiter_from_command(name: &str) -> String { + match name { + "{" => "{".to_string(), + "}" => "}".to_string(), + "|" | "Vert" => "\u{2016}".to_string(), + "vert" => "|".to_string(), + "langle" => "\u{27E8}".to_string(), + "rangle" => "\u{27E9}".to_string(), + "lfloor" => "\u{230A}".to_string(), + "rfloor" => "\u{230B}".to_string(), + "lceil" => "\u{2308}".to_string(), + "rceil" => "\u{2309}".to_string(), + _ => String::new(), + } +} + +#[allow(clippy::too_many_lines, reason = "A single flat symbol table is the clearest mapping form.")] +fn command_symbol(name: &str) -> Option<&'static str> { + Some(match name { + "alpha" => "\u{03B1}", + "beta" => "\u{03B2}", + "gamma" => "\u{03B3}", + "delta" => "\u{03B4}", + "epsilon" | "varepsilon" => "\u{03B5}", + "zeta" => "\u{03B6}", + "eta" => "\u{03B7}", + "theta" | "vartheta" => "\u{03B8}", + "iota" => "\u{03B9}", + "kappa" => "\u{03BA}", + "lambda" => "\u{03BB}", + "mu" => "\u{03BC}", + "nu" => "\u{03BD}", + "xi" => "\u{03BE}", + "pi" => "\u{03C0}", + "rho" => "\u{03C1}", + "sigma" => "\u{03C3}", + "tau" => "\u{03C4}", + "upsilon" => "\u{03C5}", + "phi" | "varphi" => "\u{03C6}", + "chi" => "\u{03C7}", + "psi" => "\u{03C8}", + "omega" => "\u{03C9}", + "Gamma" => "\u{0393}", + "Delta" => "\u{0394}", + "Theta" => "\u{0398}", + "Lambda" => "\u{039B}", + "Xi" => "\u{039E}", + "Pi" => "\u{03A0}", + "Sigma" => "\u{03A3}", + "Phi" => "\u{03A6}", + "Psi" => "\u{03A8}", + "Omega" => "\u{03A9}", + "times" => "\u{00D7}", + "div" => "\u{00F7}", + "pm" => "\u{00B1}", + "mp" => "\u{2213}", + "cdot" => "\u{22C5}", + "ast" => "\u{2217}", + "star" => "\u{22C6}", + "circ" => "\u{2218}", + "bullet" => "\u{2219}", + "le" | "leq" => "\u{2264}", + "ge" | "geq" => "\u{2265}", + "ne" | "neq" => "\u{2260}", + "approx" => "\u{2248}", + "equiv" => "\u{2261}", + "cong" => "\u{2245}", + "sim" => "\u{223C}", + "propto" => "\u{221D}", + "ll" => "\u{226A}", + "gg" => "\u{226B}", + "to" | "rightarrow" => "\u{2192}", + "leftarrow" => "\u{2190}", + "leftrightarrow" => "\u{2194}", + "Rightarrow" => "\u{21D2}", + "Leftarrow" => "\u{21D0}", + "Leftrightarrow" => "\u{21D4}", + "mapsto" => "\u{21A6}", + "infty" => "\u{221E}", + "partial" => "\u{2202}", + "nabla" => "\u{2207}", + "in" => "\u{2208}", + "notin" => "\u{2209}", + "ni" => "\u{220B}", + "subset" => "\u{2282}", + "subseteq" => "\u{2286}", + "supset" => "\u{2283}", + "supseteq" => "\u{2287}", + "cup" => "\u{222A}", + "cap" => "\u{2229}", + "setminus" => "\u{2216}", + "forall" => "\u{2200}", + "exists" => "\u{2203}", + "nexists" => "\u{2204}", + "emptyset" | "varnothing" => "\u{2205}", + "wedge" | "land" => "\u{2227}", + "vee" | "lor" => "\u{2228}", + "neg" | "lnot" => "\u{00AC}", + "oplus" => "\u{2295}", + "otimes" => "\u{2297}", + "odot" => "\u{2299}", + "surd" => "\u{221A}", + "angle" => "\u{2220}", + "triangle" => "\u{25B3}", + "square" => "\u{25A1}", + "perp" => "\u{22A5}", + "parallel" => "\u{2225}", + "prime" => "\u{2032}", + "ldots" | "dots" => "\u{2026}", + "cdots" => "\u{22EF}", + "vdots" => "\u{22EE}", + "ddots" => "\u{22F1}", + "aleph" => "\u{2135}", + "hbar" => "\u{210F}", + "ell" => "\u{2113}", + "Re" => "\u{211C}", + "Im" => "\u{2111}", + "wp" => "\u{2118}", + "leqslant" => "\u{2A7D}", + "geqslant" => "\u{2A7E}", + _ => return None, + }) +} diff --git a/crates/flowstate-docx/src/exporter/package.rs b/crates/flowstate-docx/src/exporter/package.rs index d79bf7b1..3dbaba9d 100644 --- a/crates/flowstate-docx/src/exporter/package.rs +++ b/crates/flowstate-docx/src/exporter/package.rs @@ -1,13 +1,77 @@ use std::{ + collections::HashMap, fs::File, - io::{self, Cursor}, + io::{self, Cursor, Read}, path::Path, }; use zip::{CompressionMethod, ZipArchive, ZipWriter, write::FileOptions}; +use super::blocks::is_metafile; +use super::xml_postprocess::{SideChannel, rewrite_document_xml}; + +/// The main document part rewritten by the post-process seam. +const DOCUMENT_PART: &str = "word/document.xml"; +const CONTENT_TYPES_PART: &str = "[Content_Types].xml"; +const DOCUMENT_RELS_PART: &str = "word/_rels/document.xml.rels"; + +/// §perf-heaven T8.8: docx-rs writes every embedded image to `word/media/*.png` +/// regardless of its real format. Scan the media parts for Windows metafiles +/// (EMF/WMF) by magic bytes and build a rename map `old_path -> new_path` giving +/// each its correct extension. Empty when there are none (the common case). +#[derive(Default)] +struct MetafileRenames { + /// `word/media/imageN.png` -> `word/media/imageN.emf` (or `.wmf`). + by_part: HashMap, + needs_emf: bool, + needs_wmf: bool, +} + +fn is_png_media_part(name: &str) -> bool { + name.starts_with("word/media/") + && std::path::Path::new(name) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("png")) +} + +fn metafile_renames(package: &[u8]) -> io::Result { + let mut archive = + ZipArchive::new(Cursor::new(package)).map_err(|error| io::Error::other(format!("failed to read generated docx package: {error}")))?; + let mut renames = MetafileRenames::default(); + for index in 0..archive.len() { + let mut entry = archive + .by_index(index) + .map_err(|error| io::Error::other(format!("failed to read generated docx entry: {error}")))?; + let name = entry.name().to_owned(); + if !is_png_media_part(&name) { + continue; + } + // Only the leading bytes are needed to sniff the magic (robust to short reads). + let mut head = Vec::with_capacity(64); + entry.by_ref().take(64).read_to_end(&mut head)?; + if let Some(ext) = is_metafile(&head) { + let renamed = format!("{}.{ext}", name.trim_end_matches(".png")); + renames.by_part.insert(name, renamed); + match ext { + "emf" => renames.needs_emf = true, + "wmf" => renames.needs_wmf = true, + _ => {}, + } + } + } + Ok(renames) +} + #[hotpath::measure] -pub(super) fn write_recompressed_docx(path: &Path, package: Vec) -> io::Result<()> { +pub(super) fn write_recompressed_docx(path: &Path, package: Vec, side: &SideChannel) -> io::Result<()> { + // §perf-heaven T8.8: metafile media parts get their real extension so EMF/WMF + // images round-trip LOSSLESSLY instead of falling back to `[Picture N]` text. + let MetafileRenames { + by_part: renames, + needs_emf, + needs_wmf, + } = metafile_renames(&package)?; + let mut archive = ZipArchive::new(Cursor::new(package)).map_err(|error| io::Error::other(format!("failed to read generated docx package: {error}")))?; let file = File::create(path)?; @@ -27,9 +91,34 @@ pub(super) fn write_recompressed_docx(path: &Path, package: Vec) -> io::Resu writer .add_directory(name, options) .map_err(|error| io::Error::other(format!("failed to write docx directory: {error}")))?; + continue; + } + // The media-part rename maps `word/media/imageN.png` -> `.emf`/`.wmf`. + let out_name = renames.get(&name).cloned().unwrap_or_else(|| name.clone()); + if name == DOCUMENT_PART { + // FS-124/125/127: route the main document part through the OOXML rewrite seam. + let mut bytes = Vec::with_capacity(entry.size() as usize); + entry.read_to_end(&mut bytes)?; + let rewritten = rewrite_document_xml(bytes, side); + write_bytes(&mut writer, &out_name, options, &rewritten)?; + } else if !renames.is_empty() && name == CONTENT_TYPES_PART { + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + write_bytes(&mut writer, &out_name, options, &rewrite_content_types(&bytes, needs_emf, needs_wmf))?; + } else if name == DOCUMENT_RELS_PART && (!renames.is_empty() || !side.linked_image_rels().is_empty()) { + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + if !renames.is_empty() { + bytes = rewrite_rels_targets(&bytes, &renames); + } + // §A11.9: append the linked images' `TargetMode="External"` relationships. + if !side.linked_image_rels().is_empty() { + bytes = inject_external_image_rels(&bytes, side.linked_image_rels()); + } + write_bytes(&mut writer, &out_name, options, &bytes)?; } else { writer - .start_file(name, options) + .start_file(out_name, options) .map_err(|error| io::Error::other(format!("failed to write docx entry: {error}")))?; io::copy(&mut entry, &mut writer)?; } @@ -39,3 +128,181 @@ pub(super) fn write_recompressed_docx(path: &Path, package: Vec) -> io::Resu .map_err(|error| io::Error::other(format!("failed to finish docx package: {error}")))?; Ok(()) } + +fn write_bytes(writer: &mut ZipWriter, name: &str, options: FileOptions, bytes: &[u8]) -> io::Result<()> { + writer + .start_file(name, options) + .map_err(|error| io::Error::other(format!("failed to write docx entry: {error}")))?; + io::Write::write_all(writer, bytes) +} + +/// Add `` content-type declarations for the metafile extensions we +/// renamed to, so Word resolves `word/media/*.emf`/`.wmf` parts. +fn rewrite_content_types(bytes: &[u8], needs_emf: bool, needs_wmf: bool) -> Vec { + let Ok(xml) = std::str::from_utf8(bytes) else { + return bytes.to_vec(); + }; + let mut inserts = String::new(); + if needs_emf && !xml.contains("Extension=\"emf\"") { + inserts.push_str(r#""#); + } + if needs_wmf && !xml.contains("Extension=\"wmf\"") { + inserts.push_str(r#""#); + } + if inserts.is_empty() { + return bytes.to_vec(); + } + xml + .replacen("", &format!("{inserts}"), 1) + .into_bytes() +} + +/// Rewrite relationship `Target`s for the renamed media parts (the rels are +/// relative to `word/`, so `word/media/imageN.png` -> target `media/imageN.png`). +/// §A11.9 hardening: the replace is scoped to `Target="…"` ATTRIBUTES — the old +/// blind substring replace could rewrite an external URL (or an id) that merely +/// contained a media path. +fn rewrite_rels_targets(bytes: &[u8], renames: &HashMap) -> Vec { + let Ok(mut xml) = std::str::from_utf8(bytes).map(str::to_owned) else { + return bytes.to_vec(); + }; + for (old_path, new_path) in renames { + let old_target = format!("Target=\"{}\"", old_path.trim_start_matches("word/")); + let new_target = format!("Target=\"{}\"", new_path.trim_start_matches("word/")); + xml = xml.replace(&old_target, &new_target); + } + xml.into_bytes() +} + +/// §A11.9: append one `TargetMode="External"` image relationship per linked +/// image to `word/_rels/document.xml.rels`. The `rIdFsLinkN` ids are allocated +/// by [`SideChannel::push_linked_image`] outside docx-rs's numeric `rIdN` range. +fn inject_external_image_rels(bytes: &[u8], rels: &[(String, String)]) -> Vec { + let Ok(xml) = std::str::from_utf8(bytes) else { + return bytes.to_vec(); + }; + use std::fmt::Write as _; + let mut inserts = String::new(); + for (relationship_id, url) in rels { + let _ = write!( + inserts, + "", + super::xml_postprocess::escape_attr(url), + ); + } + if inserts.is_empty() { + return bytes.to_vec(); + } + xml + .replacen("", &format!("{inserts}"), 1) + .into_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + + /// A minimal EMF header: `is_metafile` needs `0x01000000` at offset 0 and the + /// `" EMF"` signature at offset 40. + fn minimal_emf() -> Vec { + let mut bytes = vec![0u8; 88]; + bytes[0..4].copy_from_slice(&[0x01, 0x00, 0x00, 0x00]); + bytes[40..44].copy_from_slice(b" EMF"); + bytes + } + + fn zip_with(parts: &[(&str, Vec)]) -> Vec { + let mut buffer = Cursor::new(Vec::new()); + { + let mut writer = ZipWriter::new(&mut buffer); + let options = FileOptions::default().compression_method(CompressionMethod::Deflated); + for (name, bytes) in parts { + writer.start_file(*name, options).unwrap(); + writer.write_all(bytes).unwrap(); + } + writer.finish().unwrap() + }; + buffer.into_inner() + } + + /// §perf-heaven T8.8 NET: docx-rs writes every image to `word/media/*.png`, so a + /// metafile lands there mislabelled. `write_recompressed_docx` must sniff the + /// magic, rename the part to `.emf`, add the `emf` content-type Default, and + /// rewrite the relationship Target — so the EMF is a VALID docx image part (a + /// lossless round-trip) instead of the former `[Picture N]` text. + #[test] + fn recompress_renames_metafile_media_and_fixes_refs() { + let package = zip_with(&[ + ( + "[Content_Types].xml", + br#""#.to_vec(), + ), + ( + "word/_rels/document.xml.rels", + br#""#.to_vec(), + ), + ("word/media/image1.png", minimal_emf()), + ("word/document.xml", b"".to_vec()), + ]); + + let dir = std::env::temp_dir(); + let out = dir.join(format!("fs-emf-rezip-{}.docx", std::process::id())); + write_recompressed_docx(&out, package, &SideChannel::default()).expect("recompress"); + let bytes = std::fs::read(&out).expect("read out"); + let _ = std::fs::remove_file(&out); + + let mut archive = ZipArchive::new(Cursor::new(bytes)).expect("open out"); + let names: Vec = (0..archive.len()) + .map(|i| archive.by_index(i).unwrap().name().to_owned()) + .collect(); + assert!( + names.iter().any(|n| n == "word/media/image1.emf"), + "metafile part not renamed to .emf: {names:?}" + ); + assert!( + !names.iter().any(|n| n == "word/media/image1.png"), + "old .png media part still present: {names:?}" + ); + + let read = |archive: &mut ZipArchive>>, name: &str| -> String { + let mut entry = archive.by_name(name).unwrap(); + let mut text = String::new(); + entry.read_to_string(&mut text).unwrap(); + text + }; + assert!( + read(&mut archive, "[Content_Types].xml").contains(r#"Extension="emf""#), + "emf content-type not declared" + ); + let rels = read(&mut archive, "word/_rels/document.xml.rels"); + assert!(rels.contains("media/image1.emf"), "rel target not rewritten to .emf: {rels}"); + assert!(!rels.contains("media/image1.png"), "rel still points to .png: {rels}"); + } + + /// A non-metafile package must be recompressed byte-for-part unchanged (no + /// spurious renames on the common all-PNG case). + #[test] + fn recompress_leaves_png_media_untouched() { + let png = vec![137, 80, 78, 71, 13, 10, 26, 10, 0, 0]; + let package = zip_with(&[ + ("[Content_Types].xml", b"".to_vec()), + ("word/media/image1.png", png.clone()), + ("word/document.xml", b"".to_vec()), + ]); + let out = std::env::temp_dir().join(format!("fs-png-rezip-{}.docx", std::process::id())); + write_recompressed_docx(&out, package, &SideChannel::default()).expect("recompress"); + let bytes = std::fs::read(&out).unwrap(); + let _ = std::fs::remove_file(&out); + let mut archive = ZipArchive::new(Cursor::new(bytes)).unwrap(); + let names: Vec = (0..archive.len()) + .map(|i| archive.by_index(i).unwrap().name().to_owned()) + .collect(); + assert!( + names.iter().any(|n| n == "word/media/image1.png"), + "png media renamed unexpectedly: {names:?}" + ); + } +} diff --git a/crates/flowstate-docx/src/exporter/sections.rs b/crates/flowstate-docx/src/exporter/sections.rs new file mode 100644 index 00000000..18205047 --- /dev/null +++ b/crates/flowstate-docx/src/exporter/sections.rs @@ -0,0 +1,269 @@ +//! §11 page-structure export context (FS-126) and the shared unit converters +//! consumed by image fit-width sizing (FS-128) and table column widths (FS-124). +//! +//! A Flowstate [`DocumentProjection`] carries canonical page sections in +//! `document.sections`, each with an optional [`SectionPageAttrs`] payload +//! (`page`). docx-rs models the *final* section as document-level page +//! properties and every earlier section as a `w:sectPr` inside the last +//! paragraph of that section. This module resolves the effective attributes of +//! each section (its `page`, or the documented US-Letter defaults when absent) +//! and precomputes, per body block index, the section property that must be +//! attached at that boundary. + +use docx_rs::{Docx, PageMargin, PageNumType, PageOrientationType, PageSize, SectionProperty}; +use flowstate_document::{DocumentProjection, DocumentSection, block_ix_for_paragraph}; +use flowstate_fidelity::FidelityClass; +use rustc_hash::FxHashMap; + +/// EMUs per twip. 1 inch = 1440 twips = `914_400` EMU, so `914_400` / 1440 = 635. +const EMU_PER_TWIP: i64 = 635; +/// Twips per CSS pixel at 96 dpi. 1 inch = 1440 twips = 96 px, so 1440 / 96 = 15. +const TWIPS_PER_PX: i64 = 15; + +/// US-Letter page width in twips (8.5in). Mirrors the canonical Loro default. +const DEFAULT_PAGE_WIDTH_TWIPS: i64 = 12_240; +/// US-Letter page height in twips (11in). +const DEFAULT_PAGE_HEIGHT_TWIPS: i64 = 15_840; +/// One-inch margin in twips. +const DEFAULT_MARGIN_TWIPS: i64 = 1_440; +/// Word's default header/footer distance from the page edge (0.5in). +const DEFAULT_HEADER_FOOTER_TWIPS: i32 = 720; + +/// Convert twips to EMU, clamped to the `u32` domain docx-rs uses for extents. +pub(super) fn twips_to_emu(twips: i64) -> u32 { + twips + .max(0) + .saturating_mul(EMU_PER_TWIP) + .clamp(0, i64::from(u32::MAX)) as u32 +} + +/// Convert CSS pixels (96 dpi) to twips. +pub(super) fn px_to_twips(px: u32) -> i64 { + i64::from(px) * TWIPS_PER_PX +} + +/// Clamp a twip length into the `u32` docx-rs page-size domain. +fn twips_u32(twips: i64) -> u32 { + twips.clamp(0, i64::from(u32::MAX)) as u32 +} + +/// Effective, projection-independent page attributes for one section. Reading +/// primitive fields here avoids naming `gpui_flowtext::SectionPageAttrs`, which +/// `flowstate-docx` cannot reference directly (the crate-root name resolves to +/// the distinct `loro_schema` mirror). +#[derive(Clone, Copy)] +pub(super) struct EffectiveSection { + page_width_twips: i64, + page_height_twips: i64, + margin_top: i64, + margin_right: i64, + margin_bottom: i64, + margin_left: i64, + landscape: bool, + page_num_enabled: bool, + page_num_start: i64, +} + +const DEFAULT_SECTION: EffectiveSection = EffectiveSection { + page_width_twips: DEFAULT_PAGE_WIDTH_TWIPS, + page_height_twips: DEFAULT_PAGE_HEIGHT_TWIPS, + margin_top: DEFAULT_MARGIN_TWIPS, + margin_right: DEFAULT_MARGIN_TWIPS, + margin_bottom: DEFAULT_MARGIN_TWIPS, + margin_left: DEFAULT_MARGIN_TWIPS, + landscape: false, + page_num_enabled: false, + page_num_start: 1, +}; + +impl EffectiveSection { + fn content_width_twips(&self) -> i64 { + (self.page_width_twips - self.margin_left - self.margin_right).max(1) + } + + fn page_size(&self) -> PageSize { + let size = PageSize::new().size(twips_u32(self.page_width_twips), twips_u32(self.page_height_twips)); + if self.landscape { + size.orient(PageOrientationType::Landscape) + } else { + size + } + } + + fn page_margin(&self) -> PageMargin { + PageMargin::new() + .top(self.margin_top as i32) + .bottom(self.margin_bottom as i32) + .left(self.margin_left as i32) + .right(self.margin_right as i32) + .header(DEFAULT_HEADER_FOOTER_TWIPS) + .footer(DEFAULT_HEADER_FOOTER_TWIPS) + } + + fn page_num_type(&self) -> Option { + (self.page_num_enabled || self.page_num_start != 1).then(|| PageNumType::new().start(self.page_num_start.max(0) as u32)) + } + + /// Build a paragraph-level `w:sectPr` for a non-final section boundary. + fn section_property(&self) -> SectionProperty { + let mut property = SectionProperty::new() + .page_size(self.page_size()) + .page_margin(self.page_margin()); + if let Some(page_num_type) = self.page_num_type() { + property = property.page_num_type(page_num_type); + } + property + } +} + +/// Read the effective attributes of a section: its canonical `page` payload, or +/// the documented defaults when the projection left it unset. +#[allow( + clippy::default_trait_access, + reason = "the gpui-flowtext section-attr enum types are not nameable from this crate; comparing against `Default::default()` classifies them without a path" +)] +fn effective_from(section: &DocumentSection) -> EffectiveSection { + match §ion.page { + Some(page) => EffectiveSection { + page_width_twips: page.page_size.width_twips, + page_height_twips: page.page_size.height_twips, + margin_top: page.margins.top_twips, + margin_right: page.margins.right_twips, + margin_bottom: page.margins.bottom_twips, + margin_left: page.margins.left_twips, + // Both enums derive `Default` (Portrait / None), so comparing against the + // inferred default lets us classify them without naming the gpui-flowtext + // types, which are not reachable from this crate. + landscape: page.orientation != Default::default(), + page_num_enabled: page.page_numbering.format != Default::default(), + page_num_start: page.page_numbering.start, + }, + None => DEFAULT_SECTION, + } +} + +/// Resolved page-structure context for one export. +pub(super) struct SectionContext { + /// Attributes for the *final* section, applied at the document level. + doc_level: EffectiveSection, + /// Body block index -> section property to attach at that boundary. + boundaries: FxHashMap, + /// Document-wide content width (page width minus horizontal margins) used for + /// image fit-width scaling and fractional table column widths. Taken from the + /// first section for a stable single value. + pub(super) content_width_twips: i64, +} + +impl SectionContext { + pub(super) fn resolve(document: &DocumentProjection) -> Self { + if document.sections.is_empty() { + return Self { + doc_level: DEFAULT_SECTION, + boundaries: FxHashMap::default(), + content_width_twips: DEFAULT_SECTION.content_width_twips(), + }; + } + + // Resolve each section to (start paragraph index, effective attrs). Sections + // whose start paragraph no longer resolves are dropped from boundary + // placement but still contribute their attrs to the ordered set. + let mut resolved: Vec<(usize, EffectiveSection)> = Vec::with_capacity(document.sections.len()); + for section in document.sections.iter() { + if let Some(start_ix) = document + .ids + .paragraph_ids + .iter() + .position(|id| *id == section.start_paragraph) + { + resolved.push((start_ix, effective_from(section))); + } + } + + if resolved.is_empty() { + // FS-126 fidelity: no section start paragraph resolves, so only the first + // section's geometry survives as the document-level sectPr; any further + // sections drop their page geometry. + flowstate_fidelity::check( + document.sections.len() <= 1, + FidelityClass::ImportExport, + "export-dropped-section", + || { + format!( + "{} canonical sections but none resolve a start paragraph; only one geometry written", + document.sections.len() + ) + }, + ); + let doc_level = effective_from(&document.sections[0]); + return Self { + content_width_twips: doc_level.content_width_twips(), + doc_level, + boundaries: FxHashMap::default(), + }; + } + + resolved.sort_by_key(|(start_ix, _)| *start_ix); + let content_width_twips = resolved[0].1.content_width_twips(); + // OOXML: the document-level `w:sectPr` describes the final section; every + // earlier section terminates at the paragraph before the next section start. + let doc_level = resolved[resolved.len() - 1].1; + let mut boundaries = FxHashMap::default(); + for pair in resolved.windows(2) { + let (_, this_section) = pair[0]; + let (next_start_ix, _) = pair[1]; + if let Some(last_paragraph_ix) = next_start_ix.checked_sub(1) + && let Some(block_ix) = block_ix_for_paragraph(document, last_paragraph_ix) + { + boundaries.insert(block_ix, this_section); + } + } + + // FS-126 fidelity: every canonical section's page geometry must reach the + // OOXML — the final section as the document-level sectPr and each earlier + // section as a boundary sectPr. A section whose start paragraph or boundary + // block no longer resolves (or whose boundary collides with another) drops + // its geometry, so the written placement count falls below the section count. + flowstate_fidelity::check( + boundaries.len() + 1 >= document.sections.len(), + FidelityClass::ImportExport, + "export-dropped-section", + || { + format!( + "{} canonical sections but only {} sectPr placements written (1 document-level + {} boundary)", + document.sections.len(), + boundaries.len() + 1, + boundaries.len() + ) + }, + ); + + Self { + doc_level, + boundaries, + content_width_twips, + } + } + + /// Apply the final section's page attributes at the document level. + pub(super) fn apply_document_section(&self, mut docx: Docx) -> Docx { + let section = self.doc_level; + docx = docx.page_size(twips_u32(section.page_width_twips), twips_u32(section.page_height_twips)); + docx = docx.page_margin(section.page_margin()); + if section.landscape { + docx = docx.page_orient(PageOrientationType::Landscape); + } + if let Some(page_num_type) = section.page_num_type() { + docx = docx.page_num_type(page_num_type); + } + docx + } + + /// The paragraph-level section property to attach when emitting the body block + /// at `block_ix`, if it is a non-final section boundary. + pub(super) fn boundary_section_property(&self, block_ix: usize) -> Option { + self + .boundaries + .get(&block_ix) + .map(EffectiveSection::section_property) + } +} diff --git a/crates/flowstate-docx/src/exporter/xml_postprocess.rs b/crates/flowstate-docx/src/exporter/xml_postprocess.rs new file mode 100644 index 00000000..26b909d0 --- /dev/null +++ b/crates/flowstate-docx/src/exporter/xml_postprocess.rs @@ -0,0 +1,364 @@ +//! Post-generation `word/document.xml` rewrite seam (FS-124/125/127). +//! +//! docx-rs 0.4.20 cannot express several OOXML constructs Flowstate needs: +//! `wp:docPr` accessibility attributes (`descr`/`title`), `m:oMath` equations, +//! and `w:tblHeader`. Rather than fork docx-rs, the exporter emits deterministic +//! sentinels through the supported API and this pass rewrites them into correct +//! OOXML while the package is round-tripped in [`super::package`]: +//! +//! * Every embedded image emits an identical ``; +//! [`SideChannel::image_doc_prs`] carries one entry per emitted drawing (in +//! document order) so the k-th `docPr` gets the k-th image's alt text. +//! * Each convertible equation emits a placeholder run whose text is a unique +//! sentinel; the enclosing `…` is replaced by the injected OMML. +//! * §A11.9: each LINKED image (external URL, no media part) emits a sentinel +//! run the same way; the enclosing run is replaced by a full inline drawing +//! whose `a:blip` carries `r:link`, and the package writer injects the +//! matching `TargetMode="External"` relationship. +//! * Header rows are tagged with `` (the only per-row marker +//! docx-rs exposes), rewritten here to ``. +//! +//! All edits are targeted string surgery over the exact, deterministic output +//! docx-rs produces, so the rest of the document is left byte-for-byte intact. + +const MATH_NAMESPACE: &str = " xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\""; +const DOC_PR_TARGET: &str = ""; +const EQUATION_SENTINEL_PREFIX: &str = "@@FLOWSTATE_OMML_"; +const SENTINEL_SUFFIX: &str = "@@"; +/// §A11.9: sentinel prefix for LINKED images (`a:blip r:link`, no media part). +const LINKED_IMAGE_SENTINEL_PREFIX: &str = "@@FLOWSTATE_LINKIMG_"; + +/// Alt-text destined for one image's `wp:docPr`. +pub(super) struct ImageDocPr { + pub(super) descr: Option, + pub(super) title: Option, +} + +impl ImageDocPr { + fn render(&self, index: usize) -> String { + // Give each image a unique docPr id so Word does not warn about the shared + // hard-coded "1"; keep the "Figure" name docx-rs assigns. + let mut tag = format!(""); + tag + } +} + +/// One run-level injection: the placeholder sentinel and the OOXML that +/// replaces the sentinel's enclosing `…` (an equation's OMML, or a +/// linked image's drawing run — §A11.9). +pub(super) struct RunInjection { + sentinel: String, + xml: String, +} + +/// Data collected during block export and consumed by [`rewrite_document_xml`]. +#[derive(Default)] +pub(super) struct SideChannel { + image_doc_prs: Vec, + equations: Vec, + /// §A11.9: linked-image drawing runs, injected alongside the equations. + linked_images: Vec, + /// §A11.9: `(rId, url)` pairs to add to `word/_rels/document.xml.rels` as + /// `TargetMode="External"` image relationships. + linked_image_rels: Vec<(String, String)>, + warnings: Vec, +} + +impl SideChannel { + /// Register an emitted image drawing (in document order). `descr`/`title` + /// become the `wp:docPr` accessibility attributes when present. + pub(super) fn push_image(&mut self, descr: Option, title: Option) { + self.image_doc_prs.push(ImageDocPr { descr, title }); + } + + /// Register an equation's OMML and return the sentinel text to embed in the + /// placeholder run so the rewrite pass can locate and replace it. + pub(super) fn push_equation(&mut self, omml: String) -> String { + let sentinel = format!("{EQUATION_SENTINEL_PREFIX}{}{SENTINEL_SUFFIX}", self.equations.len()); + self.equations.push(RunInjection { + sentinel: sentinel.clone(), + xml: omml, + }); + sentinel + } + + /// §A11.9: register a LINKED image (external URL, no media part) and return + /// the sentinel text for its placeholder run. docx-rs has no `a:blip r:link` + /// API, so the drawing is injected whole by the rewrite pass and the external + /// relationship is added to the rels part by the package writer. The `rId` + /// scheme (`rIdFsLink1`, …) cannot collide with docx-rs's numeric `rIdN` ids. + pub(super) fn push_linked_image(&mut self, url: &str, descr: Option<&str>, width_emu: u32, height_emu: u32) -> String { + let index = self.linked_images.len(); + let sentinel = format!("{LINKED_IMAGE_SENTINEL_PREFIX}{index}{SENTINEL_SUFFIX}"); + let relationship_id = format!("rIdFsLink{}", index + 1); + let xml = linked_image_drawing_run(&relationship_id, descr, width_emu, height_emu, index); + self.linked_images.push(RunInjection { + sentinel: sentinel.clone(), + xml, + }); + self + .linked_image_rels + .push((relationship_id, url.to_string())); + sentinel + } + + /// §A11.9: the `(rId, url)` external image relationships to inject into + /// `word/_rels/document.xml.rels`. + pub(super) fn linked_image_rels(&self) -> &[(String, String)] { + &self.linked_image_rels + } + + /// Record a non-fatal export degradation (e.g. an image format that could not + /// be embedded). Surfaced through [`super::write_docx_with_report`]. + pub(super) fn push_warning(&mut self, warning: String) { + self.warnings.push(warning); + } + + pub(super) fn into_warnings(self) -> Vec { + self.warnings + } +} + +/// Rewrite `word/document.xml` bytes, applying every queued transform. +#[hotpath::measure] +pub(super) fn rewrite_document_xml(bytes: Vec, side: &SideChannel) -> Vec { + let mut xml = match String::from_utf8(bytes) { + Ok(text) => text, + // Non-UTF-8 document.xml is not something docx-rs produces; pass through. + Err(error) => return error.into_bytes(), + }; + if !side.equations.is_empty() { + xml = ensure_math_namespace(xml); + } + xml = rewrite_doc_prs(&xml, &side.image_doc_prs); + // §perf: splice every injected run (equations + linked images) in one forward + // pass over the xml instead of rebuilding the whole String once per + // injection (O(N) vs O(N·injections)). Runs AFTER `rewrite_doc_prs` so the + // linked images' own `wp:docPr` tags are never re-matched by it. + if !side.equations.is_empty() || !side.linked_images.is_empty() { + xml = inject_run_replacements(&xml, side.equations.iter().chain(side.linked_images.iter())); + } + // §perf: the two whole-document `.replace()` scans below only ever match the + // ` String { + let Some(start) = xml.find("') else { + return xml; + }; + let end = start + relative_end; + if xml[start..end].contains("xmlns:m=") { + return xml; + } + let mut out = String::with_capacity(xml.len() + MATH_NAMESPACE.len()); + out.push_str(&xml[..end]); + out.push_str(MATH_NAMESPACE); + out.push_str(&xml[end..]); + out +} + +/// Replace each identical image `wp:docPr` with a per-image variant carrying +/// alt text, matched to images by document order. +fn rewrite_doc_prs(xml: &str, entries: &[ImageDocPr]) -> String { + if entries.is_empty() || !xml.contains(DOC_PR_TARGET) { + return xml.to_string(); + } + let mut out = String::with_capacity(xml.len()); + let mut rest = xml; + let mut index = 0usize; + while let Some(pos) = rest.find(DOC_PR_TARGET) { + out.push_str(&rest[..pos]); + match entries.get(index) { + Some(entry) => out.push_str(&entry.render(index)), + None => out.push_str(DOC_PR_TARGET), + } + rest = &rest[pos + DOC_PR_TARGET.len()..]; + index += 1; + } + out.push_str(rest); + out +} + +/// Replace every placeholder run containing an injection sentinel (equation +/// OMML or linked-image drawing — §A11.9) with its OOXML in a single forward +/// pass. Each sentinel occupies a run of its own, so the enclosing +/// `…` maps 1:1 to the injection. +/// +/// §perf: the previous implementation rebuilt the whole document `String` once +/// per equation (O(N·equations)). Each injection lives in a disjoint run, so the +/// run bounds are computed against the original `xml` once, ordered by position, +/// then spliced into a single output buffer — byte-identical to the sequential +/// per-injection rewrite but linear in the document length. +fn inject_run_replacements<'injections>(xml: &str, injections: impl Iterator) -> String { + // Resolve each injection to its enclosing run span in the original xml. + let mut spans: Vec<(usize, usize, &str)> = Vec::new(); + for injection in injections { + let Some(pos) = xml.find(&injection.sentinel) else { + continue; + }; + // docx-rs emits run opens as the exact literal `` (no attributes) and + // runs never nest, so the nearest enclosing tags bound the placeholder run. + let Some(run_start) = xml[..pos].rfind("") else { + continue; + }; + let Some(relative_end) = xml[pos..].find("") else { + continue; + }; + let run_end = pos + relative_end + "".len(); + spans.push((run_start, run_end, injection.xml.as_str())); + } + // Sentinels are unique but may be registered in any order; splicing requires + // ascending, non-overlapping spans (runs are disjoint by construction). + spans.sort_by_key(|(run_start, _, _)| *run_start); + let injected_len: usize = spans.iter().map(|(_, _, injected)| injected.len()).sum(); + let mut out = String::with_capacity(xml.len() + injected_len); + let mut cursor = 0usize; + for (run_start, run_end, injected) in spans { + out.push_str(&xml[cursor..run_start]); + out.push_str(injected); + cursor = run_end; + } + out.push_str(&xml[cursor..]); + out +} + +/// §A11.9: the full `…` for a LINKED image — +/// an inline drawing whose `a:blip` carries `r:link` (an external-mode image +/// relationship) instead of `r:embed`. Namespaces are declared locally so the +/// fragment is valid even when docx-rs emitted no drawing of its own (a +/// document whose only images are linked declares no `wp:`/`a:`/`pic:` on the +/// root). `wp:docPr` ids for embedded images are `1..=n` (see +/// [`rewrite_doc_prs`]); linked images start far above to stay unique. +fn linked_image_drawing_run(relationship_id: &str, descr: Option<&str>, width_emu: u32, height_emu: u32, index: usize) -> String { + let doc_pr_id = 9000 + index; + let descr_attr = descr + .map(|descr| format!(" descr=\"{}\"", escape_attr(descr))) + .unwrap_or_default(); + format!( + concat!( + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ), + cx = width_emu, + cy = height_emu, + id = doc_pr_id, + descr = descr_attr, + rid = relationship_id, + ) +} + +/// Rewrite header-row markers into `w:tblHeader`. +fn inject_table_headers(xml: &str) -> String { + xml + .replace("", "") + .replace("", "") +} + +/// XML attribute-value escaping, shared with the package writer's external-rel +/// injection (§A11.9 — URLs routinely carry `&` in query strings). +pub(super) fn escape_attr(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn math_namespace_added_once() { + let xml = "".to_string(); + let with_ns = ensure_math_namespace(xml); + assert!(with_ns.contains("xmlns:m=")); + let idempotent = ensure_math_namespace(with_ns.clone()); + assert_eq!(with_ns.matches("xmlns:m=").count(), idempotent.matches("xmlns:m=").count()); + } + + #[test] + fn doc_pr_rewrite_is_ordered_and_unique() { + let xml = format!("a{DOC_PR_TARGET}b{DOC_PR_TARGET}c"); + let entries = vec![ + ImageDocPr { + descr: Some("first".to_string()), + title: Some("first".to_string()), + }, + ImageDocPr { descr: None, title: None }, + ]; + let out = rewrite_doc_prs(&xml, &entries); + assert!(out.contains("id=\"1\" name=\"Figure\" descr=\"first\" title=\"first\" />")); + assert!(out.contains("id=\"2\" name=\"Figure\" />")); + } + + #[test] + fn equation_injection_replaces_enclosing_run() { + let sentinel = "@@FLOWSTATE_OMML_0@@"; + let xml = format!("{sentinel}"); + let injections = [RunInjection { + sentinel: sentinel.to_string(), + xml: "".to_string(), + }]; + let out = inject_run_replacements(&xml, injections.iter()); + assert_eq!(out, ""); + } + + /// §A11.9: a linked-image sentinel run is replaced by a full inline drawing + /// whose blip carries `r:link` to the allocated relationship id. + #[test] + fn linked_image_injection_replaces_enclosing_run_with_drawing() { + let mut side = SideChannel::default(); + let sentinel = side.push_linked_image("https://example.com/a.png?x=1&y=2", Some("alt "), 914_400, 685_800); + assert_eq!( + side.linked_image_rels(), + &[("rIdFsLink1".to_string(), "https://example.com/a.png?x=1&y=2".to_string())] + ); + let xml = format!("{sentinel}").into_bytes(); + let out = String::from_utf8(rewrite_document_xml(xml, &side)).expect("utf8"); + assert!(out.contains("r:link=\"rIdFsLink1\""), "blip link missing: {out}"); + assert!(out.contains("descr=\"alt <text>\""), "docPr descr missing/unescaped: {out}"); + assert!(out.contains("cx=\"914400\""), "extent missing: {out}"); + assert!(!out.contains("FLOWSTATE_LINKIMG"), "sentinel leaked into output: {out}"); + assert!(!out.contains("r:embed"), "linked image must not emit r:embed: {out}"); + } + + #[test] + fn header_marker_becomes_tbl_header() { + let xml = ""; + assert_eq!(inject_table_headers(xml), ""); + } +} diff --git a/crates/flowstate-docx/src/fragment_parse.rs b/crates/flowstate-docx/src/fragment_parse.rs new file mode 100644 index 00000000..4fe983c7 --- /dev/null +++ b/crates/flowstate-docx/src/fragment_parse.rs @@ -0,0 +1,414 @@ +//! §act-twelve A12.3.1-full: fragment-parallel `document.xml` typed parse. +//! +//! `rdocx_oxml::CT_Document::from_xml` is a single-threaded streaming parse; on +//! large documents the typed tree build dominates the import hotpath (~107ms of +//! the fixture's ~122ms interpret step). The body is a flat sequence of +//! independent ``/`` subtrees dispatched by *local name*, so each +//! fragment parses standalone with no namespace or sibling context. This module +//! scans the body with a skip-only tokenizer pass (`read_to_end_into`, no tree +//! building) to find top-level fragment byte spans, streams each span to a +//! scoped worker pool the moment it is discovered (the parse pipelines under +//! the scan instead of waiting for it), and reassembles the `CT_Document` in +//! scan order. Everything that is not a top-level `w:p`/`w:tbl` (sectPr, +//! raw-xml captures, extra namespaces, background) is handled inline during the +//! scan with the exact upstream code paths, so the assembled tree is equal to +//! the sequential parse (`PartialEq` across the whole CT hierarchy backs the +//! oracle). +//! +//! Fail-safe: any scan or fragment error falls back to the sequential +//! `CT_Document::from_xml`, so error behavior (and messages) match upstream. +//! Knobs: `FLOWSTATE_DISABLE_FRAGMENT_PARSE` forces the sequential path; +//! `FLOWSTATE_FRAGMENT_PARSE_VERIFY` runs both and asserts tree equality; +//! `FLOWSTATE_PARSE_PROBE` prints stage timings. + +use std::collections::VecDeque; +use std::ops::Range; +use std::sync::{Condvar, Mutex, OnceLock}; + +use quick_xml_oxml::Reader; +use quick_xml_oxml::events::Event; +use quick_xml_oxml::name::QName; +use rdocx_oxml::document::{BodyContent, CT_Body, CT_Document, CT_SectPr}; +use rdocx_oxml::namespace::matches_local_name; +use rdocx_oxml::raw_xml::{capture_element, capture_empty_element}; +use rdocx_oxml::table::CT_Tbl; +use rdocx_oxml::text::CT_P; + +/// Below this size the sequential parse is already fast; thread setup would +/// only add noise. +const MIN_FRAGMENT_PARSE_BYTES: usize = 256 * 1024; +const MAX_WORKERS: usize = 8; + +fn fragment_parse_disabled() -> bool { + static DISABLED: OnceLock = OnceLock::new(); + *DISABLED.get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_FRAGMENT_PARSE").is_some()) +} + +fn fragment_parse_verify() -> bool { + static VERIFY: OnceLock = OnceLock::new(); + *VERIFY.get_or_init(|| std::env::var_os("FLOWSTATE_FRAGMENT_PARSE_VERIFY").is_some()) +} + +fn parse_probe() -> bool { + static PROBE: OnceLock = OnceLock::new(); + *PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_PARSE_PROBE").is_some()) +} + +/// Drop-in replacement for `CT_Document::from_xml` that parses top-level body +/// fragments in parallel when the document is large enough to pay for it. +pub fn ct_document_from_xml(xml: &[u8]) -> rdocx_oxml::Result { + if fragment_parse_disabled() || xml.len() < MIN_FRAGMENT_PARSE_BYTES { + return CT_Document::from_xml(xml); + } + match parse_parallel(xml) { + Some(document) => { + if fragment_parse_verify() { + let sequential = CT_Document::from_xml(xml)?; + assert!( + document == sequential, + "[flowstate-fragment-parse-verify] parallel CT_Document diverges from sequential parse" + ); + } + Ok(document) + }, + // Any scan/fragment failure: let the sequential parse decide the outcome, + // so error behavior is byte-for-byte upstream's. + None => CT_Document::from_xml(xml), + } +} + +/// A top-level body fragment: its slot in the body's content sequence and the +/// byte span of its subtree. +struct Fragment { + item_ix: usize, + span: Range, + is_paragraph: bool, +} + +/// Single-producer/multi-consumer span queue: the scanner pushes fragments as +/// it discovers them; workers block on the condvar until spans arrive or the +/// scan finishes. +struct FragmentQueue { + state: Mutex<(VecDeque, bool)>, + ready: Condvar, +} + +impl FragmentQueue { + fn new() -> Self { + FragmentQueue { + state: Mutex::new((VecDeque::new(), false)), + ready: Condvar::new(), + } + } + + fn push_batch(&self, fragments: &mut Vec) { + if fragments.is_empty() { + return; + } + let mut state = self.state.lock().expect("fragment queue poisoned"); + state.0.extend(fragments.drain(..)); + drop(state); + self.ready.notify_all(); + } + + fn finish(&self) { + let mut state = self.state.lock().expect("fragment queue poisoned"); + state.1 = true; + drop(state); + self.ready.notify_all(); + } + + fn pop(&self) -> Option { + let mut state = self.state.lock().expect("fragment queue poisoned"); + loop { + if let Some(fragment) = state.0.pop_front() { + return Some(fragment); + } + if state.1 { + return None; + } + state = self.ready.wait(state).expect("fragment queue poisoned"); + } + } +} + +/// Scan-side accumulator: `slots` holds inline-parsed content (raw-xml +/// captures) at its body position; fragment slots stay `None` until the pool's +/// results are merged back in. +struct ScannedDocument { + body_seen: bool, + slots: Vec>, + sect_pr: Option, + extra_namespaces: Vec<(String, String)>, + background_xml: Option>, +} + +fn parse_parallel(xml: &[u8]) -> Option { + let workers = std::thread::available_parallelism() + .map_or(1, |n| n.get()) + .min(MAX_WORKERS); + if workers < 2 { + return None; + } + let queue = FragmentQueue::new(); + let scan_t0 = std::time::Instant::now(); + let (scanned, parsed) = std::thread::scope(|scope| { + let handles: Vec<_> = (0..workers) + .map(|_| { + scope.spawn(|| { + let mut out = Vec::new(); + while let Some(fragment) = queue.pop() { + out.push((fragment.item_ix, parse_fragment(xml, &fragment))); + } + out + }) + }) + .collect(); + let scanned = scan_document(xml, &queue); + // Unblock the workers on every exit path, including scan errors. + queue.finish(); + let scan_elapsed = scan_t0.elapsed(); + let parsed: Vec)>> = handles + .into_iter() + .map(|handle| handle.join().expect("fragment parse worker panicked")) + .collect(); + if parse_probe() { + let fragment_count: usize = parsed.iter().map(Vec::len).sum(); + eprintln!( + "[flowstate-fragment-parse] scan={scan_elapsed:?} pool_tail={:?} fragments={fragment_count} workers={workers}", + scan_t0.elapsed().saturating_sub(scan_elapsed) + ); + } + (scanned, parsed) + }); + let mut scanned = scanned.ok()?; + if !scanned.body_seen { + // Upstream substitutes a default body (default-letter sectPr); take the + // sequential path rather than replicate that construction here. + return None; + } + for (item_ix, content) in parsed.into_iter().flatten() { + scanned.slots[item_ix] = Some(content?); + } + let content: Option> = scanned.slots.into_iter().collect(); + Some(CT_Document { + body: CT_Body { + content: content?, + sect_pr: scanned.sect_pr, + }, + extra_namespaces: scanned.extra_namespaces, + background_xml: scanned.background_xml, + }) +} + +/// Parse one standalone top-level fragment. The span starts at (or on +/// whitespace immediately before) the fragment's start tag; local-name +/// dispatch means the fragment needs no namespace declarations in scope. +fn parse_fragment(xml: &[u8], fragment: &Fragment) -> Option { + let expected_local: &[u8] = if fragment.is_paragraph { b"p" } else { b"tbl" }; + let mut reader = Reader::from_reader(&xml[fragment.span.clone()]); + reader.config_mut().trim_text(true); + let mut buf = Vec::new(); + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) => { + if !matches_local_name(e.name().as_ref(), expected_local) { + return None; + } + return if fragment.is_paragraph { + CT_P::from_xml(&mut reader).ok().map(BodyContent::Paragraph) + } else { + CT_Tbl::from_xml(&mut reader).ok().map(BodyContent::Table) + }; + }, + Ok(Event::Eof) | Err(_) => return None, + // Whitespace text is trimmed away; anything else preceding the start + // tag (comments, PIs) is ignored exactly as the sequential body loop + // ignores it. + _ => {}, + } + buf.clear(); + } +} + +/// Mirror of `CT_Document::from_xml`'s top-level loop, with the body handed to +/// `scan_body` instead of `CT_Body::from_xml`. +fn scan_document(xml: &[u8], queue: &FragmentQueue) -> rdocx_oxml::Result { + let mut reader = Reader::from_reader(xml); + reader.config_mut().trim_text(true); + + let mut scanned = ScannedDocument { + body_seen: false, + slots: Vec::new(), + sect_pr: None, + extra_namespaces: Vec::new(), + background_xml: None, + }; + let mut buf = Vec::new(); + + // Known namespace prefixes upstream always emits itself. + let known_ns: &[&[u8]] = &[b"xmlns:w", b"xmlns:r", b"xmlns:mc", b"xmlns"]; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) => { + let name = e.name(); + if matches_local_name(name.as_ref(), b"body") { + scanned.body_seen = true; + scan_body(&mut reader, &mut scanned, queue)?; + } else if matches_local_name(name.as_ref(), b"document") { + for attr in e.attributes().flatten() { + let key = attr.key.as_ref(); + if (key.starts_with(b"xmlns:") || key == b"xmlns") && !known_ns.contains(&key) { + let key_str = std::str::from_utf8(key).unwrap_or("").to_string(); + let val_str = std::str::from_utf8(&attr.value).unwrap_or("").to_string(); + scanned.extra_namespaces.push((key_str, val_str)); + } + } + } else if matches_local_name(name.as_ref(), b"background") { + scanned.background_xml = Some(capture_element(&mut reader, &e)?); + } else { + let owned_name = name.as_ref().to_vec(); + let mut skip_buf = Vec::new(); + reader.read_to_end_into(QName(&owned_name), &mut skip_buf)?; + } + }, + Ok(Event::Empty(e)) => { + if matches_local_name(e.name().as_ref(), b"background") { + scanned.background_xml = Some(capture_empty_element(&e)?); + } + }, + Ok(Event::Eof) => break, + Err(e) => return Err(e.into()), + _ => {}, + } + buf.clear(); + } + + Ok(scanned) +} + +/// Batch size for handing spans to the pool: one lock + notify per batch +/// instead of per fragment. +const PUSH_BATCH: usize = 64; + +/// Mirror of `CT_Body::from_xml`, recording `w:p`/`w:tbl` spans via a +/// tokenizer-only skip (borrowed `read_event`/`read_to_end`, no tree building +/// and no event-buffer copies), streaming spans to the worker queue in +/// batches, and handling everything else with upstream's own code paths. +fn scan_body(reader: &mut Reader<&[u8]>, scanned: &mut ScannedDocument, queue: &FragmentQueue) -> rdocx_oxml::Result<()> { + let mut pending: Vec = Vec::with_capacity(PUSH_BATCH); + loop { + let event_start = usize::try_from(reader.buffer_position()).expect("in-memory reader position fits usize"); + match reader.read_event() { + Ok(Event::Start(e)) => { + let name = e.name(); + if matches_local_name(name.as_ref(), b"p") || matches_local_name(name.as_ref(), b"tbl") { + let is_paragraph = matches_local_name(name.as_ref(), b"p"); + reader.read_to_end(QName(name.as_ref()))?; + let span = event_start..usize::try_from(reader.buffer_position()).expect("in-memory reader position fits usize"); + let item_ix = scanned.slots.len(); + scanned.slots.push(None); + pending.push(Fragment { item_ix, span, is_paragraph }); + if pending.len() >= PUSH_BATCH { + queue.push_batch(&mut pending); + } + } else if matches_local_name(name.as_ref(), b"sectPr") { + queue.push_batch(&mut pending); + scanned.sect_pr = Some(CT_SectPr::from_xml(reader)?); + } else { + queue.push_batch(&mut pending); + scanned + .slots + .push(Some(BodyContent::RawXml(capture_element(reader, &e)?))); + } + }, + Ok(Event::Empty(e)) => { + if !matches_local_name(e.name().as_ref(), b"body") { + scanned + .slots + .push(Some(BodyContent::RawXml(capture_empty_element(&e)?))); + } + }, + Ok(Event::End(e)) if matches_local_name(e.name().as_ref(), b"body") => break, + Ok(Event::Eof) => break, + Err(e) => return Err(e.into()), + _ => {}, + } + } + queue.push_batch(&mut pending); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fmt::Write as _; + + use super::*; + + /// Build a synthetic document.xml big enough (> `MIN_FRAGMENT_PARSE_BYTES`) to + /// engage the parallel path, exercising every top-level shape the scanner + /// mirrors: paragraphs, tables, unknown raw-xml subtrees, self-closing + /// elements, sectPr, background, and extra namespaces. + fn synthetic_document_xml() -> Vec { + let mut xml = String::new(); + xml.push_str(r#""#); + xml.push_str( + r#""#, + ); + xml.push_str(r#""#); + xml.push_str(""); + for ix in 0..1200 { + match ix % 5 { + 0 => write!( + xml, + r#"heading {} with some padding text to grow the file body toward the parallel threshold"#, + (ix % 4) + 1, + ix + ) + .unwrap(), + 1..=3 => write!( + xml, + r#"card body text {ix} lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliquaunderlined tail {ix}"# + ) + .unwrap(), + _ => write!( + xml, + r#"cell a {ix}cell b {ix}"# + ) + .unwrap(), + } + if ix % 97 == 0 { + // Unknown top-level subtree and a self-closing unknown element: both + // must round-trip as RawXml captures in exactly the upstream form. + xml.push_str(r"inside sdt"); + xml.push_str(r#""#); + } + } + xml + .push_str(r#""#); + xml.push_str(""); + xml.into_bytes() + } + + #[test] + fn parallel_parse_equals_sequential_on_synthetic_document() { + let xml = synthetic_document_xml(); + assert!( + xml.len() >= MIN_FRAGMENT_PARSE_BYTES, + "fixture must engage the parallel path ({} bytes)", + xml.len() + ); + let sequential = CT_Document::from_xml(&xml).expect("sequential parse"); + let parallel = parse_parallel(&xml).expect("parallel path should engage on this fixture"); + assert!(parallel == sequential, "parallel CT_Document must equal the sequential parse"); + } + + #[test] + fn small_documents_take_the_sequential_path_unchanged() { + let xml = br#"hi"#; + let via_entry = ct_document_from_xml(xml).expect("entry parse"); + let sequential = CT_Document::from_xml(xml).expect("sequential parse"); + assert_eq!(via_entry, sequential); + } +} diff --git a/crates/flowstate-docx/src/import_roundtrip_tests.rs b/crates/flowstate-docx/src/import_roundtrip_tests.rs new file mode 100644 index 00000000..fc0f0e33 --- /dev/null +++ b/crates/flowstate-docx/src/import_roundtrip_tests.rs @@ -0,0 +1,511 @@ +//! Class 5 — docx import must not produce malformed canonical state. +//! +//! The impact-doc field logs showed ~1240 paragraphs projecting with NO durable metadata +//! record and NO paragraph-style mark (`missing_paragraph_metadata` / +//! `missing_paragraph_style_mark` / `missing_paragraph_block`, plus `boundary-without-mark` +//! fidelity violations), which drove a repair storm. The current importer writes a durable +//! record + style mark for every paragraph, so a valid import SHOULD yield zero of these +//! defects — this harness locks that invariant in so a regression (an import path that skips +//! `import_paragraph_record` / the boundary style mark) fails loudly, and gives a ready knob +//! to run the SAME assertion against a real .docx (e.g. the impact doc) via an env var. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use flowstate_document::{ + AssetId, CellId, ColumnId, InputBlock, InputBlockAlignment, InputImageBlock, InputImageSizing, InputParagraph, InputRun, InputTableBlock, + InputTableCell, InputTableCellBlock, InputTableColumn, InputTableColumnWidth, InputTableRow, InputTableStyle, ParagraphStyle, RowId, + RunStyles, SOFT_LINE_BREAK, VertAlign, document_from_input_blocks, document_from_loro_with_defects, flowstate_document_theme, +}; + +use crate::{import_docx_bytes_to_loro, write_docx}; + +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn temp_docx_path() -> std::path::PathBuf { + let sequence = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!("flowstate-roundtrip-{}-{sequence}.docx", std::process::id())) +} + +/// A structurally-rich document with the impact-doc's hazardous features at scale: +/// hundreds of paragraphs, images flanked by empty paragraphs, and a table. +fn rich_document(paragraphs: usize) -> flowstate_document::DocumentProjection { + let para = |text: &str| { + InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: if text.is_empty() { + Vec::new() + } else { + vec![InputRun { + text: text.to_string(), + styles: RunStyles::default(), + }] + }, + }) + }; + let image = || { + InputBlock::Image(InputImageBlock { + asset_id: AssetId(1), + alt_text: "figure".to_string(), + caption: None, + sizing: InputImageSizing::Intrinsic, + alignment: InputBlockAlignment::Left, + external_url: None, + }) + }; + let cell = |row: u128, col: u128, text: &str| InputTableCell { + id: CellId(row * 100 + col), + row_id: RowId(row), + column_id: ColumnId(col), + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![InputRun { + text: text.to_string(), + styles: RunStyles::default(), + }], + })], + row_span: 1, + col_span: 1, + }; + let table = InputBlock::Table(InputTableBlock { + columns: vec![ + InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::Fraction(1), + }, + InputTableColumn { + id: ColumnId(2), + width: InputTableColumnWidth::Fraction(1), + }, + ], + rows: vec![ + InputTableRow { + id: RowId(1), + cells: vec![cell(1, 1, "a"), cell(1, 2, "b")], + }, + InputTableRow { + id: RowId(2), + cells: vec![cell(2, 1, "c"), cell(2, 2, "d")], + }, + ], + style: InputTableStyle { header_row: true }, + }); + + let mut blocks = vec![para("Title of the brief"), table]; + for ix in 0..paragraphs { + // §perf-heaven T8.6 guard: include the multibyte punctuation that pervades real + // debate evidence (curly quotes, em/en dashes). Paragraph byte ranges are now + // DERIVED from the block tree on export; a derive that omitted the inter-block + // `\n` separators sliced `document.text` mid-character here and panicked. Keeping + // these code points in the synthetic body makes the fast roundtrip test a guard + // for that whole class without needing the full corpus sweep. + blocks.push(para(&format!( + "Body paragraph {ix} — the author\u{2019}s claim that \u{201c}deterrence\u{201d} holds \u{2013} with citations." + ))); + if ix % 50 == 0 { + blocks.push(image()); + blocks.push(para("")); // empty paragraph adjacent to an object + } + } + // Body run with an intra-paragraph soft break: exports as a body-level + // ``, which the importer must map back to U+2028 — NOT '\n', which + // would fabricate a record-less paragraph boundary (the impact-doc defect). + blocks.push(para(&soft_break_probe_text())); + document_from_input_blocks(flowstate_document_theme(), blocks) +} + +fn soft_break_probe_text() -> String { + format!("wrapped argument line{SOFT_LINE_BREAK}continued after a soft line break") +} + +fn assert_import_defect_free(bytes: &[u8], label: &str) -> flowstate_document::DocumentProjection { + let (imported, report) = import_docx_bytes_to_loro(bytes, "roundtrip").expect("import docx bytes"); + let (projection, defects) = document_from_loro_with_defects(&imported.doc).expect("project imported docx"); + assert!( + defects.is_empty(), + "{label}: importing produced {} projection defect(s) (paragraphs_imported={:?}) — the importer left canonical state a repair pass must fix; sample: {:?}", + defects.len(), + report.paragraphs_imported, + defects.iter().take(5).collect::>(), + ); + projection +} + +/// A minimal single-part docx zip: `word/document.xml` + its rels + the fixed +/// package plumbing, plus any extra parts (media). Shared by the linked-image +/// fixtures (§A11.9). +fn build_docx_zip(document_xml: &str, document_rels: &str, extra_parts: &[(&str, &[u8])]) -> Vec { + use std::io::Write as _; + let root_rels = concat!( + r#""#, + r#""#, + r#""#, + r#""# + ); + let content_types = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""# + ); + let mut cursor = std::io::Cursor::new(Vec::new()); + { + let mut writer = zip::ZipWriter::new(&mut cursor); + let options = zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let mut parts: Vec<(&str, &[u8])> = vec![ + ("[Content_Types].xml", content_types.as_bytes()), + ("_rels/.rels", root_rels.as_bytes()), + ("word/document.xml", document_xml.as_bytes()), + ("word/_rels/document.xml.rels", document_rels.as_bytes()), + ]; + parts.extend_from_slice(extra_parts); + for (name, bytes) in parts { + writer.start_file(name, options).expect("zip entry"); + writer.write_all(bytes).expect("zip bytes"); + } + writer.finish().expect("zip finish") + }; + cursor.into_inner() +} + +fn image_blocks(projection: &flowstate_document::DocumentProjection) -> Vec { + projection + .blocks + .iter() + .filter_map(|block| match block { + flowstate_document::Block::Image(image) => Some(image.clone()), + _ => None, + }) + .collect() +} + +/// §act-eleven A11.9(a): a genuinely-LINKED `DrawingML` image (`a:blip r:link` +/// to a `TargetMode="External"` relationship, NO embedded media part) imports +/// as ONE image block carrying the external URL — not dropped, and no +/// bracketed alt text fabricated in the body. +#[test] +fn linked_drawingml_blip_imports_as_external_url_image() { + let url = "https://example.com/linked-figure.png?cache=1&v=2"; + let document_xml = concat!( + r#""#, + r#""#, + r#""#, + r#"before"#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#"after"#, + r#""# + ); + let rels = format!( + concat!( + r#""#, + r#""#, + r#""#, + r#""# + ), + url.replace('&', "&"), + ); + let bytes = build_docx_zip(document_xml, &rels, &[]); + + let projection = assert_import_defect_free(&bytes, "linked drawingml import"); + let images = image_blocks(&projection); + assert_eq!(images.len(), 1, "the link-only blip must import as ONE image block"); + assert_eq!( + images[0] + .external_url + .as_ref() + .map(|url| -> &str { url.as_ref() }), + Some(url), + "the external-mode relationship target must land on the image block" + ); + assert_eq!(images[0].alt_text, "linked figure", "docPr descr must map to alt text"); + // 914400x685800 EMU = 96x72 px. + assert_eq!( + images[0].sizing, + flowstate_document::ImageSizing::Fixed { + width_px: 96, + height_px: Some(72) + }, + "wp:extent must map to fixed pixel sizing" + ); + assert!( + !projection.text.to_string().contains("linked figure"), + "a linked image must import as an object, not bracketed body text" + ); +} + +/// §act-eleven A11.9(b): a VML `v:imagedata` whose ONLY resolvable reference is +/// the external `r:href` companion (no embeddable `r:id` part) imports as a +/// linked image carrying the URL. +#[test] +fn vml_href_external_imports_as_external_url_image() { + let url = "https://example.com/vml-tracker.gif"; + let document_xml = concat!( + r#""#, + r#""#, + r#""#, + r#"before"#, + r#""#, + r#"after"#, + r#""# + ); + let rels = format!( + concat!( + r#""#, + r#""#, + r#""#, + r#""# + ), + url = url + ); + let bytes = build_docx_zip(document_xml, &rels, &[]); + + let projection = assert_import_defect_free(&bytes, "vml external import"); + let images = image_blocks(&projection); + assert_eq!(images.len(), 1, "the external-only VML pict must import as ONE image block"); + assert_eq!( + images[0] + .external_url + .as_ref() + .map(|url| -> &str { url.as_ref() }), + Some(url), + "the r:href external target must survive" + ); + // 48pt x 36pt at 96dpi = 64px x 48px. + assert_eq!( + images[0].sizing, + flowstate_document::ImageSizing::Fixed { + width_px: 64, + height_px: Some(48) + }, + "shape style extent must map to fixed pixel sizing" + ); +} + +/// §act-eleven C7: legacy VML images (``) import +/// as real image blocks with the shape-style extent — previously dropped +/// wholesale (one of the four old corpus residuals was a 27-image VML doc). +/// Deterministic fixture so the CLASS is guarded, not just the one corpus doc. +#[test] +fn vml_imagedata_imports_as_image_block_with_style_extent() { + use std::io::Write as _; + let png = { + let buffer = image::RgbImage::from_pixel(4, 3, image::Rgb([10u8, 120, 240])); + let mut cursor = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgb8(buffer) + .write_to(&mut cursor, image::ImageFormat::Png) + .expect("encode fixture png"); + cursor.into_inner() + }; + let document_xml = concat!( + r#""#, + r#""#, + r#""#, + r#"before"#, + r#""#, + r#"after"#, + r#""# + ); + let rels = concat!( + r#""#, + r#""#, + r#""#, + r#""# + ); + let root_rels = concat!( + r#""#, + r#""#, + r#""#, + r#""# + ); + let content_types = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""#, + r#""# + ); + let mut cursor = std::io::Cursor::new(Vec::new()); + { + let mut writer = zip::ZipWriter::new(&mut cursor); + let options = zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, bytes) in [ + ("[Content_Types].xml", content_types.as_bytes()), + ("_rels/.rels", root_rels.as_bytes()), + ("word/document.xml", document_xml.as_bytes()), + ("word/_rels/document.xml.rels", rels.as_bytes()), + ("word/media/image1.png", png.as_slice()), + ] { + writer.start_file(name, options).expect("zip entry"); + writer.write_all(bytes).expect("zip bytes"); + } + writer.finish().expect("zip finish") + }; + let bytes = cursor.into_inner(); + + let projection = assert_import_defect_free(&bytes, "vml import"); + let images: Vec<_> = projection + .blocks + .iter() + .filter_map(|block| match block { + flowstate_document::Block::Image(image) => Some(image.clone()), + _ => None, + }) + .collect(); + assert_eq!(images.len(), 1, "the VML pict must import as ONE image block"); + // 48pt x 36pt at 96dpi = 64px x 48px. + assert_eq!( + images[0].sizing, + flowstate_document::ImageSizing::Fixed { + width_px: 64, + height_px: Some(48) + }, + "shape style extent must map to fixed pixel sizing" + ); + // Asset BYTES live out-of-band of Loro (package-side); assert them on the + // docx-built projection, which is what the live app persists. + let (imported, _report) = import_docx_bytes_to_loro(&bytes, "vml import assets").expect("import docx bytes"); + let asset = imported + .projection + .assets + .assets + .get(&images[0].asset_id) + .expect("asset record attached to the docx-built projection"); + assert_eq!(asset.bytes.as_slice(), png.as_slice(), "asset bytes must be the media part verbatim"); +} + +/// Build a rich document, write a real .docx, re-import it, and assert the projection has +/// ZERO defects — no record-less/mark-less paragraphs to trigger a repair storm. +#[test] +fn synthetic_docx_roundtrip_has_no_projection_defects() { + let document = rich_document(400); + let path = temp_docx_path(); + write_docx(&path, &document).expect("write docx"); + let bytes = std::fs::read(&path).expect("read docx"); + let _ = std::fs::remove_file(&path); + let projection = assert_import_defect_free(&bytes, "synthetic roundtrip"); + // Defect-freeness alone would also pass if the break were silently DROPPED; + // pin the soft break to U+2028 in the imported text as well. + assert!( + projection + .text + .to_string() + .contains(&soft_break_probe_text()), + "synthetic roundtrip: body-level soft break did not survive import as U+2028", + ); +} + +/// Collect the vertical alignment of every projected body run, in order. +fn body_run_vert_aligns(projection: &flowstate_document::DocumentProjection) -> Vec { + projection + .paragraphs + .iter() + .flat_map(|paragraph| paragraph.runs.iter()) + .map(|run| run.styles.vert_align) + .collect() +} + +/// Phase-1 parser net: a real OOXML `` run property is captured on +/// import and lands on the projected run styles (superscript AND subscript). +/// This reads genuine Word XML — not our own exporter's output — so it cannot be +/// fooled by a shared import/export bug. +#[test] +fn vert_align_run_property_is_captured_on_import() { + let document_xml = concat!( + r#""#, + r#""#, + r#""#, + r#""#, + r#"E=mc"#, + r#"2"#, + r#" and CO"#, + r#"2"#, + r#""#, + r#""# + ); + let rels = concat!( + r#""#, + r#""# + ); + let bytes = build_docx_zip(document_xml, rels, &[]); + let projection = assert_import_defect_free(&bytes, "vert-align import"); + + let aligns = body_run_vert_aligns(&projection); + assert!( + aligns.contains(&VertAlign::Superscript), + "superscript `w:vertAlign` run must import as VertAlign::Superscript; saw {aligns:?}" + ); + assert!( + aligns.contains(&VertAlign::Subscript), + "subscript `w:vertAlign` run must import as VertAlign::Subscript; saw {aligns:?}" + ); +} + +/// Phase-1+2 net: super/subscript on a model run survives a real +/// export → re-import round-trip (`write_docx` emits ``, the +/// importer reads it back). Guards the exporter half specifically. +#[test] +fn vert_align_survives_docx_write_and_reimport() { + let vert_run = |text: &str, vert_align: VertAlign| InputRun { + text: text.to_string(), + styles: RunStyles { + vert_align, + ..RunStyles::default() + }, + }; + let blocks = vec![InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![ + vert_run("E=mc", VertAlign::Baseline), + vert_run("2", VertAlign::Superscript), + vert_run(" and CO", VertAlign::Baseline), + vert_run("2", VertAlign::Subscript), + ], + })]; + let document = document_from_input_blocks(flowstate_document_theme(), blocks); + + let path = temp_docx_path(); + write_docx(&path, &document).expect("write docx"); + let bytes = std::fs::read(&path).expect("read docx"); + let _ = std::fs::remove_file(&path); + + let projection = assert_import_defect_free(&bytes, "vert-align roundtrip"); + let aligns = body_run_vert_aligns(&projection); + assert!( + aligns.contains(&VertAlign::Superscript) && aligns.contains(&VertAlign::Subscript), + "super/subscript must survive write_docx → reimport; saw {aligns:?}" + ); +} + +/// Run the same defect-free assertion against a REAL .docx supplied via +/// `FLOWSTATE_DOCX_FIXTURE=/path/to/file.docx` — point it at the impact-defense doc to +/// confirm (or catch) whether that specific file imports clean. Skips when unset so CI is +/// unaffected. +#[test] +fn external_docx_import_has_no_projection_defects() { + let Some(path) = std::env::var_os("FLOWSTATE_DOCX_FIXTURE") else { + eprintln!("FLOWSTATE_DOCX_FIXTURE unset — skipping external docx import check"); + return; + }; + let bytes = std::fs::read(&path).expect("read FLOWSTATE_DOCX_FIXTURE"); + assert_import_defect_free(&bytes, &format!("external docx {path:?}")); +} diff --git a/crates/flowstate-docx/src/interpreter.rs b/crates/flowstate-docx/src/interpreter.rs index 12d978b1..a41757ff 100644 --- a/crates/flowstate-docx/src/interpreter.rs +++ b/crates/flowstate-docx/src/interpreter.rs @@ -1,3 +1,6 @@ +mod omml; +mod structured; + use std::{io, path::Path}; use quick_xml::{ @@ -13,9 +16,21 @@ use rustc_hash::{FxHashMap, FxHashSet}; use super::cleaner::{CleanedDocx, DocxCleanReport, clean_docx_path}; use flowstate_document::{ - Document, DocumentParagraphInput, DocumentRunInput, DocumentTheme, ParagraphStyle, RunSemanticStyle, RunStyles, document_from_paragraphs, + DocumentParagraphInput, DocumentProjection, DocumentRunInput, ImportedLoroDocument, ParagraphStyle, RunSemanticStyle, RunStyles, + SOFT_LINE_BREAK_STR, VertAlign, document_from_input_blocks, document_from_paragraphs, flowstate_document_theme, import_document_projection, }; +/// Map an OOXML `w:vertAlign` value (`superscript` / `subscript` / `baseline`) +/// to the model's [`VertAlign`]. Unknown/absent ⇒ `Baseline`. +fn vert_align_from_ooxml(value: Option<&str>) -> VertAlign { + match value { + Some("superscript") => VertAlign::Superscript, + Some("subscript") => VertAlign::Subscript, + _ => VertAlign::Baseline, + } +} +use flowstate_fidelity::FidelityClass; + pub const RECOGNITION_RULES: &[RecognitionRule] = &[ RecognitionRule::ParagraphStyle { docx_style_id: "Heading1", @@ -83,35 +98,243 @@ pub struct DocxConversionReport { pub recognition_rules: &'static [RecognitionRule], pub paragraphs_imported: usize, pub runs_imported: usize, + pub tables_imported: usize, + pub images_imported: usize, + pub equations_imported: usize, pub unknown_paragraph_styles: Vec, pub unknown_run_styles: Vec, } #[hotpath::measure] -pub fn convert_docx_to_document(path: impl AsRef) -> io::Result<(Document, DocxConversionReport)> { +pub fn convert_docx_to_document(path: impl AsRef) -> io::Result<(DocumentProjection, DocxConversionReport)> { let cleaned = clean_docx_path(path)?; convert_cleaned_docx_to_document(cleaned) } #[hotpath::measure] -pub fn convert_docx_bytes_to_document(bytes: &[u8]) -> io::Result<(Document, DocxConversionReport)> { +pub fn convert_docx_bytes_to_document(bytes: &[u8]) -> io::Result<(DocumentProjection, DocxConversionReport)> { let cleaned = super::cleaner::clean_docx_bytes(bytes)?; convert_cleaned_docx_to_document(cleaned) } #[hotpath::measure] -pub fn convert_cleaned_docx_to_document(cleaned: CleanedDocx) -> io::Result<(Document, DocxConversionReport)> { - let CleanedDocx { - bytes, - main_document_xml, - report: clean_report, - } = cleaned; - let docx = RDocxDocument::from_bytes(&bytes).map_err(rdocx_error)?; - let direct_properties = match main_document_xml.as_deref() { - Some(doc_xml) => direct_properties_by_paragraph_xml(doc_xml)?, - None => direct_properties_by_paragraph_package(&bytes)?, +pub fn convert_cleaned_docx_to_document(cleaned: CleanedDocx) -> io::Result<(DocumentProjection, DocxConversionReport)> { + let interpreted = interpret_cleaned_docx(&cleaned)?; + build_structured_document(&cleaned, interpreted) +} + +/// Walks the cleaned DOCX body into the structured block model (paragraphs, +/// tables, images, equations) and folds the object counts into the report. When +/// the body has no tables/images/equations the structured walk is discarded and +/// the flat paragraph projection is used, preserving the prior behavior exactly. +#[hotpath::measure] +fn build_structured_document(cleaned: &CleanedDocx, interpreted: InterpretedDocx) -> io::Result<(DocumentProjection, DocxConversionReport)> { + // §perf: the structured walk (full DOM parse, OPC package open, per-run text + // clones) is discarded whenever the body has no tables/images/equations, yet it + // ran unconditionally. When the captured main-document XML provably contains + // none of those object markers, skip the entire build and take the object-free + // path directly — byte-identical to the discard branch below. + // + // Correctness: the walk recognizes elements by prefix-stripped local name and + // detects equations via the `oMath` substring, so scanning the same bytes for + // the bare substrings `tbl`/`drawing`/`oMath` can never produce a false + // negative (a real object's start tag always contains its local name regardless + // of namespace prefix). False positives (the substring appearing in text) only + // forgo the optimization and fall through to the identical full build. When no + // captured XML is available (`None`), `interpret_structured` would read from the + // package, so we conservatively run the full build (`is_some_and` → `false`). + let object_free = cleaned.main_document_xml.as_deref().is_some_and(|xml| { + memchr::memmem::find(xml, b"tbl").is_none() + && memchr::memmem::find(xml, b"drawing").is_none() + && memchr::memmem::find(xml, b"oMath").is_none() + // §act-eleven C7 (fixture-caught): legacy VML images (`w:pict`) are + // objects too — without this probe a VML-ONLY document took the + // object-free fast path and its images were never scanned at all. + && memchr::memmem::find(xml, b"imagedata").is_none() + }); + + let (document, report) = if object_free { + let mut report = interpreted.report; + // Matches the counts a fully-walked object-free body would yield. + report.tables_imported = 0; + report.images_imported = 0; + report.equations_imported = 0; + let document = document_from_paragraphs(flowstate_document_theme(), interpreted.paragraphs); + (document, report) + } else { + // §act-nine A9.2: the structured walk consumes the CT_Document the + // direct-properties pass already parsed instead of building its own owned + // XmlNode tree (collapses the third heavy document.xml parse). + let structured = structured::interpret_structured(cleaned, &interpreted.document, &interpreted.paragraphs)?; + let mut report = interpreted.report; + report.tables_imported = structured.tables_imported; + report.images_imported = structured.images_imported; + report.equations_imported = structured.equations_imported; + + let has_objects = structured.tables_imported + structured.images_imported + structured.equations_imported > 0; + let document = if has_objects { + let mut document = document_from_input_blocks(flowstate_document_theme(), structured.blocks); + for (asset_id, record) in structured.assets { + document.assets.assets.insert(asset_id, record); + } + document + } else { + document_from_paragraphs(flowstate_document_theme(), interpreted.paragraphs) + }; + (document, report) + }; + // Import/export fidelity: record the shape of the projection produced from the + // cleaned DOCX. Shared funnel for both the db8 and Loro import entries. + flowstate_fidelity::event(FidelityClass::ImportExport, "import-docx", || { + format!( + "paragraphs={} runs={} tables={} images={} equations={} sections={}", + report.paragraphs_imported, + report.runs_imported, + report.tables_imported, + report.images_imported, + report.equations_imported, + document.sections.len() + ) + }); + Ok((document, report)) +} + +/// Imports DOCX semantics directly into the canonical Loro document and returns +/// the frontier-matched initial projection. No package, snapshot, search cache, +/// or second Loro projection is created on the open path. +#[hotpath::measure] +pub fn import_docx_to_loro(path: impl AsRef, title: &str) -> io::Result<(ImportedLoroDocument, DocxConversionReport)> { + let cleaned = clean_docx_path(path)?; + import_cleaned_docx_to_loro(cleaned, title) +} + +#[hotpath::measure] +pub fn import_docx_bytes_to_loro(bytes: &[u8], title: &str) -> io::Result<(ImportedLoroDocument, DocxConversionReport)> { + let cleaned = super::cleaner::clean_docx_bytes(bytes)?; + import_cleaned_docx_to_loro(cleaned, title) +} + +#[hotpath::measure] +pub fn import_cleaned_docx_to_loro(cleaned: CleanedDocx, title: &str) -> io::Result<(ImportedLoroDocument, DocxConversionReport)> { + let interpreted = interpret_cleaned_docx(&cleaned)?; + let (document, report) = build_structured_document(&cleaned, interpreted)?; + let imported = import_document_projection(document, title)?; + Ok((imported, report)) +} + +struct InterpretedDocx { + paragraphs: Vec, + /// The typed main-document parse harvested for `DirectParagraphFacts`, kept + /// alive so the structured object walk can consume it too (§act-nine A9.2). + document: CT_Document, + report: DocxConversionReport, +} + +/// rdocx's run/paragraph `text()` collapses every `` (line, page, and +/// column alike) to '\n' before the break type is visible here. A docx break is +/// an INTRA-paragraph line break, while '\n' is the paragraph separator in the +/// body flow — letting it through fabricates a bare paragraph boundary with no +/// metadata/block/style record, which full reprojection reports as +/// `missing_paragraph_*` defects (see `structured::collect_run_text`, the +/// table-cell counterpart of this mapping). Remap to the model's soft break. +/// Page/column breaks also becoming soft breaks matches the table-cell path +/// and is provisional pending a product decision on their semantics. `` +/// is dropped inside rdocx itself and never reaches this text. +fn soften_rdocx_breaks(text: String) -> String { + // U+FFFC (OBJECT_REPLACEMENT) is a RESERVED body-flow encoding sentinel: the + // projector treats every U+FFFC as an object placeholder. A literal one sitting + // in source run text (seen in real docx) reads back as an OrphanObjectPlaceholder + // defect, so strip it at ingestion. Rare, so the contains-checks keep the common + // path allocation-free. + let strip_object = text.contains('\u{FFFC}'); + // §act-ten A9.6: normalize CR too. quick-xml does not perform XML line-ending + // normalization, so a literal `\r\n` (or bare `\r`) inside `w:t` survives into + // run text; the un-normalized `\r` then rides the body flow as a phantom char + // the exporter cannot represent — the corpus caught a 52-CRLF doc losing one + // char per CRLF on roundtrip. Map CRLF/CR/LF each to ONE soft line break, the + // same visual Word gives them. + let soften = text.contains('\n') || text.contains('\r'); + match (strip_object, soften) { + (false, false) => text, + (true, false) => text.replace('\u{FFFC}', ""), + (false, true) => soften_line_endings(&text), + (true, true) => soften_line_endings(&text.replace('\u{FFFC}', "")), + } +} + +fn soften_line_endings(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars().peekable(); + while let Some(ch) = chars.next() { + match ch { + '\r' => { + if chars.peek() == Some(&'\n') { + chars.next(); + } + out.push_str(SOFT_LINE_BREAK_STR); + }, + '\n' => out.push_str(SOFT_LINE_BREAK_STR), + other => out.push(other), + } + } + out +} + +#[hotpath::measure] +fn interpret_cleaned_docx(cleaned: &CleanedDocx) -> io::Result { + // §act-eleven A11.2: ONE typed parse + ONE unzip feed rdocx, the facts + // harvest, AND (via `InterpretedDocx.document`) the structured walk. The + // cleaner's already-open package supplies styles.xml; the vendored + // `from_parsed_parts` skips rdocx's own document/numbering/core parses and + // the `cleaned.bytes` re-inflate. Files whose package or captured + // document.xml is unavailable fall back to the previous overlapped + // two-parse path — output is byte-identical either way (corpus-netted). + let (docx, mut parsed_document, direct_properties) = match (cleaned.package.as_deref(), cleaned.main_document_xml.as_deref()) { + (Some(package), Some(doc_xml)) => { + // §act-five P6 carried over: the light streaming run-border pass hides + // under the one heavy typed parse. + let probe_t0 = std::time::Instant::now(); + let (document, run_borders) = std::thread::scope(|scope| { + let borders = scope.spawn(|| direct_run_borders_by_paragraph_xml(doc_xml)); + // §act-twelve A12.3.1-full: fragment-parallel typed parse (equal tree, + // corpus-netted; falls back to the sequential parse on any anomaly). + let document = crate::fragment_parse::ct_document_from_xml(doc_xml).map_err(rdocx_oxml_error); + let borders = borders + .join() + .unwrap_or_else(|_| Err(io::Error::other("docx run-border parse thread panicked"))); + (document, borders) + }); + let probe_parse = probe_t0.elapsed(); + let document = document?; + let probe_t1 = std::time::Instant::now(); + let facts = direct_paragraph_facts(&document, &run_borders?); + let docx = RDocxDocument::from_parsed_parts(package, document).map_err(rdocx_error)?; + static PARSE_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + if *PARSE_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_PARSE_PROBE").is_some()) { + eprintln!("[flowstate-parse-probe] typed_parse={probe_parse:?} facts+parts={:?}", probe_t1.elapsed()); + } + (docx, None, facts) + }, + _ => { + let (docx, direct_properties) = std::thread::scope(|scope| { + let direct_properties = scope.spawn(|| match cleaned.main_document_xml.as_deref() { + Some(doc_xml) => direct_properties_by_paragraph_xml(doc_xml), + None => direct_properties_by_paragraph_package(&cleaned.bytes), + }); + let docx = RDocxDocument::from_bytes(&cleaned.bytes).map_err(rdocx_error); + ( + docx, + direct_properties + .join() + .expect("direct-properties parse thread panicked"), + ) + }); + let docx = docx?; + let (document, direct_properties) = direct_properties?; + (docx, Some(document), direct_properties) + }, }; - drop(main_document_xml); + let probe_walk_t0 = std::time::Instant::now(); let style_resolver = StyleResolver::new(&docx); let docx_paragraphs = docx.paragraphs(); let mut paragraphs = Vec::with_capacity(docx_paragraphs.len()); @@ -144,7 +367,7 @@ pub fn convert_cleaned_docx_to_document(cleaned: CleanedDocx) -> io::Result<(Doc .runs() .enumerate() .map(|(run_ix, run)| { - let text = run.text(); + let text = soften_rdocx_breaks(run.text()); let run_style_id = run.style_id().map(str::to_owned); let run_style_id_ref = run_style_id.as_deref(); let effective_properties = run_property_cache @@ -166,6 +389,11 @@ pub fn convert_cleaned_docx_to_document(cleaned: CleanedDocx) -> io::Result<(Doc source_size_pt, size_pt: source_size_pt.or_else(|| effective.sz.map(rdocx_oxml::HalfPoint::to_pt)), color: run.color().is_some() || direct.color || effective.color.is_some() || effective.color_theme.is_some(), + vert_align: if direct.vert_align == VertAlign::Baseline { + vert_align_from_ooxml(effective.vert_align.as_deref()) + } else { + direct.vert_align + }, } }) .collect::>(); @@ -259,7 +487,7 @@ pub fn convert_cleaned_docx_to_document(cleaned: CleanedDocx) -> io::Result<(Doc } if runs.is_empty() { - let text = paragraph.text(); + let text = soften_rdocx_breaks(paragraph.text()); if !text.is_empty() { runs.push(DocumentRunInput { text, @@ -272,17 +500,33 @@ pub fn convert_cleaned_docx_to_document(cleaned: CleanedDocx) -> io::Result<(Doc paragraphs.push(DocumentParagraphInput { style, runs }); } - let paragraphs_imported = paragraphs.len(); - let document = document_from_paragraphs(DocumentTheme::default(), paragraphs); let report = DocxConversionReport { - clean: clean_report, + clean: cleaned.report.clone(), recognition_rules: RECOGNITION_RULES, - paragraphs_imported, + paragraphs_imported: paragraphs.len(), runs_imported, + tables_imported: 0, + images_imported: 0, + equations_imported: 0, unknown_paragraph_styles, unknown_run_styles, }; - Ok((document, report)) + let document = parsed_document + .take() + .unwrap_or_else(|| docx.into_typed_document()); + static PARSE_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + if *PARSE_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_PARSE_PROBE").is_some()) { + eprintln!( + "[flowstate-parse-probe] conversion_walk={:?} paragraphs={}", + probe_walk_t0.elapsed(), + paragraphs.len() + ); + } + Ok(InterpretedDocx { + paragraphs, + document, + report, + }) } #[derive(Clone, Debug)] struct RunFact { @@ -297,6 +541,7 @@ struct RunFact { source_size_pt: Option, size_pt: Option, color: bool, + vert_align: VertAlign, } #[derive(Clone, Debug, Default)] @@ -315,6 +560,7 @@ struct DirectRunProperties { border: bool, size_pt: Option, color: bool, + vert_align: VertAlign, } struct EffectiveParagraphProperties<'properties> { @@ -330,7 +576,7 @@ impl ParagraphProperties for EffectiveParagraphProperties<'_> { } #[hotpath::measure] -fn direct_properties_by_paragraph_package(bytes: &[u8]) -> io::Result> { +fn direct_properties_by_paragraph_package(bytes: &[u8]) -> io::Result<(CT_Document, Vec)> { let package = OpcPackage::from_reader(std::io::Cursor::new(bytes)).map_err(rdocx_opc_error)?; let doc_part_name = package .main_document_part() @@ -341,66 +587,87 @@ fn direct_properties_by_paragraph_package(bytes: &[u8]) -> io::Result io::Result> { - let document = CT_Document::from_xml(doc_xml).map_err(rdocx_oxml_error)?; - let run_borders_by_paragraph = direct_run_borders_by_paragraph_xml(doc_xml)?; - Ok( - document - .body - .paragraphs() - .enumerate() - .map(|(paragraph_ix, paragraph)| { - let paragraph_run_borders = run_borders_by_paragraph - .get(paragraph_ix) - .map(Vec::as_slice) - .unwrap_or_default(); - let runs = paragraph - .runs - .iter() - .enumerate() - .map(|(run_ix, run)| { - let Some(properties) = run.properties.as_ref() else { - return DirectRunProperties { - border: paragraph_run_borders - .get(run_ix) - .copied() - .unwrap_or_default(), - ..DirectRunProperties::default() - }; - }; - DirectRunProperties { - bold: properties.bold == Some(true) || properties.bold_cs == Some(true), - bold_off: properties.bold == Some(false) && properties.bold_cs != Some(true), - underline: underline_is_on(properties.underline.as_ref()), - strikethrough: properties.strike == Some(true) || properties.dstrike == Some(true), - highlight: properties.highlight.is_some() || properties.shading.is_some(), +fn direct_properties_by_paragraph_xml(doc_xml: &[u8]) -> io::Result<(CT_Document, Vec)> { + // §act-five P6: the two INDEPENDENT parses of the same bytes — the heavy + // structured `CT_Document::from_xml` and the light streaming run-border pass — + // overlap on a scoped thread so the border parse hides under the structured + // one (both borrow `doc_xml` immutably; the scope guarantees it outlives them). + let (document, run_borders_by_paragraph) = std::thread::scope(|scope| { + let borders = scope.spawn(|| direct_run_borders_by_paragraph_xml(doc_xml)); + let document = crate::fragment_parse::ct_document_from_xml(doc_xml).map_err(rdocx_oxml_error); + let borders = borders + .join() + .unwrap_or_else(|_| Err(io::Error::other("docx run-border parse thread panicked"))); + (document, borders) + }); + let document = document?; + let facts = direct_paragraph_facts(&document, &run_borders_by_paragraph?); + Ok((document, facts)) +} + +/// §act-eleven A11.2: the facts harvest over an already-parsed typed tree — +/// shared by the single-parse fast path (which hands the tree straight to +/// rdocx afterward) and the legacy overlapped path above. +#[hotpath::measure] +fn direct_paragraph_facts(document: &CT_Document, run_borders_by_paragraph: &[Vec]) -> Vec { + document + .body + .paragraphs() + .enumerate() + .map(|(paragraph_ix, paragraph)| { + let paragraph_run_borders = run_borders_by_paragraph + .get(paragraph_ix) + .map(Vec::as_slice) + .unwrap_or_default(); + let runs = paragraph + .runs + .iter() + .enumerate() + .map(|(run_ix, run)| { + let Some(properties) = run.properties.as_ref() else { + return DirectRunProperties { border: paragraph_run_borders .get(run_ix) .copied() .unwrap_or_default(), - size_pt: properties.sz.map(|size| size.to_pt()), - color: properties.color.is_some() || properties.color_theme.is_some(), - } - }) - .collect(); - DirectParagraphFacts { - outline_lvl: paragraph - .properties - .as_ref() - .and_then(|properties| properties.outline_lvl), - runs, - } - }) - .collect(), - ) + ..DirectRunProperties::default() + }; + }; + DirectRunProperties { + bold: properties.bold == Some(true) || properties.bold_cs == Some(true), + bold_off: properties.bold == Some(false) && properties.bold_cs != Some(true), + underline: underline_is_on(properties.underline.as_ref()), + strikethrough: properties.strike == Some(true) || properties.dstrike == Some(true), + highlight: properties.highlight.is_some() || properties.shading.is_some(), + border: paragraph_run_borders + .get(run_ix) + .copied() + .unwrap_or_default(), + size_pt: properties.sz.map(|size| size.to_pt()), + color: properties.color.is_some() || properties.color_theme.is_some(), + vert_align: vert_align_from_ooxml(properties.vert_align.as_deref()), + } + }) + .collect(); + DirectParagraphFacts { + outline_lvl: paragraph + .properties + .as_ref() + .and_then(|properties| properties.outline_lvl), + runs, + } + }) + .collect() } #[hotpath::measure] fn direct_run_borders_by_paragraph_xml(doc_xml: &[u8]) -> io::Result>> { let mut reader = XmlReader::from_reader(doc_xml); reader.config_mut().trim_text(false); - let mut buf = Vec::new(); let mut paragraphs = Vec::new(); let mut current_paragraph: Option> = None; let mut in_run = false; @@ -408,7 +675,9 @@ fn direct_run_borders_by_paragraph_xml(doc_xml: &[u8]) -> io::Result { current_paragraph = Some(Vec::new()); }, @@ -442,7 +711,6 @@ fn direct_run_borders_by_paragraph_xml(doc_xml: &[u8]) -> io::Result return Err(io::Error::new(io::ErrorKind::InvalidData, error)), _ => {}, } - buf.clear(); } Ok(paragraphs) @@ -641,6 +909,10 @@ fn recognize_run_styles_for_context( direct_underline: structural_run_formatting_allowed && run.underline, strikethrough: !suppress_semantic_styles && run.strikethrough, highlight: (direct_highlight_allowed && run.highlight).then_some(flowstate_document::HIGHLIGHT_SPOKEN), + // Vertical alignment is orthogonal to the debate semantics and is preserved + // as-is (a `CO₂` subscript in a Tag keeps its subscript) — losslessly carried + // even where the semantic slot is suppressed for headings. + vert_align: run.vert_align, } } @@ -974,6 +1246,7 @@ mod tests { source_size_pt: None, size_pt: None, color: false, + vert_align: VertAlign::Baseline, } } diff --git a/crates/flowstate-docx/src/interpreter/omml.rs b/crates/flowstate-docx/src/interpreter/omml.rs new file mode 100644 index 00000000..a3494161 --- /dev/null +++ b/crates/flowstate-docx/src/interpreter/omml.rs @@ -0,0 +1,478 @@ +//! Minimal Office `MathML` (OMML, `m:oMath`) → `LaTeX` conversion for DOCX import. +//! +//! Word stores equations as OMML, while the Flowstate equation model stores a +//! single LaTeX source string ([`InputEquationSyntax::Latex`] is the only syntax +//! the equation block exposes). No available crate converts OMML → LaTeX (the +//! `mitex` crate converts LaTeX → Typst), so this module implements a focused, +//! best-effort converter over the common OMML constructs (runs/text, fractions, +//! sub/superscripts, radicals, delimiters, n-ary operators, functions, accents, +//! overbars, limits and matrices). Unrecognized constructs degrade to their +//! concatenated literal text so that symbols are preserved rather than dropped. + +use flowstate_document::{InputEquationBlock, InputEquationDisplay, InputEquationSyntax}; +use quick_xml::{ + Reader as XmlReader, + events::{BytesStart, Event}, +}; + +/// Parse a captured OMML container (`m:oMath`, `m:oMathPara`, or any element that +/// wraps them) and return one [`InputEquationBlock`] per `m:oMath`, in document +/// order. `m:oMath` inside an `m:oMathPara` becomes a display equation; a bare +/// `m:oMath` becomes an inline-like equation. +#[hotpath::measure] +pub(super) fn equations_from_container_bytes(bytes: &[u8]) -> Vec { + let Some(root) = parse_tree(bytes) else { + return Vec::new(); + }; + let mut equations = Vec::new(); + collect_equations(&root, false, &mut equations); + equations +} + +/// Returns `true` when the captured bytes look like they contain Office Math, so +/// callers can cheaply skip non-math captured XML (bookmarks, SDTs, ...). +pub(super) fn contains_office_math(bytes: &[u8]) -> bool { + memmem(bytes, b"oMath") +} + +fn collect_equations(node: &Node, in_math_para: bool, out: &mut Vec) { + if node.local == "oMath" { + let source = latex_from_children(node).trim().to_string(); + if !source.is_empty() { + out.push(InputEquationBlock { + source, + syntax: InputEquationSyntax::Latex, + display: if in_math_para { + InputEquationDisplay::Display + } else { + InputEquationDisplay::InlineLikeParagraph + }, + }); + } + return; + } + let nested = in_math_para || node.local == "oMathPara"; + for child in &node.children { + collect_equations(child, nested, out); + } +} + +struct Node { + local: String, + val: Option, + text: String, + children: Vec, +} + +fn parse_tree(bytes: &[u8]) -> Option { + let mut reader = XmlReader::from_reader(bytes); + reader.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut stack: Vec = Vec::new(); + let mut root: Option = None; + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(event)) => stack.push(node_from_start(&event)), + Ok(Event::Empty(event)) => { + let node = node_from_start(&event); + if let Some(parent) = stack.last_mut() { + parent.children.push(node); + } else { + root.get_or_insert(node); + } + }, + Ok(Event::Text(event)) => { + if let Some(top) = stack.last_mut() + && let Ok(text) = event.xml10_content() + { + top.text.push_str(&text); + } + }, + Ok(Event::End(_)) => { + if let Some(node) = stack.pop() { + if let Some(parent) = stack.last_mut() { + parent.children.push(node); + } else { + root.get_or_insert(node); + } + } + }, + Ok(Event::Eof) => break, + Err(_) => return None, + _ => {}, + } + buf.clear(); + } + + root +} + +fn node_from_start(event: &BytesStart<'_>) -> Node { + let local = local_name(event.name().as_ref()).to_owned(); + let mut val = None; + for attr in event.attributes().flatten() { + if local_name(attr.key.as_ref()) == "val" { + val = std::str::from_utf8(attr.value.as_ref()) + .ok() + .map(|value| value.to_owned()); + } + } + Node { + local, + val, + text: String::new(), + children: Vec::new(), + } +} + +fn local_name(name: &[u8]) -> &str { + let name = std::str::from_utf8(name).unwrap_or_default(); + name.rsplit(':').next().unwrap_or(name) +} + +fn child<'node>(node: &'node Node, local: &str) -> Option<&'node Node> { + node.children.iter().find(|child| child.local == local) +} + +fn latex_from_children(node: &Node) -> String { + let mut out = String::new(); + for child in &node.children { + out.push_str(&latex_from_node(child)); + } + out +} + +fn latex_from_arg(node: &Node, local: &str) -> String { + child(node, local) + .map(latex_from_children) + .unwrap_or_default() +} + +fn latex_from_node(node: &Node) -> String { + match node.local.as_str() { + "t" => map_text(&node.text), + "r" => latex_from_children(node), + "f" => format!("\\frac{{{}}}{{{}}}", latex_from_arg(node, "num"), latex_from_arg(node, "den")), + "sSup" => format!("{}^{{{}}}", braced(&latex_from_arg(node, "e")), latex_from_arg(node, "sup")), + "sSub" => format!("{}_{{{}}}", braced(&latex_from_arg(node, "e")), latex_from_arg(node, "sub")), + "sSubSup" | "sPre" => format!( + "{}_{{{}}}^{{{}}}", + braced(&latex_from_arg(node, "e")), + latex_from_arg(node, "sub"), + latex_from_arg(node, "sup") + ), + "rad" => { + let inner = latex_from_arg(node, "e"); + let degree = latex_from_arg(node, "deg"); + if degree.is_empty() { + format!("\\sqrt{{{inner}}}") + } else { + format!("\\sqrt[{degree}]{{{inner}}}") + } + }, + "d" => { + let (open, close) = delimiter_chars(node); + let inner = node + .children + .iter() + .filter(|child| child.local == "e") + .map(latex_from_children) + .collect::>() + .join(","); + format!("\\left{open}{inner}\\right{close}") + }, + "nary" => latex_nary(node), + "func" => { + let name = latex_from_arg(node, "fName"); + let argument = latex_from_arg(node, "e"); + if argument.is_empty() { name } else { format!("{name} {argument}") } + }, + "acc" => latex_accent(node), + "bar" => format!("\\overline{{{}}}", latex_from_arg(node, "e")), + "limLow" => format!("{}_{{{}}}", braced(&latex_from_arg(node, "e")), latex_from_arg(node, "lim")), + "limUpp" => format!("{}^{{{}}}", braced(&latex_from_arg(node, "e")), latex_from_arg(node, "lim")), + "m" => latex_matrix(node), + "box" | "borderBox" | "groupChr" | "eqArr" | "e" | "num" | "den" | "oMath" => latex_from_children(node), + // Property containers carry presentation hints only, never rendered content. + name if name.ends_with("Pr") => String::new(), + _ => latex_from_children(node), + } +} + +fn braced(value: &str) -> String { + if value.chars().count() == 1 { + value.to_owned() + } else { + format!("{{{value}}}") + } +} + +fn delimiter_chars(node: &Node) -> (String, String) { + let properties = child(node, "dPr"); + let beginning = properties + .and_then(|properties| child(properties, "begChr")) + .and_then(|chr| chr.val.clone()); + let ending = properties + .and_then(|properties| child(properties, "endChr")) + .and_then(|chr| chr.val.clone()); + (latex_delimiter(beginning.as_deref(), "("), latex_delimiter(ending.as_deref(), ")")) +} + +fn latex_delimiter(chr: Option<&str>, default: &str) -> String { + match chr { + None => default.to_owned(), + Some("") => ".".to_owned(), + Some("{") => "\\{".to_owned(), + Some("}") => "\\}".to_owned(), + Some("‖") => "\\|".to_owned(), + Some("⟨") => "\\langle".to_owned(), + Some("⟩") => "\\rangle".to_owned(), + Some("⌊") => "\\lfloor".to_owned(), + Some("⌋") => "\\rfloor".to_owned(), + Some("⌈") => "\\lceil".to_owned(), + Some("⌉") => "\\rceil".to_owned(), + Some(other) => other.to_owned(), + } +} + +fn latex_nary(node: &Node) -> String { + let chr = child(node, "naryPr") + .and_then(|properties| child(properties, "chr")) + .and_then(|chr| chr.val.clone()); + let mut out = nary_operator(chr.as_deref()); + let sub = latex_from_arg(node, "sub"); + if !sub.is_empty() { + out.push_str("_{"); + out.push_str(&sub); + out.push('}'); + } + let sup = latex_from_arg(node, "sup"); + if !sup.is_empty() { + out.push_str("^{"); + out.push_str(&sup); + out.push('}'); + } + let operand = latex_from_arg(node, "e"); + if !operand.is_empty() { + out.push(' '); + out.push_str(&operand); + } + out +} + +fn nary_operator(chr: Option<&str>) -> String { + match chr { + None | Some("∑") => "\\sum".to_owned(), + Some("∏") => "\\prod".to_owned(), + Some("∐") => "\\coprod".to_owned(), + Some("∫") => "\\int".to_owned(), + Some("∬") => "\\iint".to_owned(), + Some("∭") => "\\iiint".to_owned(), + Some("∮") => "\\oint".to_owned(), + Some("⋃") => "\\bigcup".to_owned(), + Some("⋂") => "\\bigcap".to_owned(), + Some("⋁") => "\\bigvee".to_owned(), + Some("⋀") => "\\bigwedge".to_owned(), + Some(other) => other.to_owned(), + } +} + +fn latex_accent(node: &Node) -> String { + let chr = child(node, "accPr") + .and_then(|properties| child(properties, "chr")) + .and_then(|chr| chr.val.clone()); + let command = match chr.as_deref() { + Some("\u{0303}" | "~") => "\\tilde", + Some("\u{0304}" | "¯") => "\\bar", + Some("\u{0307}") => "\\dot", + Some("\u{0308}") => "\\ddot", + Some("\u{20D7}" | "→") => "\\vec", + Some("\u{0306}") => "\\breve", + Some("\u{030C}") => "\\check", + Some("\u{0301}") => "\\acute", + Some("\u{0300}") => "\\grave", + _ => "\\hat", + }; + format!("{command}{{{}}}", latex_from_arg(node, "e")) +} + +fn latex_matrix(node: &Node) -> String { + let rows = node + .children + .iter() + .filter(|child| child.local == "mr") + .map(|row| { + row + .children + .iter() + .filter(|child| child.local == "e") + .map(latex_from_children) + .collect::>() + .join(" & ") + }) + .collect::>() + .join(" \\\\ "); + format!("\\begin{{matrix}}{rows}\\end{{matrix}}") +} + +fn map_text(text: &str) -> String { + let mut out = String::new(); + for ch in text.chars() { + if let Some(mapped) = map_symbol(ch) { + out.push_str(mapped); + } else { + push_escaped(&mut out, ch); + } + } + out +} + +fn push_escaped(out: &mut String, ch: char) { + match ch { + '%' | '$' | '#' | '&' | '_' | '{' | '}' => { + out.push('\\'); + out.push(ch); + }, + '\\' => out.push_str("\\backslash "), + _ => out.push(ch), + } +} + +#[allow(clippy::too_many_lines, reason = "A single flat symbol table is the clearest mapping form.")] +fn map_symbol(ch: char) -> Option<&'static str> { + Some(match ch { + 'α' => "\\alpha ", + 'β' => "\\beta ", + 'γ' => "\\gamma ", + 'δ' => "\\delta ", + 'ε' => "\\epsilon ", + 'ζ' => "\\zeta ", + 'η' => "\\eta ", + 'θ' => "\\theta ", + 'ι' => "\\iota ", + 'κ' => "\\kappa ", + 'λ' => "\\lambda ", + 'μ' => "\\mu ", + 'ν' => "\\nu ", + 'ξ' => "\\xi ", + 'π' => "\\pi ", + 'ρ' => "\\rho ", + 'σ' => "\\sigma ", + 'τ' => "\\tau ", + 'υ' => "\\upsilon ", + 'φ' => "\\phi ", + 'χ' => "\\chi ", + 'ψ' => "\\psi ", + 'ω' => "\\omega ", + 'Γ' => "\\Gamma ", + 'Δ' => "\\Delta ", + 'Θ' => "\\Theta ", + 'Λ' => "\\Lambda ", + 'Ξ' => "\\Xi ", + 'Π' => "\\Pi ", + 'Σ' => "\\Sigma ", + 'Φ' => "\\Phi ", + 'Ψ' => "\\Psi ", + 'Ω' => "\\Omega ", + '×' => "\\times ", + '÷' => "\\div ", + '±' => "\\pm ", + '∓' => "\\mp ", + '⋅' => "\\cdot ", + '∗' => "\\ast ", + '≤' => "\\le ", + '≥' => "\\ge ", + '≠' => "\\ne ", + '≈' => "\\approx ", + '≡' => "\\equiv ", + '∝' => "\\propto ", + '→' => "\\to ", + '←' => "\\leftarrow ", + '⇒' => "\\Rightarrow ", + '⇔' => "\\Leftrightarrow ", + '↔' => "\\leftrightarrow ", + '∞' => "\\infty ", + '∂' => "\\partial ", + '∇' => "\\nabla ", + '∈' => "\\in ", + '∉' => "\\notin ", + '⊂' => "\\subset ", + '⊆' => "\\subseteq ", + '⊃' => "\\supset ", + '⊇' => "\\supseteq ", + '∪' => "\\cup ", + '∩' => "\\cap ", + '∀' => "\\forall ", + '∃' => "\\exists ", + '∅' => "\\emptyset ", + '∧' => "\\wedge ", + '∨' => "\\vee ", + '¬' => "\\neg ", + '∑' => "\\sum ", + '∏' => "\\prod ", + '∫' => "\\int ", + '√' => "\\surd ", + '°' => "^{\\circ}", + '′' => "'", + '″' => "''", + _ => return None, + }) +} + +fn memmem(haystack: &[u8], needle: &[u8]) -> bool { + // §perf: memchr's SIMD substring search is O(n) vs the O(n·m) `windows` scan. + // The empty-needle guard preserves the prior `false` result (memmem::find + // returns Some(0) for an empty needle). + !needle.is_empty() && memchr::memmem::find(haystack, needle).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sources(bytes: &str) -> Vec { + equations_from_container_bytes(bytes.as_bytes()) + .into_iter() + .map(|equation| equation.source) + .collect() + } + + #[test] + fn inline_fraction_converts_to_latex() { + let omml = r"12"; + let equations = equations_from_container_bytes(omml.as_bytes()); + assert_eq!(equations.len(), 1); + assert_eq!(equations[0].source, "\\frac{1}{2}"); + assert_eq!(equations[0].display, InputEquationDisplay::InlineLikeParagraph); + } + + #[test] + fn math_para_marks_display_and_handles_superscript() { + let omml = + r"x2"; + let equations = equations_from_container_bytes(omml.as_bytes()); + assert_eq!(equations.len(), 1); + assert_eq!(equations[0].source, "x^{2}"); + assert_eq!(equations[0].display, InputEquationDisplay::Display); + } + + #[test] + fn nary_sum_uses_limits_and_operand() { + let omml = r#"i=1ni"#; + assert_eq!(sources(omml), vec!["\\sum_{i=1}^{n} i".to_string()]); + } + + #[test] + fn radical_and_symbols_are_mapped() { + let omml = r"α≤β"; + assert_eq!(sources(omml), vec!["\\sqrt{\\alpha \\le \\beta }".to_string()]); + } + + #[test] + fn non_math_xml_yields_no_equations() { + let bookmark = r#""#; + assert!(equations_from_container_bytes(bookmark.as_bytes()).is_empty()); + assert!(!contains_office_math(bookmark.as_bytes())); + } +} diff --git a/crates/flowstate-docx/src/interpreter/structured.rs b/crates/flowstate-docx/src/interpreter/structured.rs new file mode 100644 index 00000000..eefbe5fd --- /dev/null +++ b/crates/flowstate-docx/src/interpreter/structured.rs @@ -0,0 +1,2037 @@ +//! Structured DOCX body import (§26): walks `w:body` in document order and emits +//! the full block model — paragraphs, tables, images and equations — rather than +//! the flat paragraph stream produced by [`super::interpret_cleaned_docx`]. +//! +//! Paragraph recognition is *not* duplicated here. `docx.paragraphs()` (and the +//! pre-computed [`DocumentParagraphInput`] slice it backs) enumerates exactly the +//! top-level `w:p` children of `w:body` in order — table-cell paragraphs are +//! filtered out by `rdocx` (`CT_Body::paragraphs` only keeps `BodyContent::Paragraph`). +//! The walk therefore consumes one slice entry per top-level `w:p` it encounters, +//! keeping the rich paragraph/run heuristics intact, and recognizes table-cell +//! paragraphs through a deliberately small independent path (plain runs + Normal +//! style). +//! +//! Ordering choice: a top-level paragraph that also carries inline drawings or +//! Office Math is emitted as the paragraph block first, then one image block per +//! drawing, then one equation block per `m:oMath`. Empty paragraphs that exist +//! only to host an object are still emitted (kept simple to preserve cursor +//! alignment) and render as a blank line next to the object. +//! +//! §act-nine A9.2: the walk consumes the ALREADY-PARSED [`CT_Document`] produced +//! by the direct-properties pass instead of building its own owned [`XmlNode`] +//! tree (that was the third heavy `document.xml` parse per import). The typed +//! model captures unknown wrappers (`mc:AlternateContent`, inline `w:sdt`, +//! `w:smartTag`, tracked changes, ...) as raw byte chunks rather than typed +//! content, so a cheap parity pre-scan ([`typed_walk_diverges`]) probes those +//! chunks (and a handful of typed-parser blind spots in the raw XML) and, on any +//! hit, falls back to the OLD `parse_tree` walk for the WHOLE document — the +//! correctness escape hatch that keeps the corpus output byte-identical for +//! exotic markup. The `XmlNode`/`parse_tree` machinery is retained solely for +//! that fallback. + +use std::{borrow::Cow, io, io::Cursor, sync::Arc}; + +use quick_xml::{ + Reader as XmlReader, + events::{BytesStart, Event}, +}; +use rdocx_opc::OpcPackage; +use rdocx_oxml::document::{BodyContent, CT_Body, CT_Document}; +use rdocx_oxml::drawing::CT_Drawing; +use rdocx_oxml::table::{CT_Tbl, CT_Tc, CellContent, VMerge}; +use rdocx_oxml::text::{BreakType, CT_P, RunContent}; +use rustc_hash::{FxHashMap, FxHashSet}; + +use crate::cleaner::CleanedDocx; +use flowstate_document::{ + AssetId, AssetRecord, CellId, ColumnId, DocumentParagraphInput, InputBlock, InputBlockAlignment, InputImageBlock, InputImageSizing, + InputParagraph, InputRun, InputTableBlock, InputTableCell, InputTableCellBlock, InputTableColumn, InputTableColumnWidth, InputTableRow, + InputTableStyle, ParagraphStyle, RowId, RunStyles, SOFT_LINE_BREAK, +}; + +use super::omml; + +/// Structured import result. Blocks are in body order; `assets` are the deduped +/// image assets keyed by a content-stable [`AssetId`] for insertion into the +/// projection's asset store. +#[derive(Default)] +pub(super) struct StructuredDocx { + pub(super) blocks: Vec, + pub(super) assets: Vec<(AssetId, AssetRecord)>, + pub(super) tables_imported: usize, + pub(super) images_imported: usize, + pub(super) equations_imported: usize, +} + +#[hotpath::measure] +pub(super) fn interpret_structured( + cleaned: &CleanedDocx, + document: &CT_Document, + paragraphs: &[DocumentParagraphInput], +) -> io::Result { + // The raw main-document XML is still needed: the parity pre-scan probes it for + // typed-parser blind spots, and the fallback path parses it into `XmlNode`s. + let owned_xml; + let doc_xml: &[u8] = match cleaned.main_document_xml.as_deref() { + Some(xml) => xml, + None => match main_document_xml_from_package(cleaned) { + Some(xml) => { + owned_xml = xml; + &owned_xml + }, + None => return Ok(StructuredDocx::default()), + }, + }; + + // §A9.2 point 3: when the typed model provably cannot reproduce the old + // XmlNode walk byte-for-byte (objects/text hiding in raw-captured wrappers, or + // markup shapes the typed parser drops without capture), take the OLD path for + // the whole document. Bounded parity risk, corpus-netted. + // Escape hatch: FLOWSTATE_DOCX_TYPED_WALK=0 forces the old tree walk (parity + // debugging + field fallback while the typed walk hardens). + static TYPED_WALK_DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + let typed_disabled = *TYPED_WALK_DISABLED.get_or_init(|| std::env::var_os("FLOWSTATE_DOCX_TYPED_WALK").is_some_and(|value| value == "0")); + if typed_disabled || typed_walk_diverges(document, doc_xml, paragraphs.len()) { + return interpret_structured_via_tree(cleaned, doc_xml, paragraphs); + } + + let mut walker = TypedWalker { + paragraphs, + cursor: 0, + blocks: Vec::new(), + images: ImageAssets::new(cleaned), + tables_imported: 0, + images_imported: 0, + equations_imported: 0, + }; + walker.walk_body(&document.body); + + Ok(finish_structured( + walker.blocks, + walker.images.assets, + walker.tables_imported, + walker.images_imported, + walker.equations_imported, + )) +} + +/// Fetches the uncompressed main-document XML when the cleaner did not capture +/// it (reuses the cleaner's in-memory package when available). +fn main_document_xml_from_package(cleaned: &CleanedDocx) -> Option> { + let opened; + let package: &OpcPackage = match cleaned.package.as_deref() { + Some(package) => package, + None => { + opened = OpcPackage::from_reader(Cursor::new(cleaned.bytes.as_slice())).ok()?; + &opened + }, + }; + let main_part = package.main_document_part()?; + package.get_part(&main_part).map(<[u8]>::to_vec) +} + +/// Shared tail for both walks: guarantee the body flow ends in a paragraph row. +/// +/// When the last imported block is an object (inline image/equation/table — +/// e.g. a doc whose final paragraph holds an image and is followed only by +/// ``), the projector otherwise FABRICATES a trailing paragraph with +/// no durable record (MissingParagraphMetadata/Block defects, boundary None). +/// Write that record ourselves: append the same Normal empty paragraph the +/// projector would synthesize, so the readback finds it. +fn finish_structured( + mut blocks: Vec, + assets: Vec<(AssetId, AssetRecord)>, + tables_imported: usize, + images_imported: usize, + equations_imported: usize, +) -> StructuredDocx { + if !matches!(blocks.last(), Some(InputBlock::Paragraph(_))) { + blocks.push(InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + })); + } + StructuredDocx { + blocks, + assets, + tables_imported, + images_imported, + equations_imported, + } +} + +// -- Image assets (shared by both walks) ------------------------------------ + +/// Image-asset resolution and dedup. The OPC package is the cleaner's +/// already-open in-memory one when available; otherwise it is opened LAZILY from the +/// packaged bytes on the first drawing that needs resolving, so tables-only +/// documents (the common carded-evidence case) never pay the unzip at all. +struct ImageAssets<'ctx> { + cleaned: &'ctx CleanedDocx, + /// `None` = not yet attempted; `Some(None)` = attempted and unavailable. + #[allow(clippy::option_option, reason = "tri-state lazy-open memo: unattempted vs attempted-and-absent vs open")] + opened: Option, String)>>, + assets: Vec<(AssetId, AssetRecord)>, + emitted_assets: FxHashSet, +} + +impl<'ctx> ImageAssets<'ctx> { + fn new(cleaned: &'ctx CleanedDocx) -> Self { + Self { + cleaned, + opened: None, + assets: Vec::new(), + emitted_assets: FxHashSet::default(), + } + } + + /// A missing/unreadable package simply yields no image assets rather than + /// failing the import (same behavior as the eager open it replaces). + fn resolve_image(&mut self, relationship_id: &str) -> Option { + let cleaned = self.cleaned; + let opened = self + .opened + .get_or_insert_with(|| open_package_main_part(cleaned)); + let (package, main_part) = opened.as_ref()?; + let relationship = package + .get_part_rels(main_part)? + .get_by_id(relationship_id)?; + let target = OpcPackage::resolve_rel_target(main_part, &relationship.target); + let bytes = package.get_part(&target)?; + let asset_id = asset_id_from_bytes(bytes); + if self.emitted_assets.insert(asset_id) { + self.assets.push(( + asset_id, + AssetRecord { + id: asset_id, + mime_type: mime_from_path(&target).into(), + original_name: Some(file_name(&target).into()), + content_hash: AssetRecord::stable_content_hash(bytes), + bytes: Arc::new(bytes.to_vec()), + }, + )); + } + Some(asset_id) + } + + /// §act-eleven A11.9: resolve a relationship id to its EXTERNAL target URL + /// (`TargetMode="External"`). A genuinely-LINKED image has no part bytes, so + /// no [`AssetRecord`] is created; the returned [`AssetId`] is derived from + /// the URL bytes (same hash as [`asset_id_from_bytes`]) so the id stays + /// stable across imports. Internal-mode (or missing) relationships return + /// `None` — the embedded path is never affected. + fn resolve_external_image(&mut self, relationship_id: &str) -> Option<(AssetId, String)> { + let cleaned = self.cleaned; + let opened = self + .opened + .get_or_insert_with(|| open_package_main_part(cleaned)); + let (package, main_part) = opened.as_ref()?; + let relationship = package + .get_part_rels(main_part)? + .get_by_id(relationship_id)?; + if !relationship + .target_mode + .as_deref() + .is_some_and(|mode| mode.eq_ignore_ascii_case("external")) + { + return None; + } + // rdocx-opc keeps attribute values RAW (no XML unescape); URLs routinely + // carry `&` in query strings — decode to the real URL. + let url = unescape_attribute_value(&relationship.target); + if url.is_empty() { + return None; + } + Some((asset_id_from_bytes(url.as_bytes()), url)) + } + + /// §act-ten A9.6: legacy VML images (``), the + /// pre-DrawingML form Word still emits for pasted/legacy content. Both walks + /// dropped these wholesale (one of the four corpus roundtrip residuals is a + /// 27-image VML doc). Extract every `v:imagedata` relationship, resolve it + /// through the SAME asset path as `DrawingML` blips, and size from the + /// enclosing shape's `style="width:…pt;height:…pt"` when present. + /// §act-eleven A11.9: when no embeddable part resolves via `r:id`/`o:relid` + /// but `r:href` names an external-mode relationship, the image imports as a + /// LINKED image carrying that URL instead of being dropped. + fn vml_images_from_chunk(&mut self, chunk: &[u8]) -> Vec { + if memchr::memmem::find(chunk, b")> = None; + loop { + match reader.read_event_into(&mut buf) { + Ok(quick_xml::events::Event::Start(node)) | Ok(quick_xml::events::Event::Empty(node)) => { + let name = node.name(); + let local = local_name(name.as_ref()); + if matches!(local, "shape" | "rect" | "oval") { + shape_extent_px = vml_style_extent_px(&node); + } else if local == "imagedata" { + let mut rel_id = None; + let mut href_id = None; + for attribute in node.attributes().flatten() { + let key = local_name(attribute.key.as_ref()); + // `r:id` is the embedded part; `o:relid` is the legacy Office + // alias. `r:href` is the EXTERNAL companion (no part bytes) — + // used only when no embeddable part resolves (§A11.9). + if matches!(key, "id" | "relid") { + rel_id = Some(String::from_utf8_lossy(&attribute.value).into_owned()); + } else if key == "href" { + href_id = Some(String::from_utf8_lossy(&attribute.value).into_owned()); + } + } + let sizing = match shape_extent_px { + Some((width_px, height_px)) => InputImageSizing::Fixed { width_px, height_px }, + None => InputImageSizing::Intrinsic, + }; + if let Some(asset_id) = rel_id.and_then(|rel_id| self.resolve_image(&rel_id)) { + images.push(InputImageBlock { + asset_id, + alt_text: String::new(), + caption: None, + sizing, + alignment: InputBlockAlignment::Left, + external_url: None, + }); + } else if let Some((asset_id, url)) = href_id.and_then(|href_id| self.resolve_external_image(&href_id)) { + // §A11.9: no embeddable part, but the `r:href` companion is a + // genuine external-mode relationship — import as a LINKED image. + images.push(InputImageBlock { + asset_id, + alt_text: String::new(), + caption: None, + sizing, + alignment: InputBlockAlignment::Left, + external_url: Some(url), + }); + } + } + }, + Ok(quick_xml::events::Event::Eof) | Err(_) => break, + _ => {}, + } + buf.clear(); + } + images + } +} + +/// Parse a VML shape `style` attribute's `width`/`height` point values into +/// pixels (96 dpi, the DrawingML-import convention). +fn vml_style_extent_px(node: &quick_xml::events::BytesStart<'_>) -> Option<(u32, Option)> { + let style = node + .attributes() + .flatten() + .find(|attribute| local_name(attribute.key.as_ref()) == "style")?; + let style = String::from_utf8_lossy(&style.value).into_owned(); + let dimension_px = |key: &str| -> Option { + let value = style.split(';').find_map(|entry| { + entry + .trim() + .strip_prefix(key) + .and_then(|rest| rest.trim_start().strip_prefix(':')) + })?; + let points: f64 = value.trim().trim_end_matches("pt").parse().ok()?; + let px = (points * 96.0 / 72.0).round(); + (px >= 1.0).then_some(px as u32) + }; + let width = dimension_px("width")?; + Some((width, dimension_px("height"))) +} + +fn open_package_main_part(cleaned: &CleanedDocx) -> Option<(Arc, String)> { + // §act-nine A9.2: reuse the package the cleaner already opened (unzip #1) + // instead of re-inflating from bytes; the from-bytes open remains only as a + // fallback for callers that built a `CleanedDocx` without one. + let package = match cleaned.package.clone() { + Some(package) => package, + None => Arc::new(OpcPackage::from_reader(Cursor::new(cleaned.bytes.as_slice())).ok()?), + }; + let main_part = package.main_document_part()?; + Some((package, main_part)) +} + +// -- Typed walk (the fast path) ---------------------------------------------- + +struct TypedWalker<'ctx> { + paragraphs: &'ctx [DocumentParagraphInput], + cursor: usize, + blocks: Vec, + images: ImageAssets<'ctx>, + tables_imported: usize, + images_imported: usize, + equations_imported: usize, +} + +impl TypedWalker<'_> { + fn walk_body(&mut self, body: &CT_Body) { + for content in &body.content { + match content { + BodyContent::Paragraph(paragraph) => self.walk_paragraph(paragraph), + BodyContent::Table(table) => { + let table = self.parse_table(table); + self.blocks.push(InputBlock::Table(table)); + }, + BodyContent::RawXml(chunk) => { + // Parity note (§A9.2 point 3): a SELF-CLOSED `` parses as + // `RawXml` (the typed body parser only types `Start`-form paragraphs), + // but the old XmlNode walker treated it as a paragraph and consumed a + // recognized-slice entry — replicate that exactly so the cursor and + // the emitted block sequence stay identical. All other raw chunks were + // vetted object-free by `typed_walk_diverges` and are skipped, which + // matches the old walker (it only descended into `p`/`tbl` children). + if chunk_root_local(chunk) == b"p" { + self.walk_empty_paragraph(); + } + }, + } + } + } + + /// The typed-parser image of a self-closed ``: no runs, no objects. + fn walk_empty_paragraph(&mut self) { + let paragraph = if self.cursor < self.paragraphs.len() { + let input = input_paragraph_from_document(&self.paragraphs[self.cursor]); + self.cursor += 1; + input + } else { + InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + } + }; + self.blocks.push(InputBlock::Paragraph(paragraph)); + } + + fn walk_paragraph(&mut self, paragraph_node: &CT_P) { + // Top-level paragraphs are 1:1 with the pre-computed slice; the cursor only + // advances on `w:p`, so tables, section properties and SDTs never desync it. + let paragraph = if self.cursor < self.paragraphs.len() { + let input = input_paragraph_from_document(&self.paragraphs[self.cursor]); + self.cursor += 1; + input + } else { + cell_paragraph_typed(paragraph_node) + }; + + // Resolve the paragraph's inline objects (images AND equations) first so we + // can tell whether this is a bare OBJECT wrapper. Drawings come from + // the typed run content; equations come from the `m:oMath`/`m:oMathPara` + // chunks the typed parser captured verbatim into `extra_xml` (exactly the + // `&[u8]` input `omml::equations_from_container_bytes` takes — no byte + // spans needed). Objects hiding in OTHER wrappers were routed to the + // fallback by `typed_walk_diverges`. + let mut images = Vec::new(); + for run in ¶graph_node.runs { + for content in &run.content { + if let RunContent::Drawing(drawing) = content + && let Some(image) = self.image_from_drawing(drawing) + { + images.push(image); + } + } + // §act-ten A9.6: legacy VML picts land in the runs' raw chunks (unknown + // to the typed parser). The parity pre-scan lets these through (they + // carry no `drawing`/`oMath`/`tbl` markers), and the OLD walker collects + // them identically, so both paths import VML the same way. + for chunk in &run.extra_xml { + images.extend(self.images.vml_images_from_chunk(chunk)); + } + } + for (_, chunk) in ¶graph_node.extra_xml { + images.extend(self.images.vml_images_from_chunk(chunk)); + } + let equations: Vec<_> = paragraph_node + .extra_xml + .iter() + .filter(|(_, chunk)| omml::contains_office_math(chunk)) + .flat_map(|(_, chunk)| omml::equations_from_container_bytes(chunk)) + .collect(); + + // §perf-heaven T7.18/T7.21/T8.10 (see the fallback walker for the full + // rationale): a that carries object(s) and no paragraph text is a bare + // object wrapper — emit only the inline object(s), no empty Paragraph. + let paragraph_is_text_empty = paragraph.runs.iter().all(|run| run.text.trim().is_empty()); + let has_objects = !images.is_empty() || !equations.is_empty(); + let collapse_bare_object_wrapper = has_objects && paragraph_is_text_empty; + if !collapse_bare_object_wrapper { + self.blocks.push(InputBlock::Paragraph(paragraph)); + } + + for image in images { + self.blocks.push(InputBlock::Image(image)); + self.images_imported += 1; + } + + for equation in equations { + self.blocks.push(InputBlock::Equation(equation)); + self.equations_imported += 1; + } + } + + fn parse_table(&mut self, table_node: &CT_Tbl) -> InputTableBlock { + self.tables_imported += 1; + let column_widths: Vec = table_node + .grid + .as_ref() + .map(|grid| { + grid + .columns + .iter() + .map(|column| column_width_from_twips(i64::from(column.width.0))) + .collect() + }) + .unwrap_or_default(); + let mut rows: Vec = Vec::new(); + let mut header_row = false; + let mut first_row = true; + // Widest row in grid columns, so the durable columns cover every cell's + // grid position even when a row spans more columns than `tblGrid` declares. + let mut grid_width = 0_usize; + // grid column -> (row index, cell index) of the cell that started a vertical + // merge, so continuation cells fold into its `row_span` instead of emitting. + let mut vertical_open: FxHashMap = FxHashMap::default(); + + for row_node in &table_node.rows { + if first_row { + first_row = false; + // `` only — attributed/expanded forms fall back to the + // old walk (the typed parser records bare presence, see the pre-scan). + header_row = row_node + .properties + .as_ref() + .is_some_and(|properties| properties.header == Some(true)); + } + // Deterministic row id, seeded per-table so row ids are GLOBALLY unique + // across every table in the document (see the fallback walker for the full + // rationale). The increment ORDER matches the old walk exactly — nested + // tables bump the seed lazily, when their anchor cell's blocks are built. + let row_id = RowId(((self.tables_imported as u128) << 40) | (rows.len() as u128 + 1)); + let mut cells: Vec = Vec::new(); + let mut grid_col = 0_usize; + for cell_node in &row_node.cells { + grid_col = self.add_table_cell(cell_node, row_id, grid_col, &mut rows, &mut cells, &mut vertical_open); + } + grid_width = grid_width.max(grid_col); + rows.push(InputTableRow { id: row_id, cells }); + } + + let columns = build_table_columns(&column_widths, grid_width); + fill_full_grid(&mut rows, &columns); + InputTableBlock { + rows, + columns, + style: InputTableStyle { header_row }, + } + } + + /// Appends one `w:tc` to the current row (or folds a vertical-merge + /// continuation into the originating cell above) and returns the next grid + /// column. `rows` holds the already-completed rows; `cells` is the row in + /// progress. + fn add_table_cell( + &mut self, + cell_node: &CT_Tc, + row_id: RowId, + grid_col: usize, + rows: &mut [InputTableRow], + cells: &mut Vec, + vertical_open: &mut FxHashMap, + ) -> usize { + let col_span = cell_grid_span_typed(cell_node); + // The cell's durable column is the sequential id of its starting grid column. + let column_id = ColumnId(grid_col as u128 + 1); + match cell_vertical_merge_typed(cell_node) { + VerticalMerge::Continue => { + if let Some(&(row_ix, cell_ix)) = vertical_open.get(&grid_col) + && let Some(row) = rows.get_mut(row_ix) + && let Some(cell) = row.cells.get_mut(cell_ix) + { + cell.row_span = cell.row_span.saturating_add(1); + } + }, + VerticalMerge::Restart => { + vertical_open.insert(grid_col, (rows.len(), cells.len())); + let blocks = self.cell_blocks(cell_node); + cells.push(InputTableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks, + row_span: 1, + col_span, + }); + }, + VerticalMerge::None => { + vertical_open.remove(&grid_col); + let blocks = self.cell_blocks(cell_node); + cells.push(InputTableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks, + row_span: 1, + col_span, + }); + }, + } + grid_col + usize::from(col_span.max(1)) + } + + fn cell_blocks(&mut self, cell_node: &CT_Tc) -> Vec { + let mut blocks = Vec::new(); + for content in &cell_node.content { + match content { + CellContent::Paragraph(paragraph) => blocks.push(InputTableCellBlock::Paragraph(cell_paragraph_typed(paragraph))), + CellContent::Table(nested) => blocks.push(InputTableCellBlock::Table(self.parse_table(nested))), + } + } + if blocks.is_empty() { + blocks.push(InputTableCellBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + })); + } + blocks + } + + fn image_from_drawing(&mut self, drawing: &CT_Drawing) -> Option { + let (embed_id, extent_cx, extent_cy, description, name) = match (drawing.inline.as_ref(), drawing.anchor.as_ref()) { + (Some(inline), _) => ( + inline.embed_id.as_str(), + inline.extent_cx.0, + inline.extent_cy.0, + inline.description.as_deref(), + inline.name.as_deref(), + ), + (None, Some(anchor)) => ( + anchor.embed_id.as_str(), + anchor.extent_cx.0, + anchor.extent_cy.0, + anchor.description.as_deref(), + anchor.name.as_deref(), + ), + (None, None) => return None, + }; + // §perf-heaven T7.20 parity: rdocx keeps attribute values RAW (quick-xml + // does not auto-unescape attributes) while the old XmlNode walker unescaped + // them — Word writes multi-line alt text with ` ` entities. + let alt_text = unescape_attribute_value(description.or(name).unwrap_or_default()); + // No `a:blip r:embed` found by the typed parse. §act-eleven A11.9: a blip + // carrying `r:link` INSTEAD (a genuinely-linked image) resolves through the + // external-mode relationship to its URL; the typed model does not parse + // `r:link`, so it is read from the drawing's preserved raw bytes. Anything + // else (VML-only drawings, malformed blips) is skipped as before. + if embed_id.is_empty() { + let link_id = blip_link_id_from_raw(drawing_raw_xml(drawing))?; + let (asset_id, url) = self.images.resolve_external_image(&link_id)?; + return Some(InputImageBlock { + asset_id, + alt_text, + caption: None, + sizing: sizing_from_extent_emu(extent_cx, extent_cy), + alignment: InputBlockAlignment::Left, + external_url: Some(url), + }); + } + let asset_id = self.images.resolve_image(embed_id)?; + Some(InputImageBlock { + asset_id, + alt_text, + caption: None, + sizing: sizing_from_extent_emu(extent_cx, extent_cy), + alignment: InputBlockAlignment::Left, + external_url: None, + }) + } +} + +/// Lightweight cell-paragraph recognition over the typed model: table-cell +/// paragraphs are not part of the recognized slice, so they are imported as +/// Normal-styled plain text. This trades run-level semantics inside tables for +/// a small, robust path. The text mapping mirrors [`collect_run_text`]: +/// * U+FFFC object-placeholder sentinels are stripped (reserved body-flow +/// encoding, see `soften_rdocx_breaks`); +/// * `w:tab` → '\t'; +/// * a LINE `w:br` → the model's soft break (U+2028), NEVER '\n' ('\n' is the +/// paragraph separator in the body flow — §perf-heaven T7.22); +/// * page/column breaks are pagination, dropped (no inline representation); +/// * drawings/fields carrying harvestable text and `` never reach here — +/// the parity pre-scan routes those documents to the fallback walk. +fn cell_paragraph_typed(paragraph_node: &CT_P) -> InputParagraph { + let mut text = String::new(); + for run in ¶graph_node.runs { + for content in &run.content { + match content { + RunContent::Text(run_text) => text.extend(run_text.text.chars().filter(|&ch| ch != '\u{FFFC}')), + RunContent::Tab => text.push('\t'), + RunContent::Break(BreakType::Line) => text.push(SOFT_LINE_BREAK), + RunContent::Break(BreakType::Page | BreakType::Column) => {}, + RunContent::Drawing(_) | RunContent::Field { .. } | RunContent::FootnoteRef { .. } | RunContent::EndnoteRef { .. } => {}, + } + } + } + let runs = if text.is_empty() { + Vec::new() + } else { + vec![InputRun { + text, + styles: RunStyles::default(), + }] + }; + InputParagraph { + style: ParagraphStyle::Normal, + runs, + } +} + +fn cell_grid_span_typed(cell_node: &CT_Tc) -> u16 { + cell_node + .properties + .as_ref() + .and_then(|properties| properties.grid_span) + .and_then(|span| u16::try_from(span).ok()) + .filter(|span| *span >= 1) + .unwrap_or(1) +} + +fn cell_vertical_merge_typed(cell_node: &CT_Tc) -> VerticalMerge { + // `` and `w:val="continue"` continue the merge above; only an + // explicit `restart` opens a new vertical span (the typed parser already + // folds every non-`restart` value to `Continue`). + match cell_node + .properties + .as_ref() + .and_then(|properties| properties.v_merge) + { + None => VerticalMerge::None, + Some(VMerge::Restart) => VerticalMerge::Restart, + Some(VMerge::Continue) => VerticalMerge::Continue, + } +} + +// -- Parity pre-scan (§A9.2 point 3) ----------------------------------------- + +/// Substrings whose presence in a raw-captured chunk means an object the typed +/// walk cannot see (same bare-substring reasoning as the `object_free` gate in +/// `build_structured_document`: a real object's start tag always contains its +/// local name; false positives only forgo the fast path). +const OBJECT_PROBES: &[&[u8]] = &[b"tbl", b"drawing", b"oMath"]; + +/// Substrings that mean HARVESTABLE CELL TEXT hides inside a raw-captured +/// wrapper (or a drawing) — the old walk's recursive `collect_run_text` +/// descended into everything and collected every `t`-local text (`w:t`, `m:t`, +/// `DrawingML` `a:t`), tabs and line breaks. Prefix matches over-trigger +/// (` bool { + let mut has_table = false; + let mut paragraph_ix = 0_usize; + for content in &document.body.content { + match content { + BodyContent::RawXml(chunk) => { + // Objects inside an unknown BODY-LEVEL wrapper (top-level w:sdt, + // mc:AlternateContent, ...) — fall back so the exotic wrapper keeps its + // old-walk treatment. + if contains_any(chunk, OBJECT_PROBES) { + return true; + } + // A self-closed `` consumes a slice entry (see `walk_body`). + if chunk_root_local(chunk) == b"p" { + paragraph_ix += 1; + } + }, + BodyContent::Paragraph(paragraph) => { + // Beyond-slice paragraphs take the cell-paragraph path in the walk, so + // they inherit the cell-text parity triggers too. + let treat_as_cell = paragraph_ix >= recognized_paragraphs; + paragraph_ix += 1; + if body_paragraph_diverges(paragraph, treat_as_cell) { + return true; + } + }, + BodyContent::Table(table) => { + has_table = true; + if table_diverges(table) { + return true; + } + }, + } + } + raw_probes_diverge(doc_xml, has_table) +} + +fn body_paragraph_diverges(paragraph: &CT_P, treat_as_cell: bool) -> bool { + if treat_as_cell && cell_paragraph_diverges(paragraph) { + return true; + } + for (_, chunk) in ¶graph.extra_xml { + // A drawing inside ANY paragraph-level raw chunk (unknown wrapper, or even + // nested inside a captured oMath) was found by the old `collect_descendants`. + if contains_any(chunk, &[b"drawing"]) { + return true; + } + // `m:oMath`/`m:oMathPara`-ROOTED chunks are the typed walk's equation input + // (fed verbatim to omml); oMath inside any OTHER wrapper is invisible to it. + if !chunk_root_is_office_math(chunk) && contains_any(chunk, &[b"oMath"]) { + return true; + } + } + for run in ¶graph.runs { + for chunk in &run.extra_xml { + // Run-level unknown wrappers (mc:AlternateContent, w:pict, w:object, ...) + // hiding a drawing or math. + if contains_any(chunk, OBJECT_PROBES) { + return true; + } + } + for content in &run.content { + if let RunContent::Drawing(drawing) = content { + let raw = drawing_raw_xml(drawing); + // Equations inside a drawing's text box were reachable through the old + // whole-paragraph byte span. + if contains_any(raw, &[b"oMath"]) { + return true; + } + // Multiple embedded blips in one drawing: the old walk took the FIRST + // `blip@r:embed` in document order, the typed parse keeps the LAST. + if memchr::memmem::find_iter(raw, b":embed=").take(2).count() > 1 { + return true; + } + } + } + } + false +} + +fn cell_paragraph_diverges(paragraph: &CT_P) -> bool { + // pPr>tabs>w:tab STOP DEFINITIONS: the old recursive text collection matched + // every element with local name `tab` — including tab-stop definitions inside + // pPr — fabricating one '\t' per stop in the cell text. Bug-for-bug parity is + // owed to the corpus net, so route those documents to the old walk. + if paragraph + .properties + .as_ref() + .is_some_and(|properties| properties.tabs.is_some()) + { + return true; + } + for (_, chunk) in ¶graph.extra_xml { + if contains_any(chunk, CELL_TEXT_PROBES) { + return true; + } + } + for run in ¶graph.runs { + for chunk in &run.extra_xml { + if contains_any(chunk, CELL_TEXT_PROBES) { + return true; + } + } + for content in &run.content { + match content { + // `w:fldSimple` becomes a synthetic Field run and its display runs are + // skipped by the typed parse; the old walk collected their text. + RunContent::Field { .. } => return true, + // Text boxes inside a cell drawing: the old walk collected their + // `w:t`/`a:t` descendants into the cell text. + RunContent::Drawing(drawing) if contains_any(drawing_raw_xml(drawing), CELL_TEXT_PROBES) => { + return true; + }, + _ => {}, + } + } + } + false +} + +fn table_diverges(table: &CT_Tbl) -> bool { + table.rows.iter().any(|row| { + row.cells.iter().any(|cell| { + cell.content.iter().any(|content| match content { + CellContent::Paragraph(paragraph) => cell_paragraph_diverges(paragraph), + CellContent::Table(nested) => table_diverges(nested), + }) + }) + }) +} + +/// Raw-XML probes for shapes the typed parser drops WITHOUT capturing anything +/// (so no typed-side trigger can exist for them). +fn raw_probes_diverge(doc_xml: &[u8], has_table: bool) -> bool { + if has_table { + // `` is dropped by the typed run parse; the old cell walk emitted a + // soft break for it. Body paragraphs are unaffected (their text comes from + // the same rdocx parse on both paths), so this only matters with tables. + if contains_any(doc_xml, &[b"...`) or attributed + // `tblHeader` forms therefore diverge from the old attr-aware walk. + if tbl_header_probe_diverges(doc_xml) { + return true; + } + for tag in [&b"`/`` are dropped by the typed table parse but + // produced (empty) rows/cells in the old walk. + if tag_occurrence_matches(doc_xml, b"` INSIDE a table cell: `CT_Tc` drops it silently, but + // the old walk emitted an empty cell paragraph for it. A cell whose ONLY + // paragraph is one self-closed `` is benign — the typed walk's empty + // placeholder equals the old empty paragraph — so only a dropped `w:p` + // COEXISTING with other cell paragraphs diverges (this keeps the fast path + // for the very common Word empty-cell shape). (Body-level self-closed + // paragraphs are typed as RawXml and replayed exactly — see `walk_body`.) + // A nested table truncates its host cell's span at the first inner + // ``; the host's trailing region is the one documented residual this + // span heuristic can miss. + if any_wrapped_span(doc_xml, b"", cell_span_drops_paragraph) { + return true; + } + } + // The typed paragraph parser keeps only `w:r` children of `w:hyperlink` and + // skips everything else with NO raw capture; the old walk descended into the + // whole hyperlink subtree (equations there were imported; wrapped text inside + // cells was collected). + if any_wrapped_span(doc_xml, b"", |span| { + contains_any(span, HYPERLINK_INTERIOR_PROBES) + }) { + return true; + } + // `w:fldSimple` children are skipped by the typed parse; the old walk found + // drawings/equations inside them. + any_wrapped_span(doc_xml, b"", |span| { + contains_any(span, &[b"drawing", b"oMath"]) + }) +} + +/// `` is the only form both parsers agree on; anything else +/// (attributes — the typed parser ignores `w:val="false"` — or an expanded +/// element it cannot see at all) falls back. +fn tbl_header_probe_diverges(doc_xml: &[u8]) -> bool { + memchr::memmem::find_iter(doc_xml, b"` ALONGSIDE at least +/// one other paragraph (the shape where the typed parse's silent drop changes +/// the emitted block list — see the call site). Truncated markup conservatively +/// diverges. +fn cell_span_drops_paragraph(span: &[u8]) -> bool { + let mut paragraphs = 0_usize; + let mut self_closed = 0_usize; + for start in memchr::memmem::find_iter(span, b" return true, + Some(b' ' | b'\t' | b'\r' | b'\n' | b'/' | b'>') => {}, + Some(_) => continue, + } + paragraphs += 1; + let Some(close) = memchr::memchr(b'>', rest) else { + return true; + }; + if close > 0 && rest[close - 1] == b'/' { + self_closed += 1; + } + } + self_closed >= 1 && paragraphs >= 2 +} + +/// Whether any occurrence of `tag_open` (name-boundary checked) is self-closed +/// (`self_closed == true`) or expanded (`self_closed == false`). A truncated +/// tag (no `>` found) conservatively matches either form. +fn tag_occurrence_matches(haystack: &[u8], tag_open: &[u8], self_closed: bool) -> bool { + for start in memchr::memmem::find_iter(haystack, tag_open) { + let rest = &haystack[start + tag_open.len()..]; + // The needle must end the tag NAME (` return true, + Some(b' ' | b'\t' | b'\r' | b'\n' | b'/' | b'>') => {}, + Some(_) => continue, + } + let Some(close) = memchr::memchr(b'>', rest) else { + return true; + }; + let occurrence_self_closed = close > 0 && rest[close - 1] == b'/'; + if occurrence_self_closed == self_closed { + return true; + } + } + false +} + +/// Runs `probe` over every `open`..`close` span. Non-nesting elements only; a +/// self-closed `open` with a later sibling makes the span overshoot into +/// following content, which can only OVER-trigger the fallback (safe direction). +fn any_wrapped_span(doc_xml: &[u8], open: &[u8], close: &[u8], probe: impl Fn(&[u8]) -> bool) -> bool { + for start in memchr::memmem::find_iter(doc_xml, open) { + let rest = &doc_xml[start + open.len()..]; + let Some(end) = memchr::memmem::find(rest, close) else { + continue; + }; + if probe(&rest[..end]) { + return true; + } + } + false +} + +fn contains_any(haystack: &[u8], needles: &[&[u8]]) -> bool { + needles + .iter() + .any(|needle| memchr::memmem::find(haystack, needle).is_some()) +} + +/// Local name of a raw-captured chunk's root element (empty when malformed). +fn chunk_root_local(chunk: &[u8]) -> &[u8] { + let Some(rest) = chunk.strip_prefix(b"<") else { + return &[]; + }; + let name_end = rest + .iter() + .position(|&byte| matches!(byte, b' ' | b'\t' | b'\r' | b'\n' | b'/' | b'>')) + .unwrap_or(rest.len()); + let name = &rest[..name_end]; + name.rsplit(|&byte| byte == b':').next().unwrap_or(name) +} + +fn chunk_root_is_office_math(chunk: &[u8]) -> bool { + let local = chunk_root_local(chunk); + local == b"oMath" || local == b"oMathPara" +} + +/// The verbatim bytes of a drawing's `wp:inline`/`wp:anchor` (rdocx preserves +/// them for round-trip). Empty when absent — a hand-constructed drawing, which +/// never comes out of the parser this walk consumes. +fn drawing_raw_xml(drawing: &CT_Drawing) -> &[u8] { + drawing + .inline + .as_ref() + .and_then(|inline| inline.raw_xml.as_deref()) + .or_else(|| { + drawing + .anchor + .as_ref() + .and_then(|anchor| anchor.raw_xml.as_deref()) + }) + .unwrap_or_default() +} + +/// §act-eleven A11.9: the `a:blip r:link` relationship id in a drawing's raw +/// `wp:inline`/`wp:anchor` bytes, for blips that carry NO `r:embed`. The typed +/// model (`CT_Inline`/`CT_Anchor`) only parses `r:embed`, so link-only blips +/// are recovered from the verbatim bytes rdocx preserves. A blip that carries +/// `r:embed` returns `None` (it is an embedded image, handled by the typed +/// field; `r:link` on such a blip is only Word's edit-time refresh hint). +fn blip_link_id_from_raw(raw: &[u8]) -> Option { + memchr::memmem::find(raw, b":link=")?; + let mut reader = XmlReader::from_reader(raw); + reader.config_mut().check_end_names = false; + let mut buf = Vec::new(); + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(node)) | Ok(Event::Empty(node)) => { + if local_name(node.name().as_ref()) == "blip" { + let mut link = None; + for attribute in node.attributes().flatten() { + match local_name(attribute.key.as_ref()) { + "embed" => return None, + "link" => link = Some(String::from_utf8_lossy(&attribute.value).into_owned()), + _ => {}, + } + } + return link.filter(|id| !id.is_empty()); + } + }, + Ok(Event::Eof) | Err(_) => return None, + _ => {}, + } + buf.clear(); + } +} + +// -- Fallback walk over the owned XmlNode tree (§A9.2 escape hatch) ---------- + +/// The pre-A9.2 walk, retained verbatim as the whole-document fallback for +/// markup the typed model cannot reproduce byte-for-byte. +#[hotpath::measure] +fn interpret_structured_via_tree(cleaned: &CleanedDocx, doc_xml: &[u8], paragraphs: &[DocumentParagraphInput]) -> io::Result { + let Some(root) = parse_tree(doc_xml) else { + return Ok(StructuredDocx::default()); + }; + let Some(body) = child(&root, "body") else { + return Ok(StructuredDocx::default()); + }; + + let mut walker = StructuredWalker { + doc_xml, + paragraphs, + cursor: 0, + blocks: Vec::new(), + images: ImageAssets::new(cleaned), + tables_imported: 0, + images_imported: 0, + equations_imported: 0, + }; + walker.walk_body(body); + + Ok(finish_structured( + walker.blocks, + walker.images.assets, + walker.tables_imported, + walker.images_imported, + walker.equations_imported, + )) +} + +struct StructuredWalker<'ctx> { + doc_xml: &'ctx [u8], + paragraphs: &'ctx [DocumentParagraphInput], + cursor: usize, + blocks: Vec, + images: ImageAssets<'ctx>, + tables_imported: usize, + images_imported: usize, + equations_imported: usize, +} + +impl StructuredWalker<'_> { + fn walk_body(&mut self, body: &XmlNode) { + for node in &body.children { + match node.local.as_ref() { + "p" => self.walk_paragraph(node), + "tbl" => { + let table = self.parse_table(node); + self.blocks.push(InputBlock::Table(table)); + }, + _ => {}, + } + } + } + + fn walk_paragraph(&mut self, paragraph_node: &XmlNode) { + // Top-level paragraphs are 1:1 with the pre-computed slice; the cursor only + // advances on `w:p`, so tables, section properties and SDTs never desync it. + let paragraph = if self.cursor < self.paragraphs.len() { + let input = input_paragraph_from_document(&self.paragraphs[self.cursor]); + self.cursor += 1; + input + } else { + cell_paragraph(paragraph_node) + }; + + // Resolve the paragraph's inline objects (images AND equations) first so we + // can tell whether this is a bare OBJECT wrapper. + let mut drawings = Vec::new(); + collect_descendants(paragraph_node, "drawing", &mut drawings); + let mut images: Vec<_> = drawings + .into_iter() + .filter_map(|drawing| self.image_from_drawing(drawing)) + .collect(); + // §act-ten A9.6: legacy VML picts (same collection as the typed walk, off + // the pict subtree's raw byte span so both paths import VML identically). + let mut picts = Vec::new(); + collect_descendants(paragraph_node, "pict", &mut picts); + for pict in picts { + if let Some(span) = self.doc_xml.get(pict.start..pict.end) { + images.extend(self.images.vml_images_from_chunk(span)); + } + } + let equations: Vec<_> = self + .doc_xml + .get(paragraph_node.start..paragraph_node.end) + .filter(|bytes| omml::contains_office_math(bytes)) + .map(omml::equations_from_container_bytes) + .unwrap_or_default(); + + // §perf-heaven T7.18/T7.21: an image OR an equation is an INLINE object (a + // U+FFFC anchored in the surrounding paragraph's flow), NOT a block-level + // paragraph — both go through `push_object` in `loro_import`. But the exporter + // writes each as its OWN (image = a drawing run; equation = an + // `m:oMath` paragraph), so a naive reimport pushes an EMPTY Paragraph whose + // `\n` boundary is a spurious extra separator around the object. When a + // carries object(s) and no paragraph text, collapse it: emit only the inline + // object(s), no empty Paragraph. A paragraph with BOTH text and an object + // keeps its Paragraph block (the object stays inline within it). + let paragraph_is_text_empty = paragraph.runs.iter().all(|run| run.text.trim().is_empty()); + let has_objects = !images.is_empty() || !equations.is_empty(); + // §perf-heaven T8.10: collapse a bare object wrapper even at the START of the + // document. The import plan always begins with a sentinel `\n` (a Normal first + // paragraph, `FlowTextImportPlan::new`), so a leading object anchors inline in + // that sentinel paragraph — no wrapper needed, and no spurious leading + // boundary. (The earlier `emitted_paragraph` guard was overly cautious: it + // predated confirming the sentinel provides the initial paragraph/style.) + let collapse_bare_object_wrapper = has_objects && paragraph_is_text_empty; + if !collapse_bare_object_wrapper { + self.blocks.push(InputBlock::Paragraph(paragraph)); + } + + for image in images { + self.blocks.push(InputBlock::Image(image)); + self.images_imported += 1; + } + + for equation in equations { + self.blocks.push(InputBlock::Equation(equation)); + self.equations_imported += 1; + } + } + + fn parse_table(&mut self, node: &XmlNode) -> InputTableBlock { + self.tables_imported += 1; + let column_widths = table_column_widths(node); + let mut rows: Vec = Vec::new(); + let mut header_row = false; + let mut first_row = true; + // Widest row in grid columns, so the durable columns cover every cell's + // grid position even when a row spans more columns than `tblGrid` declares. + let mut grid_width = 0_usize; + // grid column -> (row index, cell index) of the cell that started a vertical + // merge, so continuation cells fold into its `row_span` instead of emitting. + let mut vertical_open: FxHashMap = FxHashMap::default(); + + for row_node in &node.children { + if row_node.local.as_ref() != "tr" { + continue; + } + if first_row { + first_row = false; + header_row = row_is_header(row_node); + } + // Deterministic row id, seeded per-table so row ids are GLOBALLY unique + // across every table in the document. Cell identity is derived from + // `(row_id, column_id)`, and cell text flows live in one global registry, + // so two tables that reused a `(row, column)` coordinate would collide + // their flows and lose text. A distinct per-table high-bit seed keeps + // every cell coordinate unique while columns stay 1-based (below). + let row_id = RowId(((self.tables_imported as u128) << 40) | (rows.len() as u128 + 1)); + let mut cells: Vec = Vec::new(); + let mut grid_col = 0_usize; + for cell_node in &row_node.children { + if cell_node.local.as_ref() == "tc" { + grid_col = self.add_table_cell(cell_node, row_id, grid_col, &mut rows, &mut cells, &mut vertical_open); + } + } + grid_width = grid_width.max(grid_col); + rows.push(InputTableRow { id: row_id, cells }); + } + + let columns = build_table_columns(&column_widths, grid_width); + fill_full_grid(&mut rows, &columns); + InputTableBlock { + rows, + columns, + style: InputTableStyle { header_row }, + } + } + + /// Appends one `w:tc` to the current row (or folds a vertical-merge + /// continuation into the originating cell above) and returns the next grid + /// column. `rows` holds the already-completed rows; `cells` is the row in + /// progress. + fn add_table_cell( + &mut self, + cell_node: &XmlNode, + row_id: RowId, + grid_col: usize, + rows: &mut [InputTableRow], + cells: &mut Vec, + vertical_open: &mut FxHashMap, + ) -> usize { + let col_span = cell_grid_span(cell_node); + // The cell's durable column is the sequential id of its starting grid column. + let column_id = ColumnId(grid_col as u128 + 1); + match cell_vertical_merge(cell_node) { + VerticalMerge::Continue => { + if let Some(&(row_ix, cell_ix)) = vertical_open.get(&grid_col) + && let Some(row) = rows.get_mut(row_ix) + && let Some(cell) = row.cells.get_mut(cell_ix) + { + cell.row_span = cell.row_span.saturating_add(1); + } + }, + VerticalMerge::Restart => { + vertical_open.insert(grid_col, (rows.len(), cells.len())); + let blocks = self.cell_blocks(cell_node); + cells.push(InputTableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks, + row_span: 1, + col_span, + }); + }, + VerticalMerge::None => { + vertical_open.remove(&grid_col); + let blocks = self.cell_blocks(cell_node); + cells.push(InputTableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, + blocks, + row_span: 1, + col_span, + }); + }, + } + grid_col + usize::from(col_span.max(1)) + } + + fn cell_blocks(&mut self, cell_node: &XmlNode) -> Vec { + let mut blocks = Vec::new(); + for node in &cell_node.children { + match node.local.as_ref() { + "p" => blocks.push(InputTableCellBlock::Paragraph(cell_paragraph(node))), + "tbl" => blocks.push(InputTableCellBlock::Table(self.parse_table(node))), + _ => {}, + } + } + if blocks.is_empty() { + blocks.push(InputTableCellBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + })); + } + blocks + } + + fn image_from_drawing(&mut self, drawing: &XmlNode) -> Option { + let alt_text = find_descendant(drawing, "docPr") + .and_then(|doc_pr| doc_pr.attr("descr").or_else(|| doc_pr.attr("name"))) + .unwrap_or_default() + .to_owned(); + if let Some(relationship_id) = find_descendant_with_attr(drawing, "blip", "embed").and_then(|blip| blip.attr("embed")) { + let asset_id = self.images.resolve_image(relationship_id)?; + return Some(InputImageBlock { + asset_id, + alt_text, + caption: None, + sizing: drawing_sizing(drawing), + alignment: InputBlockAlignment::Left, + external_url: None, + }); + } + // §act-eleven A11.9: a link-only blip (`a:blip r:link`, no `r:embed`) + // resolves through the external-mode relationship to a LINKED image. + let link_id = find_descendant_with_attr(drawing, "blip", "link")?.attr("link")?; + let (asset_id, url) = self.images.resolve_external_image(link_id)?; + Some(InputImageBlock { + asset_id, + alt_text, + caption: None, + sizing: drawing_sizing(drawing), + alignment: InputBlockAlignment::Left, + external_url: Some(url), + }) + } +} + +// -- Shared table assembly ---------------------------------------------------- + +/// Columns carry durable ids and widths; pad any grid position past the +/// declared widths with `Auto` so every cell's `column_id` resolves. +fn build_table_columns(column_widths: &[InputTableColumnWidth], grid_width: usize) -> Vec { + let column_count = column_widths.len().max(grid_width); + (0..column_count) + .map(|index| InputTableColumn { + id: ColumnId(index as u128 + 1), + width: column_widths + .get(index) + .cloned() + .unwrap_or(InputTableColumnWidth::Auto), + }) + .collect() +} + +/// §P2b/FS-010: the canonical table grid is a FULL rectangle — the projection +/// requires one cell record at EVERY (row, column) coordinate. The `w:tc` walk +/// emits a cell only where a `` starts, so vMerge continuations, +/// `gridSpan`-covered columns, and short rows leave holes that +/// `table_topology::normalize` would otherwise report as `MissingCell` defects +/// (and a runtime repair pass would then have to synthesize). Fill those holes +/// here with the SAME empty placeholder the projection already synthesizes +/// (`CellId::from_coordinate`, span 1/1, one empty Normal paragraph) — the merge +/// stays carried solely by the anchor cell's `row_span`/`col_span`, so the +/// read-back grid, render, and docx export are all identical, minus the defect. +fn fill_full_grid(rows: &mut [InputTableRow], columns: &[InputTableColumn]) { + for row in rows { + let row_id = row.id; + let mut present: FxHashMap = row + .cells + .drain(..) + .map(|cell| (cell.column_id.0, cell)) + .collect(); + row.cells = columns + .iter() + .map(|column| { + present + .remove(&column.id.0) + .unwrap_or_else(|| InputTableCell { + id: CellId::from_coordinate(row_id, column.id), + row_id, + column_id: column.id, + blocks: vec![InputTableCellBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + })], + row_span: 1, + col_span: 1, + }) + }) + .collect(); + } +} + +/// Vertical-merge state for a table cell (`w:tcPr/w:vMerge`). +enum VerticalMerge { + None, + Restart, + Continue, +} + +fn input_paragraph_from_document(paragraph: &DocumentParagraphInput) -> InputParagraph { + InputParagraph { + style: paragraph.style, + runs: paragraph + .runs + .iter() + .map(|run| InputRun { + text: run.text.clone(), + styles: run.styles, + }) + .collect(), + } +} + +/// Lightweight cell-paragraph recognition: table-cell paragraphs are not part of +/// the recognized slice, so they are imported as Normal-styled plain text. This +/// trades run-level semantics inside tables for a small, robust path. +fn cell_paragraph(paragraph_node: &XmlNode) -> InputParagraph { + let mut text = String::new(); + collect_run_text(paragraph_node, &mut text); + let runs = if text.is_empty() { + Vec::new() + } else { + vec![InputRun { + text, + styles: RunStyles::default(), + }] + }; + InputParagraph { + style: ParagraphStyle::Normal, + runs, + } +} + +fn collect_run_text(node: &XmlNode, out: &mut String) { + for child_node in &node.children { + match child_node.local.as_ref() { + // Strip the reserved U+FFFC object-placeholder sentinel from literal cell + // text (see `soften_rdocx_breaks`) so it never reads back as an orphan. + "t" => out.extend(child_node.text.chars().filter(|&ch| ch != '\u{FFFC}')), + "tab" => out.push('\t'), + // A docx / is an INTRA-paragraph line break (Word Shift+Enter), + // not a paragraph boundary. Emit the model's soft-break char (U+2028), never + // '\n' — '\n' is the paragraph-separator in the body flow, so pushing it here + // fabricates a bare boundary with no paragraph metadata/block/style record, + // which the full reprojection then reports as missing_paragraph_* defects and + // segments differently from the incremental projection (divergence). The + // exporter already round-trips SOFT_LINE_BREAK back to a TextWrapping . + // + // §perf-heaven T7.22: but only a LINE break (`w:type` absent or + // `textWrapping`) is intra-paragraph content. `w:type="page"`/`"column"` + // is pagination, which the model represents structurally (the + // `pageBreakBefore` paragraph property), NOT as an inline char — mapping it + // to U+2028 fabricated a spurious soft break in the text on import. The + // model has no inline page-break, so a mid-run page/column break is dropped + // (its surrounding text simply joins). This also disarms the exporter + // landmine: were a Page break run ever emitted, reimport now drops it rather + // than corrupting the text with a U+2028. + "br" | "cr" => match child_node.attr("type") { + Some("page") | Some("column") => {}, + _ => out.push(SOFT_LINE_BREAK), + }, + _ => collect_run_text(child_node, out), + } + } +} + +fn table_column_widths(node: &XmlNode) -> Vec { + let Some(grid) = child(node, "tblGrid") else { + return Vec::new(); + }; + grid + .children + .iter() + .filter(|candidate| candidate.local.as_ref() == "gridCol") + .map(|column| { + column + .attr("w") + .and_then(|width| width.parse::().ok()) + .map_or(InputTableColumnWidth::Auto, column_width_from_twips) + }) + .collect() +} + +/// Twips → fixed pixel column width; non-positive or overflowing widths fall +/// back to `Auto` (shared by the typed and `XmlNode` walks). +fn column_width_from_twips(twips: i64) -> InputTableColumnWidth { + Some(twips) + .filter(|twips| *twips > 0) + .and_then(|twips| u32::try_from(twips.saturating_mul(96) / 1440).ok()) + .map_or(InputTableColumnWidth::Auto, InputTableColumnWidth::FixedPx) +} + +fn cell_grid_span(cell_node: &XmlNode) -> u16 { + child(cell_node, "tcPr") + .and_then(|properties| child(properties, "gridSpan")) + .and_then(|grid_span| grid_span.attr("val")) + .and_then(|value| value.parse::().ok()) + .filter(|span| *span >= 1) + .unwrap_or(1) +} + +fn cell_vertical_merge(cell_node: &XmlNode) -> VerticalMerge { + let Some(properties) = child(cell_node, "tcPr") else { + return VerticalMerge::None; + }; + let Some(vertical_merge) = child(properties, "vMerge") else { + return VerticalMerge::None; + }; + // `` and `w:val="continue"` continue the merge above; only an + // explicit `restart` opens a new vertical span. + match vertical_merge.attr("val") { + Some("restart") => VerticalMerge::Restart, + _ => VerticalMerge::Continue, + } +} + +fn row_is_header(row_node: &XmlNode) -> bool { + child(row_node, "trPr") + .and_then(|properties| child(properties, "tblHeader")) + .is_some_and(|header| !matches!(header.attr("val"), Some("false" | "0" | "off"))) +} + +fn drawing_sizing(drawing: &XmlNode) -> InputImageSizing { + let Some(extent) = find_descendant(drawing, "extent") else { + return InputImageSizing::Intrinsic; + }; + let Some(width_px) = extent.attr("cx").and_then(emu_to_px) else { + return InputImageSizing::Intrinsic; + }; + InputImageSizing::Fixed { + width_px, + height_px: extent.attr("cy").and_then(emu_to_px), + } +} + +/// Typed-walk sizing from the parsed `wp:extent` EMUs; mirrors [`drawing_sizing`] +/// (a missing extent parses as 0 EMU, which maps to `Intrinsic`/`None` exactly +/// like a missing/non-positive attribute did). +fn sizing_from_extent_emu(extent_cx: i64, extent_cy: i64) -> InputImageSizing { + let Some(width_px) = emu_value_to_px(extent_cx) else { + return InputImageSizing::Intrinsic; + }; + InputImageSizing::Fixed { + width_px, + height_px: emu_value_to_px(extent_cy), + } +} + +fn emu_to_px(value: &str) -> Option { + emu_value_to_px(value.parse::().ok()?) +} + +fn emu_value_to_px(emu: i64) -> Option { + if emu <= 0 { + return None; + } + u32::try_from(emu.saturating_mul(96) / 914_400).ok() +} + +fn unescape_attribute_value(value: &str) -> String { + quick_xml::escape::unescape(value) + .map(Cow::into_owned) + .unwrap_or_else(|_| value.to_owned()) +} + +fn asset_id_from_bytes(bytes: &[u8]) -> AssetId { + AssetId(u128::from(AssetRecord::stable_content_hash(bytes))) +} + +fn mime_from_path(path: &str) -> &'static str { + let extension = path + .rsplit('.') + .next() + .unwrap_or_default() + .to_ascii_lowercase(); + match extension.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "bmp" => "image/bmp", + "tif" | "tiff" => "image/tiff", + "webp" => "image/webp", + "svg" => "image/svg+xml", + "emf" => "image/emf", + "wmf" => "image/wmf", + _ => "application/octet-stream", + } +} + +fn file_name(path: &str) -> String { + path.rsplit('/').next().unwrap_or(path).to_owned() +} + +// -- Lightweight DOM with byte spans (fallback path only) -------------------- + +/// A minimal element tree. `start..end` spans the element (from the opening `<` +/// of its start tag to just past its end tag) in the source XML, so equation +/// subtrees can be handed verbatim to [`omml`]. +struct XmlNode { + local: Cow<'static, str>, + attrs: Vec<(Cow<'static, str>, String)>, + text: String, + children: Vec, + start: usize, + end: usize, +} + +impl XmlNode { + fn attr(&self, local: &str) -> Option<&str> { + self + .attrs + .iter() + .find(|entry| entry.0.as_ref() == local) + .map(|entry| entry.1.as_str()) + } +} + +/// §perf-heaven T8.1: intern an OOXML element/attribute local-name to a `'static` +/// reference. The structured walker builds one [`XmlNode`] per element in the body; +/// every node's `local` (and every attribute key) came from `String::to_owned`, +/// one heap allocation apiece across the whole tree. OOXML local names are a fixed +/// schema vocabulary, so the high-frequency ones (every run's `r`/`t`/`rPr` and its +/// formatting children, every paragraph's `p`/`pPr`) map to a `'static` string with +/// no allocation. Unknown names (rare, non-schema) fall back to an owned copy — +/// correct, just not allocation-free. Netted by the corpus import-fidelity sweep +/// (the projection must stay byte-identical). +fn intern_local(name: &str) -> Cow<'static, str> { + // Ordered roughly by body frequency; the `match` compiles to length-bucketed + // comparisons, so the hot run/paragraph names resolve in a few byte checks. + let interned: Option<&'static str> = match name { + "r" => Some("r"), + "t" => Some("t"), + "rPr" => Some("rPr"), + "p" => Some("p"), + "pPr" => Some("pPr"), + "rFonts" => Some("rFonts"), + "sz" => Some("sz"), + "szCs" => Some("szCs"), + "color" => Some("color"), + "highlight" => Some("highlight"), + "b" => Some("b"), + "bCs" => Some("bCs"), + "i" => Some("i"), + "iCs" => Some("iCs"), + "u" => Some("u"), + "shd" => Some("shd"), + "jc" => Some("jc"), + "spacing" => Some("spacing"), + "ind" => Some("ind"), + "vertAlign" => Some("vertAlign"), + "lang" => Some("lang"), + "noProof" => Some("noProof"), + "pStyle" => Some("pStyle"), + "rStyle" => Some("rStyle"), + "br" => Some("br"), + "tab" => Some("tab"), + "bookmarkStart" => Some("bookmarkStart"), + "bookmarkEnd" => Some("bookmarkEnd"), + "hyperlink" => Some("hyperlink"), + "body" => Some("body"), + "tbl" => Some("tbl"), + "tr" => Some("tr"), + "tc" => Some("tc"), + "tcPr" => Some("tcPr"), + "trPr" => Some("trPr"), + "tblPr" => Some("tblPr"), + "tblGrid" => Some("tblGrid"), + "gridCol" => Some("gridCol"), + "gridSpan" => Some("gridSpan"), + "vMerge" => Some("vMerge"), + "tcW" => Some("tcW"), + "drawing" => Some("drawing"), + "oMath" => Some("oMath"), + "oMathPara" => Some("oMathPara"), + "blip" => Some("blip"), + "docPr" => Some("docPr"), + "inline" => Some("inline"), + "anchor" => Some("anchor"), + "extent" => Some("extent"), + "sdt" => Some("sdt"), + "sdtContent" => Some("sdtContent"), + "sdtPr" => Some("sdtPr"), + "sectPr" => Some("sectPr"), + "numPr" => Some("numPr"), + _ => None, + }; + match interned { + Some(known) => Cow::Borrowed(known), + None => Cow::Owned(name.to_owned()), + } +} + +fn parse_tree(xml: &[u8]) -> Option { + let mut reader = XmlReader::from_reader(xml); + reader.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut stack: Vec = Vec::new(); + let mut root: Option = None; + + loop { + // Captured before the read: for a tag event this is the offset of its `<`. + let start = reader.buffer_position() as usize; + match reader.read_event_into(&mut buf) { + Ok(Event::Start(event)) => { + let mut node = node_from_start(&event); + node.start = start; + stack.push(node); + }, + Ok(Event::Empty(event)) => { + let mut node = node_from_start(&event); + node.start = start; + node.end = reader.buffer_position() as usize; + push_node(&mut stack, &mut root, node); + }, + Ok(Event::Text(event)) => { + if let Some(top) = stack.last_mut() + && let Ok(text) = event.xml10_content() + { + top.text.push_str(&text); + } + }, + Ok(Event::End(_)) => { + if let Some(mut node) = stack.pop() { + node.end = reader.buffer_position() as usize; + push_node(&mut stack, &mut root, node); + } + }, + Ok(Event::Eof) => break, + Err(_) => return None, + _ => {}, + } + buf.clear(); + } + + root +} + +fn push_node(stack: &mut [XmlNode], root: &mut Option, node: XmlNode) { + if let Some(parent) = stack.last_mut() { + parent.children.push(node); + } else { + root.get_or_insert(node); + } +} + +fn node_from_start(event: &BytesStart<'_>) -> XmlNode { + let local = intern_local(local_name(event.name().as_ref())); + let mut attrs = Vec::new(); + for attribute in event.attributes().flatten() { + let key = intern_local(local_name(attribute.key.as_ref())); + // §perf-heaven T7.20: XML-unescape the attribute value (quick-xml does NOT + // auto-unescape attributes, unlike the `Event::Text` path). Without this, + // an image's `docPr@descr` alt-text containing ` ` (Word's multi-line + // descriptions) leaked the literal characters ` ` into body text via the + // `[alt]` image fallback. `escape::unescape` decodes the entities directly + // (no `Decoder` needed); fall back to the raw string only if unescape fails. + let raw = String::from_utf8_lossy(attribute.value.as_ref()); + let value = quick_xml::escape::unescape(&raw) + .map(|value| value.into_owned()) + .unwrap_or_else(|_| raw.into_owned()); + attrs.push((key, value)); + } + XmlNode { + local, + attrs, + text: String::new(), + children: Vec::new(), + start: 0, + end: 0, + } +} + +fn local_name(name: &[u8]) -> &str { + let name = std::str::from_utf8(name).unwrap_or_default(); + name.rsplit(':').next().unwrap_or(name) +} + +fn child<'tree>(node: &'tree XmlNode, local: &str) -> Option<&'tree XmlNode> { + node + .children + .iter() + .find(|candidate| candidate.local.as_ref() == local) +} + +fn find_descendant<'tree>(node: &'tree XmlNode, local: &str) -> Option<&'tree XmlNode> { + for candidate in &node.children { + if candidate.local.as_ref() == local { + return Some(candidate); + } + if let Some(found) = find_descendant(candidate, local) { + return Some(found); + } + } + None +} + +fn find_descendant_with_attr<'tree>(node: &'tree XmlNode, local: &str, attr: &str) -> Option<&'tree XmlNode> { + for candidate in &node.children { + if candidate.local.as_ref() == local && candidate.attr(attr).is_some() { + return Some(candidate); + } + if let Some(found) = find_descendant_with_attr(candidate, local, attr) { + return Some(found); + } + } + None +} + +fn collect_descendants<'tree>(node: &'tree XmlNode, local: &str, out: &mut Vec<&'tree XmlNode>) { + for candidate in &node.children { + if candidate.local.as_ref() == local { + out.push(candidate); + } else { + collect_descendants(candidate, local, out); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn doc_xml(body: &str) -> Vec { + format!( + r#"{body}"# + ) + .into_bytes() + } + + fn context(body: &str) -> (CleanedDocx, CT_Document) { + let xml = doc_xml(body); + let document = CT_Document::from_xml(&xml).expect("typed main-document parse"); + let cleaned = CleanedDocx { + bytes: Vec::new(), + main_document_xml: Some(std::sync::Arc::from(xml)), + package: None, + report: crate::cleaner::DocxCleanReport { + stats: crate::cleaner::DocxCleanStats::default(), + actions: crate::cleaner::CLEANING_RULES, + }, + }; + (cleaned, document) + } + + fn normal(text: &str) -> DocumentParagraphInput { + DocumentParagraphInput { + style: ParagraphStyle::Normal, + runs: vec![flowstate_document::DocumentRunInput { + text: text.to_owned(), + styles: RunStyles::default(), + }], + } + } + + /// §act-eleven C3: typed-vs-fallback equivalence over representative + /// constructs. The old `XmlNode` walk is the typed walk's correctness escape + /// hatch; this pins (a) both walks producing IDENTICAL output on the shapes + /// the typed walk claims, and (b) the parity pre-scan actually ROUTING these + /// shapes to the typed walk (so a probe regression that silently sends + /// everything to the fallback fails loudly here, not as a perf mystery). + #[test] + fn typed_walk_matches_fallback_walk_on_representative_bodies() { + let fixtures: &[(&str, &str, &[&str])] = &[ + ( + "plain paragraphs", + r"alphabeta", + &["alpha", "beta"], + ), + ( + "self-closed empty paragraph between text", + r"alphabeta", + &["alpha", "", "beta"], + ), + ( + "table with header, spans, merges, soft break + tab cell text", + concat!( + r#"intro"#, + r#""#, + r#"head"#, + r#"abc"#, + r#"right"#, + r#"tail"# + ), + &["intro"], + ), + ( + "bare equation wrapper collapses in both walks", + r"beforex+1after", + &["before", "", "after"], + ), + ( + "nested table in a cell", + concat!( + r#""#, + r#"outer"#, + r#"inner"#, + r#""# + ), + &[], + ), + ]; + for (label, body, slice_texts) in fixtures { + let (cleaned, document) = context(body); + let paragraphs: Vec = slice_texts.iter().map(|text| normal(text)).collect(); + let xml = cleaned.main_document_xml.clone().expect("fixture xml"); + assert!( + !typed_walk_diverges(&document, &xml, paragraphs.len()), + "{label}: the parity pre-scan must route this shape to the TYPED walk (coverage regression)" + ); + let typed = interpret_structured(&cleaned, &document, ¶graphs).expect("typed walk"); + let fallback = interpret_structured_via_tree(&cleaned, &xml, ¶graphs).expect("fallback walk"); + assert_eq!(typed.blocks, fallback.blocks, "{label}: typed and fallback walks diverged on blocks"); + assert_eq!(typed.tables_imported, fallback.tables_imported, "{label}: table count diverged"); + assert_eq!(typed.images_imported, fallback.images_imported, "{label}: image count diverged"); + assert_eq!(typed.equations_imported, fallback.equations_imported, "{label}: equation count diverged"); + } + } + + #[test] + fn table_with_paragraph_cell_is_emitted_in_body_order() { + let body = r#"introleftright"#; + let (cleaned, document) = context(body); + let paragraphs = [normal("intro")]; + + let structured = interpret_structured(&cleaned, &document, ¶graphs).expect("structured import"); + + assert_eq!(structured.tables_imported, 1); + assert!(matches!(structured.blocks.first(), Some(InputBlock::Paragraph(_)))); + let Some(InputBlock::Table(table)) = structured.blocks.get(1) else { + panic!("expected a table block after the intro paragraph"); + }; + assert!(table.style.header_row); + assert_eq!( + table.columns, + vec![ + InputTableColumn { + id: ColumnId(1), + width: InputTableColumnWidth::FixedPx(96), + }, + InputTableColumn { + id: ColumnId(2), + width: InputTableColumnWidth::FixedPx(192), + }, + ] + ); + assert_eq!(table.rows.len(), 1); + assert_eq!(table.rows[0].cells.len(), 2); + let InputTableCellBlock::Paragraph(left) = &table.rows[0].cells[0].blocks[0] else { + panic!("expected a paragraph in the first cell"); + }; + assert_eq!(left.runs[0].text, "left"); + } + + #[test] + fn vertical_merge_carries_span_and_fills_continuation_cell() { + let body = r#"top"#; + let (cleaned, document) = context(body); + + let structured = interpret_structured(&cleaned, &document, &[]).expect("structured import"); + + let Some(InputBlock::Table(table)) = structured.blocks.first() else { + panic!("expected a table block"); + }; + assert_eq!(table.rows.len(), 2); + // The merge is carried solely by the anchor cell's row_span... + assert_eq!(table.rows[0].cells.len(), 1); + assert_eq!(table.rows[0].cells[0].row_span, 2); + // ...and the continuation coordinate now gets a FULL-GRID placeholder cell + // (empty, span 1/1) so the projection reads a complete rectangle with no + // MissingCell defect (§P2b/FS-010). Merge semantics are unchanged. + assert_eq!(table.rows[1].cells.len(), 1); + let continuation = &table.rows[1].cells[0]; + assert_eq!(continuation.row_span, 1); + assert_eq!(continuation.col_span, 1); + assert_eq!(continuation.column_id, ColumnId(1)); + let InputTableCellBlock::Paragraph(placeholder) = &continuation.blocks[0] else { + panic!("continuation placeholder should be an empty paragraph"); + }; + assert!(placeholder.runs.is_empty(), "continuation placeholder is empty"); + } + + #[test] + fn inline_office_math_emits_equation_after_paragraph() { + let body = r"see12"; + let (cleaned, document) = context(body); + let paragraphs = [normal("see")]; + + let structured = interpret_structured(&cleaned, &document, ¶graphs).expect("structured import"); + + assert_eq!(structured.equations_imported, 1); + assert!(matches!(structured.blocks.first(), Some(InputBlock::Paragraph(_)))); + let Some(InputBlock::Equation(equation)) = structured.blocks.get(1) else { + panic!("expected an equation block after the paragraph"); + }; + assert_eq!(equation.source, "\\frac{1}{2}"); + } + + #[test] + fn body_without_objects_reports_zero_counts() { + let (cleaned, document) = context(r"plain"); + let structured = interpret_structured(&cleaned, &document, &[normal("plain")]).expect("structured import"); + + assert_eq!(structured.tables_imported, 0); + assert_eq!(structured.images_imported, 0); + assert_eq!(structured.equations_imported, 0); + assert!(structured.assets.is_empty()); + assert_eq!(structured.blocks.len(), 1); + } +} diff --git a/crates/flowstate-docx/src/lib.rs b/crates/flowstate-docx/src/lib.rs index 1b19e348..f265a000 100644 --- a/crates/flowstate-docx/src/lib.rs +++ b/crates/flowstate-docx/src/lib.rs @@ -1,5 +1,8 @@ mod cleaner; mod exporter; +mod fragment_parse; +#[cfg(test)] +mod import_roundtrip_tests; mod interpreter; mod pdf; mod pdf_recovery; @@ -7,18 +10,22 @@ mod pdf_recovery; use std::{io, path::Path}; pub use cleaner::{CleanAction, CleanedDocx, DocxCleanReport, DocxCleanStats, clean_docx_bytes}; -pub use exporter::{convert_db8_to_docx, write_docx}; +pub use exporter::{ExportWarning, convert_db8_to_docx, write_docx, write_docx_with_report}; pub use interpreter::{ DocxConversionReport, RecognitionRule, convert_cleaned_docx_to_document, convert_docx_bytes_to_document, convert_docx_to_document, + import_cleaned_docx_to_loro, import_docx_bytes_to_loro, import_docx_to_loro, }; pub use pdf::{convert_db8_to_pdf, convert_docx_to_pdf, write_pdf, write_pdf_with_db8_bytes}; pub use pdf_recovery::{FlowstatePdfPayloadInfo, convert_pdf_to_db8, embed_db8_bytes_in_pdf, embed_db8_file_in_pdf, extract_db8_bytes_from_pdf}; -use flowstate_document::write_db8; - #[hotpath::measure] pub fn convert_docx_to_db8(input: impl AsRef, output: impl AsRef) -> io::Result { - let (document, report) = convert_docx_to_document(input)?; - write_db8(output, &document)?; + let (imported, report) = import_docx_to_loro(input, "Imported DOCX")?; + flowstate_document::DocumentPackage::from_loro_snapshot_with_assets( + &imported.doc, + "Imported DOCX", + flowstate_document::loro_import::assets_from_document(&imported.projection), + )? + .write(output)?; Ok(report) } diff --git a/crates/flowstate-docx/src/pdf.rs b/crates/flowstate-docx/src/pdf.rs index a7a3f06b..233b9db7 100644 --- a/crates/flowstate-docx/src/pdf.rs +++ b/crates/flowstate-docx/src/pdf.rs @@ -1,13 +1,16 @@ use std::{ fs, io, path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, time::{SystemTime, UNIX_EPOCH}, }; -use flowstate_document::{Document, db8_bytes, read_db8}; +use flowstate_document::{DocumentProjection, read_db8}; use crate::{embed_db8_bytes_in_pdf, write_docx}; +static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); + #[hotpath::measure] pub fn convert_docx_to_pdf(input: impl AsRef, output: impl AsRef) -> io::Result<()> { let input = input.as_ref(); @@ -19,17 +22,37 @@ pub fn convert_docx_to_pdf(input: impl AsRef, output: impl AsRef) -> fs::create_dir_all(parent)?; } - docxide_pdf::convert_docx_to_pdf(input, output).map_err(|error| io::Error::other(format!("failed to convert DOCX to PDF: {error}"))) + let temp_output = temp_sibling_path(output, "pdf"); + let result = (|| { + docxide_pdf::convert_docx_to_pdf(input, &temp_output) + .map_err(|error| io::Error::other(format!("failed to convert DOCX to PDF: {error}")))?; + fs::rename(&temp_output, output)?; + Ok(()) + })(); + let _ = fs::remove_file(&temp_output); + result } #[hotpath::measure] -pub fn write_pdf(path: impl AsRef, document: &Document) -> io::Result<()> { - let db8 = db8_bytes(document)?; - write_pdf_with_db8_bytes(path, document, &db8) +pub fn write_pdf(path: impl AsRef, document: &DocumentProjection) -> io::Result<()> { + let path = path.as_ref(); + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent)?; + } + let temp_docx = temp_sibling_path(path, "docx"); + let result = (|| { + write_docx(&temp_docx, document)?; + convert_docx_to_pdf(&temp_docx, path) + })(); + let _ = fs::remove_file(&temp_docx); + result } #[hotpath::measure] -pub fn write_pdf_with_db8_bytes(path: impl AsRef, document: &Document, db8_bytes: &[u8]) -> io::Result<()> { +pub fn write_pdf_with_db8_bytes(path: impl AsRef, document: &DocumentProjection, db8_bytes: &[u8]) -> io::Result<()> { let path = path.as_ref(); if let Some(parent) = path .parent() @@ -71,7 +94,21 @@ fn temp_sibling_path(target: &Path, extension: &str) -> PathBuf { .unwrap_or("flowstate-export"); let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or_default(); - dir.join(format!(".{stem}.{nanos}.tmp.{extension}")) + .map_or(0, |duration| duration.as_nanos()); + let pid = std::process::id(); + let sequence = TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed); + dir.join(format!(".{stem}.{pid}.{nanos}.{sequence}.tmp.{extension}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn temp_sibling_path_uses_unique_names() { + let path = Path::new("exports/report.pdf"); + let a = temp_sibling_path(path, "pdf"); + let b = temp_sibling_path(path, "pdf"); + assert_ne!(a, b); + } } diff --git a/crates/flowstate-docx/src/pdf_recovery.rs b/crates/flowstate-docx/src/pdf_recovery.rs index 8c45d893..ac165827 100644 --- a/crates/flowstate-docx/src/pdf_recovery.rs +++ b/crates/flowstate-docx/src/pdf_recovery.rs @@ -1,13 +1,24 @@ -use std::{fs, io, path::Path}; +use std::{ + fs, + io::{self, Cursor, Read}, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; +use flowstate_document::DocumentPackage; use lopdf::{Dictionary, Document as PdfDocument, Object, ObjectId, Stream, dictionary}; -const PAYLOAD_NAME: &str = "flowstate-source.db8.zst"; -const PAYLOAD_DESCRIPTION: &str = "Flowstate DB8 source document"; -const PAYLOAD_MIME_TYPE: &str = "application/x-flowstate-db8+zstd"; -const PAYLOAD_MAGIC: &[u8; 8] = b"FSDB8ZST"; +static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); + +const PAYLOAD_NAME: &str = "flowstate-loro-source.db8.zst"; +const PAYLOAD_DESCRIPTION: &str = "Flowstate Loro-native DB8 source package"; +const PAYLOAD_MIME_TYPE: &str = "application/x-flowstate-loro-db8+zstd"; +const PAYLOAD_MAGIC: &[u8; 8] = b"FSL8ZST\0"; const PAYLOAD_VERSION: u32 = 1; const ZSTD_LEVEL: i32 = 3; +const MAX_PAYLOAD_COMPRESSED_LEN: usize = 64 * 1024 * 1024; +const MAX_PAYLOAD_UNCOMPRESSED_LEN: usize = 256 * 1024 * 1024; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FlowstatePdfPayloadInfo { @@ -39,16 +50,22 @@ pub fn embed_db8_bytes_in_pdf( fs::create_dir_all(parent)?; } + let temp_output_pdf = temp_sibling_path(output_pdf, "pdf"); let payload = encode_payload(db8_bytes)?; let info = FlowstatePdfPayloadInfo { original_len: db8_bytes.len() as u64, compressed_len: payload.len() as u64, }; - let mut pdf = PdfDocument::load(input_pdf).map_err(pdf_error)?; - attach_payload(&mut pdf, payload, db8_bytes.len() as u64)?; - pdf.save(output_pdf)?; - Ok(info) + let result = (|| { + let mut pdf = PdfDocument::load(input_pdf).map_err(pdf_error)?; + attach_payload(&mut pdf, payload, db8_bytes.len() as u64)?; + pdf.save(&temp_output_pdf)?; + fs::rename(&temp_output_pdf, output_pdf)?; + Ok(info) + })(); + let _ = fs::remove_file(&temp_output_pdf); + result } #[hotpath::measure] @@ -65,17 +82,12 @@ pub fn convert_pdf_to_db8(input_pdf: impl AsRef, output_db8: impl AsRef io::Result> { + DocumentPackage::from_bytes(db8_bytes)?; let compressed = zstd::bulk::compress(db8_bytes, ZSTD_LEVEL).map_err(io::Error::other)?; let mut payload = Vec::with_capacity(PAYLOAD_MAGIC.len() + 4 + 8 + compressed.len()); payload.extend_from_slice(PAYLOAD_MAGIC); @@ -111,10 +123,32 @@ fn decode_payload(payload: &[u8]) -> io::Result> { .try_into() .expect("length slice has fixed length"), ); - let decompressed = zstd::bulk::decompress(&payload[header_len..], original_len as usize).map_err(io::Error::other)?; - if decompressed.len() as u64 != original_len { + let compressed = &payload[header_len..]; + if compressed.len() > MAX_PAYLOAD_COMPRESSED_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Flowstate PDF payload exceeds the compressed size limit", + )); + } + let original_len = usize::try_from(original_len) + .ok() + .filter(|len| *len <= MAX_PAYLOAD_UNCOMPRESSED_LEN) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Flowstate PDF payload exceeds the uncompressed size limit"))?; + + let mut decompressed = vec![0; original_len]; + let mut decoder = zstd::stream::read::Decoder::new(Cursor::new(compressed)).map_err(io::Error::other)?; + decoder.read_exact(&mut decompressed).map_err(|error| { + if error.kind() == io::ErrorKind::UnexpectedEof { + io::Error::new(io::ErrorKind::InvalidData, "Flowstate PDF payload length mismatch") + } else { + error + } + })?; + let mut extra = [0u8; 1]; + if decoder.read(&mut extra).map_err(io::Error::other)? != 0 { return Err(io::Error::new(io::ErrorKind::InvalidData, "Flowstate PDF payload length mismatch")); } + DocumentPackage::from_bytes(&decompressed)?; Ok(decompressed) } @@ -274,3 +308,57 @@ fn object_string_equals(object: &Object, expected: &[u8]) -> bool { fn pdf_error(error: lopdf::Error) -> io::Error { io::Error::other(error) } + +#[hotpath::measure] +fn temp_sibling_path(target: &Path, extension: &str) -> PathBuf { + let dir = target + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let stem = target + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or("flowstate-export"); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_nanos()); + let pid = std::process::id(); + let sequence = TEMP_FILE_COUNTER.fetch_add(1, Ordering::Relaxed); + dir.join(format!(".{stem}.{pid}.{nanos}.{sequence}.tmp.{extension}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_payload_rejects_length_mismatch() { + let doc = flowstate_document::loro_schema::new_loro_document("payload").expect("new document"); + let bytes = DocumentPackage::from_loro_snapshot(&doc, "payload") + .expect("package") + .to_bytes() + .expect("bytes"); + let mut payload = encode_payload(&bytes).expect("payload"); + let len_offset = PAYLOAD_MAGIC.len() + 4; + let original_len = u64::from_be_bytes( + payload[len_offset..len_offset + 8] + .try_into() + .expect("fixed length"), + ); + payload[len_offset..len_offset + 8].copy_from_slice(&(original_len - 1).to_be_bytes()); + + let error = decode_payload(&payload).expect_err("mismatch must be rejected"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } + + #[test] + fn decode_payload_rejects_oversized_uncompressed_length() { + let mut payload = Vec::new(); + payload.extend_from_slice(PAYLOAD_MAGIC); + payload.extend_from_slice(&PAYLOAD_VERSION.to_be_bytes()); + payload.extend_from_slice(&((MAX_PAYLOAD_UNCOMPRESSED_LEN as u64) + 1).to_be_bytes()); + + let error = decode_payload(&payload).expect_err("oversized payload must be rejected"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } +} diff --git a/crates/flowstate-fidelity/Cargo.toml b/crates/flowstate-fidelity/Cargo.toml new file mode 100644 index 00000000..6088bf2b --- /dev/null +++ b/crates/flowstate-fidelity/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "flowstate-fidelity" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +tracing = { workspace = true } diff --git a/crates/flowstate-fidelity/src/lib.rs b/crates/flowstate-fidelity/src/lib.rs new file mode 100644 index 00000000..e61ee20c --- /dev/null +++ b/crates/flowstate-fidelity/src/lib.rs @@ -0,0 +1,381 @@ +//! Zero-cost-when-off diagnostics for Flowstate CRDT/document fidelity. +//! +//! This crate is a thin, dependency-light (only `tracing`) instrumentation +//! layer. Every entry point is gated by a single relaxed atomic load, so when +//! the `FLOWSTATE_TRACE_FIDELITY` environment variable is unset the detail +//! closures are never evaluated and instrumentation is effectively free. +//! +//! Instrumentation is strictly additive: nothing here mutates caller state or +//! changes control flow. [`check`] returns its input condition unchanged so it +//! can wrap an existing boolean without altering behavior. + +use std::{ + cell::RefCell, + collections::VecDeque, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// Global on/off switch. A single relaxed load gates every event, so the cost +/// of disabled instrumentation is one atomic read and nothing else. +static ENABLED: AtomicBool = AtomicBool::new(false); + +/// Number of recent firehose entries retained per thread for post-mortem dumps. +const RING_CAPACITY: usize = 512; + +thread_local! { + /// Recent firehose entries for the current thread. On a violation the tail of + /// this buffer is dumped so the events leading up to the failure are visible. + static RING: RefCell> = RefCell::new(VecDeque::with_capacity(RING_CAPACITY)); + + /// Test/diagnostic sink: every fired [`violation`] (only ever when enabled) + /// appends its rendered line here so an integration or soak test can assert the + /// invariant set stayed silent for a whole run. Drained by [`take_violations`]. + /// Independent of the firehose [`RING`]; empty and untouched when diagnostics + /// are off (a disabled `violation` returns before recording anything). + static VIOLATIONS: RefCell> = const { RefCell::new(Vec::new()) }; +} + +/// Categories of fidelity events. `Display` renders a short lowercase tag used +/// both in ring-buffer lines and as the `class` field on `tracing` events; keep +/// these strings stable, downstream log filters and instrumentation match them. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum FidelityClass { + Caret, + Text, + Reconcile, + Projection, + Identity, + Structure, + Frontier, + Convergence, + Persistence, + Presence, + Undo, + ImportExport, + Asset, +} + +impl std::fmt::Display for FidelityClass { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let name = match self { + Self::Caret => "caret", + Self::Text => "text", + Self::Reconcile => "reconcile", + Self::Projection => "projection", + Self::Identity => "identity", + Self::Structure => "structure", + Self::Frontier => "frontier", + Self::Convergence => "convergence", + Self::Persistence => "persistence", + Self::Presence => "presence", + Self::Undo => "undo", + Self::ImportExport => "importexport", + Self::Asset => "asset", + }; + f.write_str(name) + } +} + +/// Enables diagnostics when `FLOWSTATE_TRACE_FIDELITY` holds any non-empty value +/// other than `0`. Call once at startup after logging is initialized. +pub fn init_from_env() { + let on = std::env::var("FLOWSTATE_TRACE_FIDELITY").is_ok_and(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != "0" + }); + set_enabled(on); +} + +/// Turns diagnostics on or off at runtime. +pub fn set_enabled(on: bool) { + ENABLED.store(on, Ordering::Relaxed); +} + +/// Returns whether diagnostics are currently enabled. A single relaxed atomic +/// load; safe to call on hot paths to guard nontrivial invariant computation. +#[inline] +#[must_use] +pub fn enabled() -> bool { + ENABLED.load(Ordering::Relaxed) +} + +/// Returns whether diagnostics may run expensive whole-document verification. +/// +/// `FLOWSTATE_TRACE_FIDELITY=1` should keep the normal firehose and cheap +/// invariants available without perturbing large-document hot paths. Set +/// `FLOWSTATE_TRACE_FIDELITY_HEAVY=1` when a debugging session specifically needs +/// full reprojection or replay checks. +#[must_use] +pub fn expensive_checks_enabled() -> bool { + enabled() + && std::env::var("FLOWSTATE_TRACE_FIDELITY_HEAVY").is_ok_and(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != "0" + }) +} + +/// Records a gated firehose event. The `detail` closure runs only when enabled, +/// so instrumentation is free when off. The rendered line is pushed to the +/// thread-local ring buffer and emitted at `debug` under target `fidelity`. +pub fn event(class: FidelityClass, kind: &str, detail: impl FnOnce() -> String) { + if !enabled() { + return; + } + let detail = detail(); + RING.with(|ring| { + let mut ring = ring.borrow_mut(); + if ring.len() >= RING_CAPACITY { + ring.pop_front(); + } + ring.push_back(format!("[{class}] {kind}: {detail}")); + }); + tracing::debug!(target: "fidelity", %class, kind, detail = %detail); +} + +/// Records a gated LOUD event: emits at `error` under target `fidelity.violation` +/// and dumps the recent ring buffer so the lead-up to the failure is visible. +pub fn violation(class: FidelityClass, kind: &str, detail: impl FnOnce() -> String) { + if !enabled() { + return; + } + let detail = detail(); + // Record for `take_violations` before the (potentially verbose) dump. This is + // strictly additive diagnostics: it never changes control flow and only runs + // on the already-cold violation path while enabled. + VIOLATIONS.with(|sink| { + sink + .borrow_mut() + .push(format!("[{class}] {kind}: {detail}")); + }); + tracing::error!( + target: "fidelity.violation", + %class, + kind, + detail = %detail, + "FIDELITY INVARIANT VIOLATED" + ); + dump_recent(64); +} + +/// Drains and returns the fidelity violations recorded on the **current thread** +/// since the last drain, each rendered as `[class] kind: detail`. +/// +/// Intended for tests: arm diagnostics with [`set_enabled`], drive a workload, +/// then assert this returns empty. Because the sink is thread-local, the workload +/// under test must run its invariant checks on the calling thread (the soak +/// harness drives a synchronous `CrdtRuntime` + editor entirely on the test +/// thread, so every `violation` lands here). Records only while [`enabled`]. +#[must_use] +pub fn take_violations() -> Vec { + VIOLATIONS.with(|sink| std::mem::take(&mut *sink.borrow_mut())) +} + +/// Drains and returns the thread-local firehose ring buffer (every event +/// recorded since the last drain, oldest first). Intended for tests that want +/// the full event trail leading up to a failure printed alongside the assertion. +#[must_use] +pub fn drain_ring() -> Vec { + RING.with(|ring| ring.borrow_mut().drain(..).collect()) +} + +/// Asserts an invariant without changing behavior: returns `condition` +/// unchanged, and only when enabled and the condition is false does it fire a +/// [`violation`]. The `detail` closure is not evaluated unless it fires. +pub fn check(condition: bool, class: FidelityClass, kind: &str, detail: impl FnOnce() -> String) -> bool { + if enabled() && !condition { + violation(class, kind, detail); + } + condition +} + +/// On-demand marker: stamps "the bug is happening now" into the diagnostics +/// stream and dumps the entire recent ring buffer as a self-contained context +/// blob, so a bug that trips no automatic invariant still hands its lead-up to a +/// parsing agent. Wire this to a UI action/keybinding. No-op when disabled. +pub fn marker(label: &str) { + if !enabled() { + return; + } + tracing::error!(target: "fidelity.marker", label, "FIDELITY MARKER"); + dump_recent(RING_CAPACITY); +} + +/// Emits the last `n` entries of the thread-local ring buffer at `error` under +/// target `fidelity.violation`. No-op contribution when the buffer is empty. +pub fn dump_recent(n: usize) { + RING.with(|ring| { + let ring = ring.borrow(); + let skip = ring.len().saturating_sub(n); + for entry in ring.iter().skip(skip) { + tracing::error!(target: "fidelity.violation", "{entry}"); + } + }); +} + +#[cfg(test)] +mod tests { + use std::{cell::Cell, sync::Mutex}; + + use super::{FidelityClass, RING, RING_CAPACITY, check, enabled, event, set_enabled, take_violations}; + + // Tests toggle the global `ENABLED`, so serialize them to keep the flag + // deterministic under the parallel test harness. + static TEST_LOCK: Mutex<()> = Mutex::new(()); + + fn guard() -> std::sync::MutexGuard<'static, ()> { + TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn clear_ring() { + RING.with(|ring| ring.borrow_mut().clear()); + } + + fn ring_len() -> usize { + RING.with(|ring| ring.borrow().len()) + } + + fn ring_front() -> Option { + RING.with(|ring| ring.borrow().front().cloned()) + } + + #[test] + fn event_records_only_when_enabled() { + let _lock = guard(); + clear_ring(); + + set_enabled(false); + event(FidelityClass::Caret, "disabled", || "should not appear".to_string()); + assert_eq!(ring_len(), 0, "disabled events must not touch the ring"); + + set_enabled(true); + event(FidelityClass::Caret, "enabled", || "value".to_string()); + assert_eq!(ring_len(), 1, "enabled events must be recorded"); + assert_eq!(ring_front().as_deref(), Some("[caret] enabled: value")); + + set_enabled(false); + clear_ring(); + } + + #[test] + fn ring_buffer_evicts_at_capacity() { + let _lock = guard(); + clear_ring(); + set_enabled(true); + + for index in 0..(RING_CAPACITY + 10) { + event(FidelityClass::Text, "fill", || index.to_string()); + } + + assert_eq!(ring_len(), RING_CAPACITY, "ring must cap at capacity"); + // The first 10 entries (indices 0..10) must have been evicted; the oldest + // surviving entry is index 10. + assert_eq!(ring_front().as_deref(), Some("[text] fill: 10")); + + set_enabled(false); + clear_ring(); + } + + #[test] + fn check_returns_condition_and_fires_only_on_false_when_enabled() { + let _lock = guard(); + set_enabled(true); + + let evaluated = Cell::new(false); + let ok = check(true, FidelityClass::Reconcile, "ok", || { + evaluated.set(true); + "unused".to_string() + }); + assert!(ok, "check must return its condition unchanged"); + assert!(!evaluated.get(), "detail must not run when condition holds"); + + let evaluated = Cell::new(false); + let bad = check(false, FidelityClass::Reconcile, "bad", || { + evaluated.set(true); + "violated".to_string() + }); + assert!(!bad, "check must return its condition unchanged"); + assert!(evaluated.get(), "detail must run for a violation when enabled"); + + set_enabled(false); + clear_ring(); + } + + #[test] + fn check_does_not_evaluate_detail_when_disabled() { + let _lock = guard(); + set_enabled(false); + + let evaluated = Cell::new(false); + let result = check(false, FidelityClass::Convergence, "disabled", || { + evaluated.set(true); + "should not run".to_string() + }); + assert!(!result, "check must still return its condition when disabled"); + assert!(!evaluated.get(), "detail closure must not run while disabled"); + } + + #[test] + fn event_detail_not_evaluated_when_disabled() { + let _lock = guard(); + set_enabled(false); + + let evaluated = Cell::new(false); + event(FidelityClass::Persistence, "disabled", || { + evaluated.set(true); + "should not run".to_string() + }); + assert!(!evaluated.get(), "event detail must not run while disabled"); + assert!(!enabled()); + } + + #[test] + fn take_violations_collects_fired_violations_and_drains() { + let _lock = guard(); + set_enabled(true); + // Clear any residue from a prior test that shared this thread. + let _ = take_violations(); + + // A passing check records nothing. + check(true, FidelityClass::Caret, "ok", || "unused".to_string()); + assert!(take_violations().is_empty(), "a satisfied invariant must not record a violation"); + + // A failing check records exactly one rendered line. + check(false, FidelityClass::Caret, "reconcile-regressed", || "before=1 after=0".to_string()); + let fired = take_violations(); + assert_eq!(fired.len(), 1, "one failed invariant must record one violation"); + assert_eq!(fired[0], "[caret] reconcile-regressed: before=1 after=0"); + + // Draining is destructive: a second drain is empty. + assert!(take_violations().is_empty(), "take_violations must drain the sink"); + + set_enabled(false); + } + + #[test] + fn take_violations_records_nothing_while_disabled() { + let _lock = guard(); + set_enabled(false); + let _ = take_violations(); + check(false, FidelityClass::Caret, "reconcile-regressed", || "must not record".to_string()); + assert!(take_violations().is_empty(), "disabled violations must never be recorded"); + } + + #[test] + fn display_tags_are_stable_lowercase() { + assert_eq!(FidelityClass::Caret.to_string(), "caret"); + assert_eq!(FidelityClass::Text.to_string(), "text"); + assert_eq!(FidelityClass::Reconcile.to_string(), "reconcile"); + assert_eq!(FidelityClass::Projection.to_string(), "projection"); + assert_eq!(FidelityClass::Identity.to_string(), "identity"); + assert_eq!(FidelityClass::Structure.to_string(), "structure"); + assert_eq!(FidelityClass::Frontier.to_string(), "frontier"); + assert_eq!(FidelityClass::Convergence.to_string(), "convergence"); + assert_eq!(FidelityClass::Persistence.to_string(), "persistence"); + assert_eq!(FidelityClass::Presence.to_string(), "presence"); + assert_eq!(FidelityClass::Undo.to_string(), "undo"); + assert_eq!(FidelityClass::ImportExport.to_string(), "importexport"); + assert_eq!(FidelityClass::Asset.to_string(), "asset"); + } +} diff --git a/crates/flowstate-flow/Cargo.toml b/crates/flowstate-flow/Cargo.toml index 1333a38f..50935679 100644 --- a/crates/flowstate-flow/Cargo.toml +++ b/crates/flowstate-flow/Cargo.toml @@ -13,6 +13,7 @@ hotpath = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +tracing = { workspace = true } uuid = { workspace = true } [features] diff --git a/crates/flowstate-flow/src/actions.rs b/crates/flowstate-flow/src/actions.rs index c05add08..927dfb91 100644 --- a/crates/flowstate-flow/src/actions.rs +++ b/crates/flowstate-flow/src/actions.rs @@ -56,6 +56,9 @@ impl FlowDocument { pub fn apply_action(&mut self, action: Action) -> Action { match action { Action::Add { parent, id, index, value } => { + if self.nodes.contains_key(&id) { + return Action::Identity; + } let Some(parent_node) = self.nodes.get(&parent) else { return Action::Identity; }; @@ -74,6 +77,13 @@ impl FlowDocument { Action::Delete { id } }, Action::Delete { id } => { + let inverse_replace = self + .nodes + .get(&id) + .is_some_and(|node| !node.children.is_empty()) + .then(|| Action::Replace { + new_nodes: self.nodes.clone(), + }); let Some(node) = self.nodes.get(&id).cloned() else { return Action::Identity; }; @@ -87,13 +97,13 @@ impl FlowDocument { return Action::Identity; }; parent.children.remove(index); - self.nodes.remove(&id); - Action::Add { + remove_subtree(&mut self.nodes, &id); + inverse_replace.unwrap_or(Action::Add { parent: parent_id, id, index, value: node.value, - } + }) }, Action::Update { id, new_value } => { let Some(node) = self.nodes.get_mut(&id) else { @@ -145,6 +155,20 @@ impl FlowDocument { } } +/// Removes `id` and all of its descendants from `nodes`. +/// +/// Removing each node before recursing into its (former) children keeps the +/// walk cycle-safe: a malformed child edge pointing back at an already-removed +/// ancestor short-circuits on the `remove` miss instead of recursing forever. +#[hotpath::measure] +fn remove_subtree(nodes: &mut Nodes, id: &NodeId) { + if let Some(node) = nodes.remove(id) { + for child in &node.children { + remove_subtree(nodes, child); + } + } +} + #[hotpath::measure] pub fn new_box_action(parent: NodeId, parent_flow_id: NodeId, index: usize, placeholder: Option) -> Action { Action::Add { @@ -163,7 +187,6 @@ pub fn new_box_action(parent: NodeId, parent_flow_id: NodeId, index: usize, plac } } -#[hotpath::measure] pub const fn new_extension_action(parent: NodeId, parent_flow_id: NodeId, id: NodeId) -> Action { Action::Add { parent, @@ -181,7 +204,6 @@ pub const fn new_extension_action(parent: NodeId, parent_flow_id: NodeId, id: No } } -#[hotpath::measure] #[must_use] pub const fn new_update_action(id: NodeId, new_value: NodeValue) -> Action { Action::Update { id, new_value } diff --git a/crates/flowstate-flow/src/document.rs b/crates/flowstate-flow/src/document.rs index f8b3b31e..2d243587 100644 --- a/crates/flowstate-flow/src/document.rs +++ b/crates/flowstate-flow/src/document.rs @@ -269,7 +269,6 @@ pub fn constrain_index(index: usize, len: usize) -> usize { index.min(len) } -#[hotpath::measure] const fn is_false(value: &bool) -> bool { !*value } diff --git a/crates/flowstate-flow/src/history.rs b/crates/flowstate-flow/src/history.rs index b3c2bbb9..64cbf884 100644 --- a/crates/flowstate-flow/src/history.rs +++ b/crates/flowstate-flow/src/history.rs @@ -16,7 +16,6 @@ pub struct History { actions: Vec, } -#[hotpath::measure_all] impl History { pub fn add(&mut self, action_bundle: ActionBundle, before_focus: Option, after_focus: Option) { let keep = self.index.map_or(0, |index| index + 1); diff --git a/crates/flowstate-flow/src/persistence.rs b/crates/flowstate-flow/src/persistence.rs index 339e9093..5a5c7c5f 100644 --- a/crates/flowstate-flow/src/persistence.rs +++ b/crates/flowstate-flow/src/persistence.rs @@ -1,16 +1,18 @@ -use std::fs; -use std::path::Path; +use std::{fs, path::Path}; use anyhow::{Context as _, Result, bail}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use uuid::Uuid; use crate::actions::{Action, new_box_action}; use crate::document::{BoxNode, Flow, FlowDocument, NodeId, NodeValue, Nodes, ROOT_ID, new_box_id, new_flow_id}; pub const CURRENT_SAVE_VERSION: u32 = 1; -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +const MAX_FL0_BYTES: u64 = 64 * 1024 * 1024; + +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SaveableFlowDocument { pub nodes: Nodes, pub version: u32, @@ -45,6 +47,15 @@ pub fn load_nodes(data: Value) -> Result { #[hotpath::measure] pub fn load_flow_document(path: impl AsRef) -> Result { let path = path.as_ref(); + let metadata = fs::metadata(path).with_context(|| format!("failed to stat {}", path.display()))?; + if metadata.len() > MAX_FL0_BYTES { + bail!( + "refusing to read {}: file too large ({} bytes > {} bytes)", + path.display(), + metadata.len(), + MAX_FL0_BYTES + ); + } let text = fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; let value: Value = serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display()))?; let nodes = load_nodes(value)?; @@ -53,7 +64,28 @@ pub fn load_flow_document(path: impl AsRef) -> Result { #[hotpath::measure] pub fn load_flow_document_or_new(path: impl AsRef) -> FlowDocument { - load_flow_document(path).unwrap_or_else(|_| FlowDocument::new()) + let path = path.as_ref(); + match load_flow_document(path) { + Ok(document) => document, + Err(error) => { + if path.exists() { + let quarantine = path.with_file_name(format!( + "{}.corrupt.{}", + path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("flowstate.fl0"), + Uuid::new_v4() + )); + if let Err(rename_error) = fs::rename(path, &quarantine) { + tracing::warn!(path = %path.display(), quarantine = %quarantine.display(), error = %format_args!("{error:#}"), rename_error = %rename_error, "failed to quarantine existing .fl0 document; using a new empty document"); + } else { + tracing::warn!(path = %path.display(), quarantine = %quarantine.display(), error = %format_args!("{error:#}"), "failed to load existing .fl0 document; quarantined it and using a new empty document"); + } + } + FlowDocument::new() + }, + } } #[hotpath::measure] @@ -66,7 +98,17 @@ pub fn save_flow_document(path: impl AsRef, document: &FlowDocument) -> Re fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; } let text = get_json(document)?; - fs::write(path, text).with_context(|| format!("failed to write {}", path.display())) + let temp_path = path.with_file_name(format!( + ".{}.{}.tmp", + path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("flowstate"), + Uuid::new_v4() + )); + fs::write(&temp_path, text).with_context(|| format!("failed to write temporary {}", temp_path.display()))?; + fs::rename(&temp_path, path).with_context(|| format!("failed to atomically replace {} with {}", path.display(), temp_path.display()))?; + Ok(()) } #[derive(Clone, Debug, Deserialize)] @@ -97,54 +139,78 @@ struct OldBox { #[hotpath::measure] fn upgrade_0_1(saved: Value) -> Result { + #[derive(Debug)] + enum LegacyNode { + Flow(OldFlow), + Box(OldBox), + } + let old_flows: Vec = serde_json::from_value(saved).context("invalid legacy debate-flow document")?; let mut document = FlowDocument::new(); - for (index, old_flow) in old_flows.into_iter().enumerate() { - let flow_id = new_flow_id(); - let add_flow = Action::Add { - parent: ROOT_ID.to_owned(), - id: flow_id.clone(), - index, - value: NodeValue::Flow(Flow { - content: old_flow.content, - invert: old_flow.invert, - columns: old_flow.columns, - }), - }; - document.apply_action(add_flow); - for (box_index, old_box) in old_flow.children.into_iter().enumerate() { - upgrade_0_1_add_boxes_rec(&mut document, flow_id.clone(), flow_id.clone(), old_box, box_index); + let mut pending = Vec::new(); + for (index, old_flow) in old_flows.into_iter().enumerate().rev() { + pending.push((ROOT_ID.to_owned(), index, LegacyNode::Flow(old_flow))); + } + + let mut processed_nodes = 0usize; + const MAX_LEGACY_NODES: usize = 100_000; + while let Some((parent_id, index, node)) = pending.pop() { + processed_nodes += 1; + if processed_nodes > MAX_LEGACY_NODES { + bail!("legacy .fl0 document is too deeply nested or large to upgrade safely"); + } + + match node { + LegacyNode::Flow(old_flow) => { + let flow_id = new_flow_id(); + let add_flow = Action::Add { + parent: parent_id, + id: flow_id.clone(), + index, + value: NodeValue::Flow(Flow { + content: old_flow.content, + invert: old_flow.invert, + columns: old_flow.columns, + }), + }; + document.apply_action(add_flow); + for (box_index, child) in old_flow.children.into_iter().enumerate().rev() { + pending.push((flow_id.clone(), box_index, LegacyNode::Box(child))); + } + }, + LegacyNode::Box(old_box) => { + let id = new_box_id(); + let flow_id = document + .parent_flow_id(&parent_id) + .unwrap_or_else(|| parent_id.clone()); + let add = Action::Add { + parent: parent_id, + id: id.clone(), + index, + value: NodeValue::Box(BoxNode { + content: old_box.content, + flow_id: flow_id.clone(), + placeholder: old_box.placeholder, + empty: old_box.empty, + crossed: old_box.crossed, + bold: false, + is_extension: false, + }), + }; + document.apply_action(add); + for (child_index, child) in old_box.children.into_iter().enumerate().rev() { + pending.push((id.clone(), child_index, LegacyNode::Box(child))); + } + }, } } + Ok(SaveableFlowDocument { nodes: document.nodes, version: CURRENT_SAVE_VERSION, }) } -#[hotpath::measure] -fn upgrade_0_1_add_boxes_rec(document: &mut FlowDocument, flow_id: NodeId, parent_id: NodeId, old_box: OldBox, index: usize) { - let id = new_box_id(); - let add = Action::Add { - parent: parent_id, - id: id.clone(), - index, - value: NodeValue::Box(BoxNode { - content: old_box.content, - flow_id: flow_id.clone(), - placeholder: old_box.placeholder, - empty: old_box.empty, - crossed: old_box.crossed, - bold: false, - is_extension: false, - }), - }; - document.apply_action(add); - for (child_index, child) in old_box.children.into_iter().enumerate() { - upgrade_0_1_add_boxes_rec(document, flow_id.clone(), id.clone(), child, child_index); - } -} - #[allow(dead_code, reason = "Legacy save migration helper is retained for older persisted flow documents.")] #[hotpath::measure] fn _new_empty_box(parent_id: NodeId, flow_id: NodeId, index: usize) -> Action { @@ -154,6 +220,49 @@ fn _new_empty_box(parent_id: NodeId, flow_id: NodeId, index: usize) -> Action { #[cfg(test)] mod tests { use super::*; + use std::{path::PathBuf, time::SystemTime}; + + fn test_dir(name: &str) -> PathBuf { + let unique = format!( + "{name}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let path = std::env::temp_dir().join(unique); + fs::create_dir_all(&path).unwrap(); + path + } + + #[test] + fn rejects_oversized_documents_before_reading() { + let dir = test_dir("flowstate-fl0-size"); + let path = dir.join("oversized.fl0"); + fs::write(&path, vec![b'{'; (MAX_FL0_BYTES as usize) + 1]).unwrap(); + + let error = load_flow_document(&path).unwrap_err(); + assert!(error.to_string().contains("file too large")); + } + + #[test] + fn quarantines_bad_existing_document_on_fallback() { + let dir = test_dir("flowstate-fl0-corrupt"); + let path = dir.join("broken.fl0"); + fs::write(&path, b"{not json").unwrap(); + + let document = load_flow_document_or_new(&path); + assert!(document.flow_ids().is_empty()); + assert!(!path.exists()); + assert!(fs::read_dir(&dir).unwrap().any(|entry| { + entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("broken.fl0.corrupt.") + })); + } #[test] #[hotpath::measure] diff --git a/crates/flowstate-soak/Cargo.toml b/crates/flowstate-soak/Cargo.toml new file mode 100644 index 00000000..23a25a9d --- /dev/null +++ b/crates/flowstate-soak/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "flowstate-soak" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +flowstate-collab.workspace = true +flowstate-document.workspace = true +flowstate-docx.workspace = true +hotpath.workspace = true +loro.workspace = true +postcard.workspace = true + +[features] +hotpath = ["hotpath/hotpath", "flowstate-collab/hotpath", "flowstate-docx/hotpath", "flowstate-document/hotpath"] +hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu", "flowstate-collab/hotpath-cpu", "flowstate-docx/hotpath-cpu", "flowstate-document/hotpath-cpu"] +hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc", "flowstate-collab/hotpath-alloc", "flowstate-docx/hotpath-alloc", "flowstate-document/hotpath-alloc"] + +[lints] +workspace = true diff --git a/crates/flowstate-soak/src/bin/pkg_inspect.rs b/crates/flowstate-soak/src/bin/pkg_inspect.rs new file mode 100644 index 00000000..3e4c9ec3 --- /dev/null +++ b/crates/flowstate-soak/src/bin/pkg_inspect.rs @@ -0,0 +1,122 @@ +//! Scratch diagnostic: break down a .db8 package's size by chunk class. +use flowstate_document::DocumentPackage; + +fn main() -> anyhow::Result<()> { + let path = std::env::args() + .nth(1) + .expect("usage: pkg_inspect "); + let started = std::time::Instant::now(); + let preview = DocumentPackage::read_cached_projection(&path)?; + println!( + "phase read_cached_projection (preview fast path): {:?} ({})", + started.elapsed(), + preview.map_or("cache stale/absent".to_string(), |document| format!( + "{} paragraphs", + document.paragraphs.len() + )) + ); + let started = std::time::Instant::now(); + let package = DocumentPackage::read(&path)?; + println!("phase read: {:?}", started.elapsed()); + let started = std::time::Instant::now(); + let doc = package.load_loro_doc_from_validated()?; + println!("phase load_loro_doc_from_validated: {:?}", started.elapsed()); + let started = std::time::Instant::now(); + let projection = flowstate_document::document_from_loro(&doc)?; + println!( + "phase document_from_loro: {:?} ({} paragraphs)", + started.elapsed(), + projection.paragraphs.len() + ); + let snapshot_bytes: usize = package.loro_snapshots.iter().map(|c| c.bytes.len()).sum(); + let segment_bytes: usize = package + .loro_update_segments + .iter() + .map(|c| c.bytes.len()) + .sum(); + let asset_bytes: usize = package + .assets + .iter() + .map(|c| c.bytes.len() + c.metadata.len()) + .sum(); + let cache_bytes: usize = package + .projection_caches + .iter() + .map(|c| c.bytes.len()) + .sum(); + let search_bytes: usize = package + .search_units + .iter() + .map(|c| c.frontier.len() + c.unit_kind.len() + 64) + .sum(); + let thumb_bytes: usize = package.thumbnails.iter().map(|c| c.bytes.len()).sum(); + println!("snapshots: {} chunks, {} bytes", package.loro_snapshots.len(), snapshot_bytes); + println!("update segments: {} chunks, {} bytes", package.loro_update_segments.len(), segment_bytes); + println!("assets: {} chunks, {} bytes", package.assets.len(), asset_bytes); + println!("revisions: {} entries", package.revisions.len()); + println!("projection caches: {} chunks, {} bytes", package.projection_caches.len(), cache_bytes); + println!("search units: {} chunks, {} bytes", package.search_units.len(), search_bytes); + println!("thumbnails: {} chunks, {} bytes", package.thumbnails.len(), thumb_bytes); + for (ix, snapshot) in package.loro_snapshots.iter().enumerate().take(8) { + println!(" snapshot[{ix}]: id {} bytes {}", snapshot.snapshot_id, snapshot.bytes.len()); + } + println!("integrity index: {} entries", package.integrity_index.len()); + // Re-encode each top-level section to find where the file size actually lives. + let encoded = |label: &str, len: usize| println!(" encoded {label}: {len} bytes"); + encoded( + "manifest", + postcard::to_stdvec(&package.manifest) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "snapshots", + postcard::to_stdvec(&package.loro_snapshots) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "segments", + postcard::to_stdvec(&package.loro_update_segments) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "assets", + postcard::to_stdvec(&package.assets) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "revisions", + postcard::to_stdvec(&package.revisions) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "projection_caches", + postcard::to_stdvec(&package.projection_caches) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "search_units", + postcard::to_stdvec(&package.search_units) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "thumbnails", + postcard::to_stdvec(&package.thumbnails) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded( + "integrity_index", + postcard::to_stdvec(&package.integrity_index) + .map(|b| b.len()) + .unwrap_or(0), + ); + encoded("whole package", postcard::to_stdvec(&package).map(|b| b.len()).unwrap_or(0)); + Ok(()) +} diff --git a/crates/flowstate-soak/src/bin/pkg_restore.rs b/crates/flowstate-soak/src/bin/pkg_restore.rs new file mode 100644 index 00000000..4e5571a5 --- /dev/null +++ b/crates/flowstate-soak/src/bin/pkg_restore.rs @@ -0,0 +1,42 @@ +//! One-off remediation: restore a .db8 to its LATEST SNAPSHOT state by +//! dropping all update segments (used to strip accidental headless-soak +//! edits appended after a save; the pre-soak package had zero segments). +use flowstate_document::DocumentPackage; + +fn main() -> anyhow::Result<()> { + let input = std::env::args() + .nth(1) + .expect("usage: pkg_restore "); + let output = std::env::args() + .nth(2) + .expect("usage: pkg_restore "); + let mut package = DocumentPackage::read(&input)?; + let snapshot = package + .latest_snapshot() + .ok_or_else(|| anyhow::anyhow!("package has no snapshot"))? + .clone(); + let dropped = package.loro_update_segments.len(); + package.loro_update_segments.clear(); + package.manifest.latest_frontier = snapshot.frontier.clone(); + package.manifest.latest_version_vector = snapshot.version_vector.clone(); + package.manifest.projection_cache_frontier = None; + package.projection_caches.clear(); + package.manifest.search_cache_frontier = None; + package.search_units.clear(); + package.refresh_manifest_indexes_public(); + package.validate()?; + package.write(&output)?; + println!("restored to snapshot frontier; dropped {dropped} update segments"); + let restored = DocumentPackage::read(&output)?; + let doc = restored.load_loro_doc()?; + let projection = flowstate_document::document_from_loro(&doc)?; + println!( + "verified: {} paragraphs, {} body chars", + projection.paragraphs.len(), + flowstate_document::loro_schema::body_text(&doc) + .to_string() + .chars() + .count() + ); + Ok(()) +} diff --git a/crates/flowstate-soak/src/main.rs b/crates/flowstate-soak/src/main.rs new file mode 100644 index 00000000..491d79ab --- /dev/null +++ b/crates/flowstate-soak/src/main.rs @@ -0,0 +1,685 @@ +//! Headless collaboration hotpath soak (field diagnosis, 2026-07-07). +//! +//! Reproduces the impact-doc slowness classes without a window or network — a +//! thin bin over the collab stack, so soak cycles never pay the app binary's +//! GPUI link: +//! +//! 1. **Local typing** — intents through the real `LocalDocHandle` write path +//! plus the editor-side ordered-stream drain and patch apply (the exact +//! projection work `sync_projection_from_authority` performs, minus GPU +//! layout). +//! 2. **Paragraph splits, joins, cross-paragraph deletes** — the structural +//! local-edit shapes. +//! 3. **Remote imports** — a second peer runtime types text and splits +//! paragraphs; its update chunks import into the main runtime under the +//! gate (the `doc_io` `import-remote-update` shape from the field logs), +//! then drain into the simulated editor. +//! +//! Run with `--features hotpath-cpu` for the per-stage breakdown table that +//! prints on exit; the wall-clock distributions below print unconditionally. +//! +//! ```text +//! cargo run -p flowstate-soak --release --features hotpath-cpu -- [--audit off|N] +//! ``` + +use std::{ + num::NonZeroU32, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; + +use clap::Parser; +use flowstate_collab::{ + crdt_runtime::CrdtRuntime, + local_write::{ + DeleteRangeIntent, GateHolder, InsertTextIntent, JoinParagraphsIntent, LocalDocHandle, LocalWriteAuthority, LocalWriteConfig, + SplitParagraphIntent, TextAnchor, + }, +}; +use flowstate_document::{DocumentProjection, ParagraphId, ParagraphStyle, ProjectionStreamItem}; + +/// Headless collaboration hotpath soak over a real document. +#[derive(Parser)] +#[command(name = "flowstate-soak")] +struct Cli { + /// Input `.docx` or package document. + input: PathBuf, + /// Local typing keystrokes to measure. + #[arg(long, default_value_t = 160)] + keystrokes: usize, + /// Local paragraph splits to measure (joins/cross-deletes run at half this). + #[arg(long, default_value_t = 8)] + splits: usize, + /// Remote import chunks to measure (every 6th is structural). + #[arg(long, default_value_t = 24)] + imports: usize, + /// Release audit sampling: `off`, or audit every N-th intent + /// (debug builds always audit every commit regardless). + #[arg(long)] + audit: Option, + /// Run the pure-Loro select-all micro-matrix (marks × subscription × undo) + /// instead of the document soak. + #[arg(long, default_value_t = false)] + loro_micro: bool, + /// Cap the select-all-restyle phase at N paragraphs (default: all). + #[arg(long)] + restyle_cap: Option, +} + +/// Release-audit sampling requested on the CLI. +enum AuditSampling { + ProfileDefault, + Off, + Every(NonZeroU32), +} + +#[hotpath::main(functions_limit = 60)] +fn main() { + let cli = Cli::parse(); + if cli.loro_micro { + loro_micro(); + return; + } + let audit = cli + .audit + .as_deref() + .map_or(AuditSampling::ProfileDefault, |value| { + if value.eq_ignore_ascii_case("off") { + AuditSampling::Off + } else { + AuditSampling::Every( + value + .parse() + .expect("--audit takes `off` or a positive integer"), + ) + } + }); + run(&cli.input, cli.keystrokes, cli.splits, cli.imports, &audit, cli.restyle_cap).expect("collab hotpath soak failed"); +} + +fn run(path: &Path, keystrokes: usize, splits: usize, imports: usize, audit: &AuditSampling, restyle_cap: Option) -> anyhow::Result<()> { + let build_profile = if cfg!(debug_assertions) { "debug" } else { "release" }; + println!("collab-hotpath soak — build profile: {build_profile}"); + + // ---- Load the document exactly the way the app open path does ------------- + let load_started = Instant::now(); + let runtime = load_runtime(path)?; + println!("document load: {:?}", load_started.elapsed()); + + let mut config = LocalWriteConfig::default(); + match audit { + AuditSampling::ProfileDefault => {}, + AuditSampling::Off => config.release_audit_sample = None, + AuditSampling::Every(sample) => config.release_audit_sample = Some(*sample), + } + println!( + "release audit sampling: {:?} (ignored in debug builds — those audit every commit)", + config.release_audit_sample + ); + let (handle, gate) = LocalDocHandle::new(runtime, config); + + let projection = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let paragraph_count = projection.ids.paragraph_ids.len(); + let body_chars: usize = (0..projection.paragraphs.len()) + .map(|ix| { + flowstate_document::paragraph_text(&projection, ix) + .chars() + .count() + }) + .sum(); + println!( + "document shape: {paragraph_count} paragraphs, {body_chars} body chars, {} blocks", + projection.blocks.len() + ); + + // ---- Simulated editor: canonical attach + ordered-stream drains ------------ + let mut editor = LocalWriteAuthority::canonical_projection(&handle).map_err(|error| anyhow::anyhow!("{error}"))?; + drain_into_editor(&handle, &mut editor)?; + + let target = mid_paragraph(&editor); + + // ---- Phase 1: local typing -------------------------------------------------- + let mut commit_times = Vec::with_capacity(keystrokes); + let mut editor_apply_times = Vec::with_capacity(keystrokes); + for i in 0..keystrokes { + let started = Instant::now(); + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(target, usize::MAX), + text: ((b'a' + (i % 26) as u8) as char).to_string(), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("typing intent rejected: {error}"))?; + commit_times.push(started.elapsed()); + let apply_started = Instant::now(); + drain_into_editor(&handle, &mut editor)?; + editor_apply_times.push(apply_started.elapsed()); + } + summarize("local keystroke (intent commit, gate-held)", &mut commit_times); + summarize("local keystroke (editor drain + patch apply)", &mut editor_apply_times); + + // ---- Phase 2: local paragraph splits --------------------------------------- + let mut split_times = Vec::with_capacity(splits); + for _ in 0..splits { + let started = Instant::now(); + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(target, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .map_err(|error| anyhow::anyhow!("split intent rejected: {error}"))?; + split_times.push(started.elapsed()); + drain_into_editor(&handle, &mut editor)?; + } + summarize("local split (intent commit, gate-held)", &mut split_times); + + // ---- Phase 2b: joins (Backspace at paragraph start) and cross-paragraph + // deletes — the structural local edits from the field report. + let mut join_times = Vec::new(); + for _ in 0..splits / 2 { + let projection = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let Some(ix) = projection + .ids + .paragraph_ids + .iter() + .position(|id| *id == target) + else { + break; + }; + if ix + 1 >= projection.ids.paragraph_ids.len() { + break; + } + let second = projection.ids.paragraph_ids[ix + 1]; + let started = Instant::now(); + handle + .join_paragraphs(JoinParagraphsIntent { first: target, second }) + .map_err(|error| anyhow::anyhow!("join intent rejected: {error}"))?; + join_times.push(started.elapsed()); + drain_into_editor(&handle, &mut editor)?; + } + summarize("local join (intent commit, gate-held)", &mut join_times); + + let mut cross_delete_times = Vec::new(); + for _ in 0..splits / 2 { + // Create a fresh boundary, then delete across it. + handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(target, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .map_err(|error| anyhow::anyhow!("cross-delete setup split rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + let projection = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let Some(ix) = projection + .ids + .paragraph_ids + .iter() + .position(|id| *id == target) + else { + break; + }; + let Some(next) = projection.ids.paragraph_ids.get(ix + 1).copied() else { + break; + }; + let started = Instant::now(); + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new( + target, + flowstate_document::paragraph_text(&projection, ix) + .len() + .saturating_sub(2), + ), + end: TextAnchor::new(next, 0), + }) + .map_err(|error| anyhow::anyhow!("cross-paragraph delete rejected: {error}"))?; + cross_delete_times.push(started.elapsed()); + drain_into_editor(&handle, &mut editor)?; + } + summarize("local cross-paragraph delete (gate-held)", &mut cross_delete_times); + + // ---- Phase 3: remote imports ------------------------------------------------- + // A converged peer produced from the main doc's own snapshot, editing through + // its own full write path; its update chunks import here like session_io does. + let snapshot = { + let guard = gate + .lock(GateHolder::ExportUpdates) + .map_err(|_| anyhow::anyhow!("gate poisoned"))?; + guard + .doc() + .export(loro::ExportMode::Snapshot) + .map_err(|error| anyhow::anyhow!("snapshot export: {error}"))? + }; + let peer_doc = loro::LoroDoc::new(); + peer_doc + .import_with(&snapshot, "remote") + .map_err(|error| anyhow::anyhow!("peer join import: {error}"))?; + // Baseline BEFORE runtime startup: `from_doc` records replica metadata as + // new commits, and the peer's typing ops causally depend on them — update + // exports must include them or every main-side import reports pending + // missing dependencies (and falls back to a full rebuild). + let snapshot_vv = peer_doc.state_vv(); + let peer_runtime = CrdtRuntime::from_doc(peer_doc, None, None)?; + let (peer_handle, peer_gate) = LocalDocHandle::new(peer_runtime, LocalWriteConfig { release_audit_sample: None }); + let peer_projection = peer_handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let peer_target = mid_paragraph(&peer_projection); + let mut peer_vv = snapshot_vv; + + let mut text_import_times = Vec::new(); + let mut structural_import_times = Vec::new(); + let mut import_apply_times = Vec::new(); + for i in 0..imports { + // Realistic peer traffic: mostly short typing bursts, every 6th chunk is + // structural (an Enter) — the field logs' import mix. + let structural = i % 6 == 5; + if structural { + peer_handle + .split_paragraph(SplitParagraphIntent { + at: TextAnchor::new(peer_target, usize::MAX), + inherited_style: ParagraphStyle::Normal, + }) + .map_err(|error| anyhow::anyhow!("peer split rejected: {error}"))?; + } else { + for _ in 0..5 { + peer_handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(peer_target, usize::MAX), + text: "r".to_string(), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("peer typing rejected: {error}"))?; + } + } + let update = { + let guard = peer_gate + .lock(GateHolder::ExportUpdates) + .map_err(|_| anyhow::anyhow!("gate poisoned"))?; + let update = guard + .doc() + .export(loro::ExportMode::updates(&peer_vv)) + .map_err(|error| anyhow::anyhow!("peer update export: {error}"))?; + peer_vv = guard.doc().state_vv(); + update + }; + let started = Instant::now(); + let mut guard = gate + .lock(GateHolder::ImportChunk) + .map_err(|_| anyhow::anyhow!("gate poisoned"))?; + let events = guard.import_remote_update(&update)?; + drop(guard); + for event in &events { + if let flowstate_collab::crdt_runtime::RuntimeEvent::ProjectionUpdated { invalidation, .. } = event { + println!( + " import {i} took the REBUILD path (structural={structural}, reason={:?})", + invalidation.fallback_reason + ); + } + } + if structural { + structural_import_times.push(started.elapsed()); + } else { + text_import_times.push(started.elapsed()); + } + let apply_started = Instant::now(); + drain_into_editor(&handle, &mut editor)?; + import_apply_times.push(apply_started.elapsed()); + } + summarize("remote import (text chunk, gate-held)", &mut text_import_times); + summarize("remote import (structural chunk, gate-held)", &mut structural_import_times); + summarize("remote import (editor drain + apply)", &mut import_apply_times); + + // ---- Phase 3b: replace-all storm (find & replace). Every occurrence of a + // common trigram across the whole doc, ONE compound intent — the §11 + // anti-amplification law under the heaviest realistic match count. Then one + // undo (the storm is one undo member) to restore the text for phase 4. + { + let projection = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let mut matches = Vec::new(); + for (paragraph_ix, paragraph_id) in projection.ids.paragraph_ids.iter().enumerate() { + let text = flowstate_document::paragraph_text(&projection, paragraph_ix); + let mut from = 0usize; + while let Some(found) = text[from..].find("the") { + let byte = from + found; + matches.push(flowstate_collab::local_write::ReplaceMatch { + start: TextAnchor::new(*paragraph_id, byte), + end: TextAnchor::new(*paragraph_id, byte + 3), + styles: None, + }); + from = byte + 3; + } + } + let match_count = matches.len(); + if match_count > 0 { + let started = Instant::now(); + handle + .replace_matches(flowstate_collab::local_write::ReplaceMatchesIntent { + matches, + replacement: "thy".to_string(), + }) + .map_err(|error| anyhow::anyhow!("replace-all storm rejected: {error}"))?; + println!("replace-all storm ({match_count} matches, commit): {:?}", started.elapsed()); + let apply_started = Instant::now(); + drain_into_editor(&handle, &mut editor)?; + println!("replace-all storm (editor drain + apply): {:?}", apply_started.elapsed()); + let started = Instant::now(); + handle + .apply_undo() + .map_err(|error| anyhow::anyhow!("undo replace-all rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("undo replace-all storm (commit + drain): {:?}", started.elapsed()); + } else { + println!("replace-all storm: no matches found (skipped)"); + } + } + + // ---- Phase 3c: select-all restyle (the 64.7s hotpath1 freeze). The editor + // now sends ONE batched SetParagraphStyles intent for a selection-wide + // restyle (§11 anti-amplification; the per-paragraph loop it replaced cost + // one full write-path round trip per paragraph). Undo afterwards restores + // styles and measures single-member undo of the mass restyle. + { + let projection = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let restyle_count = projection + .ids + .paragraph_ids + .len() + .min(restyle_cap.unwrap_or(usize::MAX)); + if restyle_count == 0 { + println!("select-all restyle: skipped (--restyle-cap 0)"); + // fall through to phase 4 with styles untouched + } else { + let paragraphs: Vec<_> = projection + .ids + .paragraph_ids + .iter() + .copied() + .take(restyle_count) + .collect(); + let started = Instant::now(); + handle + .set_paragraph_styles(flowstate_collab::local_write::SetParagraphStylesIntent { + paragraphs, + style: ParagraphStyle::Custom(1), + }) + .map_err(|error| anyhow::anyhow!("batched restyle rejected: {error}"))?; + println!( + "select-all restyle ({restyle_count} paragraphs, ONE batched commit): {:?}", + started.elapsed() + ); + let apply_started = Instant::now(); + drain_into_editor(&handle, &mut editor)?; + println!("select-all restyle (editor drain + apply): {:?}", apply_started.elapsed()); + let started = Instant::now(); + handle + .apply_undo() + .map_err(|error| anyhow::anyhow!("undo restyle rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("undo select-all restyle (commit + drain): {:?}", started.elapsed()); + let started = Instant::now(); + handle + .apply_redo() + .map_err(|error| anyhow::anyhow!("redo restyle rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("redo select-all restyle (commit + drain): {:?}", started.elapsed()); + let started = Instant::now(); + handle + .apply_undo() + .map_err(|error| anyhow::anyhow!("re-undo restyle rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("re-undo select-all restyle (commit + drain): {:?}", started.elapsed()); + } + } + + // ---- Phase 4: select-all delete + retype (the ctrl-A field freeze). Runs + // LAST because it guts the document. ------------------------------------------ + let projection = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))?; + let first = projection.ids.paragraph_ids[0]; + let last = *projection.ids.paragraph_ids.last().expect("paragraphs"); + let started = Instant::now(); + handle + .delete_range(DeleteRangeIntent { + start: TextAnchor::new(first, 0), + end: TextAnchor::new(last, usize::MAX), + }) + .map_err(|error| anyhow::anyhow!("select-all delete rejected: {error}"))?; + println!("select-all delete (intent commit, gate-held): {:?}", started.elapsed()); + let apply_started = Instant::now(); + drain_into_editor(&handle, &mut editor)?; + println!("select-all delete (editor drain + apply): {:?}", apply_started.elapsed()); + let started = Instant::now(); + let remaining = handle + .projection() + .map_err(|error| anyhow::anyhow!("{error}"))? + .ids + .paragraph_ids[0]; + handle + .insert_text(InsertTextIntent { + at: TextAnchor::new(remaining, usize::MAX), + text: "replacement text after select-all".to_string(), + style_override: None, + }) + .map_err(|error| anyhow::anyhow!("post-select-all retype rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("select-all retype (commit + drain): {:?}", started.elapsed()); + + // ---- Phase 4b: undo/redo of the select-all (the ctrl-A + undo field + // freeze): undo retype, undo the mass delete (restores the whole doc), redo. + for label in ["undo retype", "undo select-all delete"] { + let started = Instant::now(); + handle + .apply_undo() + .map_err(|error| anyhow::anyhow!("{label} rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!( + "{label} (commit + drain):{:pad$}{:?}", + "", + started.elapsed(), + pad = 31_usize.saturating_sub(label.len()) + ); + } + let started = Instant::now(); + handle + .apply_redo() + .map_err(|error| anyhow::anyhow!("redo select-all delete rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("redo select-all delete (commit + drain): {:?}", started.elapsed()); + + // ---- Phase 4c: repeated mass undo/redo cycles. Registry hygiene proof: + // every whole-doc restore fabricates a fresh record generation, and the + // dead generations previously compounded the registry (field: scan P95 + // grew to seconds across a stress session). Non-increasing cycle times ⇒ + // the repair-pass prune is holding. + for cycle in 0..2 { + let started = Instant::now(); + handle + .apply_undo() + .map_err(|error| anyhow::anyhow!("cycle undo rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + handle + .apply_redo() + .map_err(|error| anyhow::anyhow!("cycle redo rejected: {error}"))?; + drain_into_editor(&handle, &mut editor)?; + println!("mass undo+redo cycle {cycle} (commit + drain): {:?}", started.elapsed()); + } + + // ---- Phase 4d: PEER receipt of the whole-doc delete + restore history — + // the 2026-07-07 field hang (peer froze importing the undo of a select-all + // delete: thousands of dead-cursor history traces). Must complete in + // seconds via the rebuild path, never trace per record. + let peer_guard = peer_gate + .lock(GateHolder::ExportUpdates) + .map_err(|_| anyhow::anyhow!("peer gate poisoned"))?; + let peer_known_vv = peer_guard.doc().state_vv(); + drop(peer_guard); + let main_guard = gate + .lock(GateHolder::ExportUpdates) + .map_err(|_| anyhow::anyhow!("gate poisoned"))?; + let mass_history_update = main_guard + .doc() + .export(loro::ExportMode::updates(&peer_known_vv)) + .map_err(|error| anyhow::anyhow!("mass-history export: {error}"))?; + drop(main_guard); + let mass_receipt_started = Instant::now(); + let mut peer_import_guard = peer_gate + .lock(GateHolder::ImportChunk) + .map_err(|_| anyhow::anyhow!("peer gate poisoned"))?; + peer_import_guard.import_remote_update(&mass_history_update)?; + drop(peer_import_guard); + println!( + "peer receipt of mass delete/restore history ({} KB): {:?}", + mass_history_update.len() / 1024, + mass_receipt_started.elapsed() + ); + + // ---- Convergence proof: the soak measured real work, not dropped work ------- + let canonical = LocalWriteAuthority::canonical_projection(&handle).map_err(|error| anyhow::anyhow!("{error}"))?; + anyhow::ensure!(editor.frontier == canonical.frontier, "simulated editor diverged from canonical frontier"); + println!( + "\nconverged: editor tracked all {} paragraphs at the canonical frontier", + canonical.paragraphs.len() + ); + Ok(()) +} + +/// Pure-Loro bisect for the select-all-delete commit freeze: which combination +/// of {style marks, root subscription, `UndoManager`} makes `doc.commit()` of +/// a whole-body delete explode on a 2.6M-char text? +fn loro_micro() { + let paragraph = "x".repeat(430); + for marks in [false, true] { + for with_subscription in [false, true] { + for with_undo in [false, true] { + let doc = loro::LoroDoc::new(); + let text = doc.get_text("t"); + for _ in 0..6000 { + text.insert(text.len_unicode(), ¶graph).expect("insert"); + text + .insert(text.len_unicode(), "\n") + .expect("insert boundary"); + } + if marks { + for i in 0..6000 { + let start = i * 431; + text.mark(start..start + 100, "bold", true).expect("mark"); + } + } + doc.commit(); + let _subscription = with_subscription.then(|| doc.subscribe_root(std::sync::Arc::new(|_event| {}))); + let _undo = with_undo.then(|| loro::UndoManager::new(&doc)); + let started = Instant::now(); + text + .delete(0, text.len_unicode()) + .expect("select-all delete"); + let delete_elapsed = started.elapsed(); + let started = Instant::now(); + doc.commit(); + let commit_elapsed = started.elapsed(); + let undo_elapsed = _undo.map(|mut undo| { + let started = Instant::now(); + let undone = undo.undo().expect("undo select-all delete"); + (started.elapsed(), undone) + }); + println!( + "marks={marks:<5} subscription={with_subscription:<5} undo={with_undo:<5} delete={delete_elapsed:>12.3?} commit={commit_elapsed:>12.3?} undo_op={undo_elapsed:?}" + ); + } + } + } +} + +fn load_runtime(path: &Path) -> anyhow::Result { + let path = &scratch_input_copy(path)?; + let is_docx = path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("docx")); + if is_docx { + let (imported, _) = flowstate_docx::import_docx_to_loro(path, "Hotpath Soak")?; + return CrdtRuntime::from_imported_document(imported); + } + CrdtRuntime::open_package(path) +} + +/// The runtime PERSISTS update segments to the package path it was opened +/// from — running the soak directly against a user's .db8 appends every +/// synthetic edit into their document (learned the hard way on a field file). +/// Work on a scratch copy, always. +fn scratch_input_copy(path: &Path) -> anyhow::Result { + let scratch = std::env::temp_dir().join(format!( + "flowstate-soak-{}-{}", + std::process::id(), + path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("input") + )); + std::fs::copy(path, &scratch)?; + Ok(scratch) +} + +fn mid_paragraph(projection: &DocumentProjection) -> ParagraphId { + projection.ids.paragraph_ids[projection.ids.paragraph_ids.len() / 2] +} + +fn drain_into_editor(handle: &LocalDocHandle, editor: &mut DocumentProjection) -> anyhow::Result<()> { + for item in handle + .drain_projection_stream() + .map_err(|error| anyhow::anyhow!("{error}"))? + { + match item { + ProjectionStreamItem::Patches(batch) => { + if std::env::var_os("SOAK_PATCH_DEBUG").is_some() { + eprintln!( + "[batch] {} patches: {:?}", + batch.patches.len(), + batch + .patches + .iter() + .map(std::mem::discriminant) + .collect::>() + ); + } + if batch.new_frontier == editor.frontier { + continue; + } + // Mirror `sync_projection_from_authority` exactly: in-place apply + // (the batch apply is internally transactional) — so the measured + // cost is the real editor-side cost. + flowstate_document::apply_projection_patch_batch(editor, &batch).map_err(|error| anyhow::anyhow!("ordered batch failed: {error:?}"))?; + }, + ProjectionStreamItem::Replace(document) => *editor = *document, + } + } + Ok(()) +} + +fn summarize(label: &str, times: &mut [Duration]) { + if times.is_empty() { + println!("{label:<48} (no samples)"); + return; + } + times.sort_unstable(); + let pick = |q: f64| times[((times.len() - 1) as f64 * q) as usize]; + println!( + "{label:<48} n={:<4} p50={:>10.3?} p90={:>10.3?} p99={:>10.3?} max={:>10.3?}", + times.len(), + pick(0.50), + pick(0.90), + pick(0.99), + times[times.len() - 1], + ); +} diff --git a/crates/flowstate-tub/src/lib.rs b/crates/flowstate-tub/src/lib.rs index c8028e24..9c9fadbe 100644 --- a/crates/flowstate-tub/src/lib.rs +++ b/crates/flowstate-tub/src/lib.rs @@ -9,8 +9,8 @@ use std::{ use anyhow::{Context as _, Result}; use flowstate_document::{ - Document, DocumentSection, InputParagraph, InputRun, SectionId, SectionKind, document_text_slice, paragraph_byte_range, - paragraph_index_for_id, paragraph_text_len, read_db8, + DocumentPackage, DocumentProjection, InputParagraph, InputRun, SearchUnitChunk, document_text_slice, paragraph_byte_range, paragraph_text_len, + read_db8, }; use ignore::WalkBuilder; use notify::{ @@ -27,7 +27,7 @@ use tantivy::{ }; const CATALOG_FILE: &str = "catalog.sqlite3"; -const TANTIVY_DIR: &str = "tantivy-v1"; +const TANTIVY_DIR: &str = "tantivy-v2"; const FILENAME_TOKENIZER: &str = "filename_ngram"; const WRITER_MEMORY_BYTES: usize = 96_000_000; @@ -49,7 +49,7 @@ impl FileKind { } } -#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] pub enum SearchUnitKind { File, Pocket, @@ -60,13 +60,18 @@ pub enum SearchUnitKind { Card, Cite, Paragraph, + ImageAlt, + ImageCaption, + Equation, + TableCell, FlowNode, Document, + Unknown(String), } impl SearchUnitKind { #[must_use] - pub const fn as_str(self) -> &'static str { + pub fn as_str(&self) -> &str { match self { Self::File => "file", Self::Pocket => "pocket", @@ -77,38 +82,35 @@ impl SearchUnitKind { Self::Card => "card", Self::Cite => "cite", Self::Paragraph => "paragraph", + Self::ImageAlt => "image_alt", + Self::ImageCaption => "image_caption", + Self::Equation => "equation", + Self::TableCell => "table_cell", Self::FlowNode => "flow_node", Self::Document => "document", + Self::Unknown(value) => value.as_str(), } } fn from_str(value: &str) -> Option { - match value { - "file" => Some(Self::File), - "pocket" => Some(Self::Pocket), - "hat" => Some(Self::Hat), - "block" => Some(Self::BlockSection), - "tag" => Some(Self::TagSection), - "analytic" => Some(Self::Analytic), - "card" => Some(Self::Card), - "cite" => Some(Self::Cite), - "paragraph" => Some(Self::Paragraph), - "flow_node" => Some(Self::FlowNode), - "document" => Some(Self::Document), - _ => None, - } - } - - const fn from_section(kind: SectionKind) -> Self { - match kind { - SectionKind::Custom(0) => Self::Pocket, - SectionKind::Custom(1) => Self::Hat, - SectionKind::Custom(2) => Self::BlockSection, - SectionKind::Custom(3) => Self::TagSection, - SectionKind::Custom(4) => Self::Analytic, - SectionKind::Custom(5) => Self::Card, - SectionKind::Custom(_) => Self::Paragraph, - } + Some(match value { + "file" => Self::File, + "pocket" => Self::Pocket, + "hat" => Self::Hat, + "block" => Self::BlockSection, + "tag" => Self::TagSection, + "analytic" => Self::Analytic, + "card" => Self::Card, + "cite" => Self::Cite, + "paragraph" => Self::Paragraph, + "image_alt" => Self::ImageAlt, + "image_caption" => Self::ImageCaption, + "equation" => Self::Equation, + "table_cell" => Self::TableCell, + "flow_node" => Self::FlowNode, + "document" => Self::Document, + _ => Self::Unknown(value.to_owned()), + }) } } @@ -155,6 +157,10 @@ pub struct SearchHit { pub score: f32, pub paragraph_start: Option, pub paragraph_end_exclusive: Option, + #[serde(default)] + pub paragraph_start_cursor: Option>, + #[serde(default)] + pub paragraph_end_cursor: Option>, } #[derive(Clone, Debug)] @@ -230,14 +236,14 @@ impl TubIndex { let path = canonicalize_file(path)?; seen_paths.insert(path.clone()); let metadata = fs::metadata(&path)?; + let size_bytes = metadata.len(); + let modified_ns = modified_ns(&metadata); let display_path = display_path_for(&self.root, &path); let parent_display_path = parent_display_path(&display_path); let file_name = path .file_name() .map_or_else(|| display_path.clone(), |name| name.to_string_lossy().to_string()); - let size_bytes = metadata.len(); - let modified_ns = modified_ns(&metadata); - let fingerprint = fingerprint(size_bytes, modified_ns); + let fingerprint = fingerprint(size_bytes, modified_ns, kind, &path)?; let existing = existing.get(&path); let file_id = existing.map_or_else(|| stable_file_id(&self.root, &path), |record| record.file_id.clone()); @@ -382,7 +388,7 @@ impl TubIndex { } else { kinds }; - let allowed = kinds.iter().copied().collect::>(); + let allowed = kinds.iter().cloned().collect::>(); let mut hits = Vec::with_capacity(limit); for file in self.list_files()? { @@ -435,7 +441,7 @@ impl TubIndex { &query, &tantivy::collector::TopDocs::with_limit(limit.saturating_mul(8).max(limit)).order_by_score(), )?; - let allowed = allowed_kinds.iter().copied().collect::>(); + let allowed = allowed_kinds.iter().cloned().collect::>(); let mut hits = Vec::new(); for (score, address) in top_docs { @@ -461,7 +467,14 @@ impl TubIndex { if !hit.preview_paragraphs.is_empty() { return Ok(()); } + if hit.paragraph_start_cursor.is_some() && !hit.insert_text.trim().is_empty() { + hit.preview_paragraphs = vec![preview_paragraph_from_text(&hit.insert_text)]; + return Ok(()); + } let Some(start) = hit.paragraph_start else { + if !hit.insert_text.trim().is_empty() { + hit.preview_paragraphs = vec![preview_paragraph_from_text(&hit.insert_text)]; + } return Ok(()); }; let Some(end) = hit.paragraph_end_exclusive else { @@ -686,6 +699,8 @@ struct TubSchema { insert_text: Field, paragraph_start: Field, paragraph_end: Field, + paragraph_start_cursor: Field, + paragraph_end_cursor: Field, size_bytes: Field, modified_ns: Field, } @@ -740,6 +755,8 @@ impl From for SearchHit { score: 0.0, paragraph_start: None, paragraph_end_exclusive: None, + paragraph_start_cursor: None, + paragraph_end_cursor: None, } } } @@ -759,6 +776,8 @@ struct IndexUnit { insert_text: String, paragraph_start: Option, paragraph_end_exclusive: Option, + paragraph_start_cursor: Option>, + paragraph_end_cursor: Option>, } struct FileDocumentInput<'input> { @@ -794,6 +813,8 @@ fn build_schema() -> (Schema, TubSchema) { let insert_text = builder.add_text_field("insert_text", STORED); let paragraph_start = builder.add_text_field("paragraph_start", STORED); let paragraph_end = builder.add_text_field("paragraph_end", STORED); + let paragraph_start_cursor = builder.add_text_field("paragraph_start_cursor", STORED); + let paragraph_end_cursor = builder.add_text_field("paragraph_end_cursor", STORED); let size_bytes = builder.add_u64_field("size_bytes", STORED); let modified_ns = builder.add_u64_field("modified_ns", STORED); let schema = builder.build(); @@ -812,6 +833,8 @@ fn build_schema() -> (Schema, TubSchema) { insert_text, paragraph_start, paragraph_end, + paragraph_start_cursor, + paragraph_end_cursor, size_bytes, modified_ns, }; @@ -840,6 +863,8 @@ fn file_document(schema: &TubSchema, input: FileDocumentInput<'_>) -> TantivyDoc schema.insert_text => "", schema.paragraph_start => "", schema.paragraph_end => "", + schema.paragraph_start_cursor => "", + schema.paragraph_end_cursor => "", schema.size_bytes => input.size_bytes, schema.modified_ns => input.modified_ns, ) @@ -861,6 +886,8 @@ fn unit_document(schema: &TubSchema, unit: &IndexUnit) -> TantivyDocument { schema.insert_text => unit.insert_text.as_str(), schema.paragraph_start => unit.paragraph_start.map(|value| value.to_string()).unwrap_or_default(), schema.paragraph_end => unit.paragraph_end_exclusive.map(|value| value.to_string()).unwrap_or_default(), + schema.paragraph_start_cursor => unit.paragraph_start_cursor.as_deref().map(hex_bytes).unwrap_or_default(), + schema.paragraph_end_cursor => unit.paragraph_end_cursor.as_deref().map(hex_bytes).unwrap_or_default(), schema.size_bytes => unit.insert_text.len() as u64, schema.modified_ns => 0_u64, ) @@ -883,6 +910,8 @@ fn hit_from_unit(unit: IndexUnit) -> SearchHit { score: 0.0, paragraph_start: unit.paragraph_start, paragraph_end_exclusive: unit.paragraph_end_exclusive, + paragraph_start_cursor: unit.paragraph_start_cursor, + paragraph_end_cursor: unit.paragraph_end_cursor, } } @@ -910,6 +939,8 @@ fn hit_from_document(schema: &TubSchema, document: &TantivyDocument, score: f32) score, paragraph_start: stored_text(document, schema.paragraph_start).and_then(|value| value.parse::().ok()), paragraph_end_exclusive: stored_text(document, schema.paragraph_end).and_then(|value| value.parse::().ok()), + paragraph_start_cursor: stored_text(document, schema.paragraph_start_cursor).and_then(|value| unhex_bytes(&value)), + paragraph_end_cursor: stored_text(document, schema.paragraph_end_cursor).and_then(|value| unhex_bytes(&value)), }) } @@ -920,103 +951,124 @@ fn stored_text(document: &TantivyDocument, field: Field) -> Option { .map(ToOwned::to_owned) } +fn hex_bytes(bytes: &[u8]) -> String { + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write as _; + let _ = write!(&mut out, "{byte:02x}"); + } + out +} + +fn unhex_bytes(value: &str) -> Option> { + if value.is_empty() { + return None; + } + let mut bytes = Vec::with_capacity(value.len() / 2); + let mut chunks = value.as_bytes().chunks_exact(2); + if !chunks.remainder().is_empty() { + return None; + } + for chunk in &mut chunks { + let text = std::str::from_utf8(chunk).ok()?; + bytes.push(u8::from_str_radix(text, 16).ok()?); + } + Some(bytes) +} + fn db8_index_units(file_id: &str, path: &Path, display_path: &str, file_name: &str) -> Result> { - let document = read_db8(path).with_context(|| format!("reading {}", path.display()))?; - let sections_by_id = document - .sections - .iter() - .map(|section| (section.id, section)) - .collect::>(); - let mut units = Vec::new(); - - for section in document.sections.iter() { - let unit_kind = SearchUnitKind::from_section(section.kind); - if !matches!( - unit_kind, - SearchUnitKind::BlockSection | SearchUnitKind::TagSection | SearchUnitKind::Analytic - ) { - continue; - } - let Some(start) = paragraph_index_for_id(&document, section.start_paragraph) else { - continue; - }; - let end = section_end_index(&document, section).unwrap_or(document.paragraphs.len()); - let end = end.max(start + 1).min(document.paragraphs.len()); - let body = paragraph_range_text(&document, start, end); - let heading = section - .heading_paragraph - .and_then(|id| paragraph_index_for_id(&document, id)) - .map(|ix| paragraph_text_trimmed(&document, ix)) - .filter(|text| !text.is_empty()) - .unwrap_or_else(|| first_non_empty_line(&body).unwrap_or_else(|| unit_kind.as_str().to_owned())); - let heading_path = section_heading_path(&document, §ions_by_id, section); - let cite = cite_for_range(&document, start, end); - let unit_id = format!("{file_id}:{}:{}", unit_kind.as_str(), section.start_paragraph.0); - units.push(IndexUnit { - file_id: file_id.to_owned(), - unit_id, - unit_kind, - path: path.to_path_buf(), - display_path: display_path.to_owned(), - file_name: file_name.to_owned(), - heading_path, - heading, - cite, - body: body.clone(), - insert_text: body, - paragraph_start: Some(start), - paragraph_end_exclusive: Some(end), - }); + if let Some(units) = + DocumentPackage::read_cached_search_units(path).with_context(|| format!("reading cached Flowstate search units {}", path.display()))? + { + return Ok( + units + .iter() + .filter_map(|unit| package_search_unit(file_id, path, display_path, file_name, unit)) + .collect(), + ); + } + let mut package = DocumentPackage::read(path).with_context(|| format!("reading Flowstate package {}", path.display()))?; + if package.current_search_units().is_empty() { + let doc = package + .load_loro_doc() + .with_context(|| format!("loading Loro document {}", path.display()))?; + package + .rebuild_search_units_from_loro(&doc) + .with_context(|| format!("rebuilding Loro search units {}", path.display()))?; } + Ok( + package + .current_search_units() + .iter() + .filter_map(|unit| package_search_unit(file_id, path, display_path, file_name, unit)) + .collect(), + ) +} - Ok(units) +fn package_search_unit(file_id: &str, path: &Path, display_path: &str, file_name: &str, unit: &SearchUnitChunk) -> Option { + let unit_kind = SearchUnitKind::from_str(&unit.unit_kind)?; + let body = unit.body.trim().to_string(); + if body.is_empty() { + return None; + } + let heading = if unit.heading.trim().is_empty() { + first_non_empty_line(&body).unwrap_or_else(|| unit_kind.as_str().to_string()) + } else { + unit.heading.clone() + }; + Some(IndexUnit { + file_id: file_id.to_owned(), + unit_id: format!("{file_id}:loro:{:032x}", unit.unit_id), + unit_kind, + path: path.to_path_buf(), + display_path: display_path.to_owned(), + file_name: file_name.to_owned(), + heading_path: unit.heading_path.clone(), + heading, + cite: None, + body: body.clone(), + insert_text: if unit.insert_text.is_empty() { body } else { unit.insert_text.clone() }, + paragraph_start: None, + paragraph_end_exclusive: None, + paragraph_start_cursor: Some(cursor_for_index(unit)).filter(|cursor| !cursor.is_empty()), + paragraph_end_cursor: Some(end_cursor_for_index(unit)).filter(|cursor| !cursor.is_empty()), + }) } -fn section_end_index(document: &Document, section: &DocumentSection) -> Option { - let id = section.end_paragraph_exclusive?; - paragraph_index_for_id(document, id) +fn cursor_for_index(unit: &SearchUnitChunk) -> Vec { + if unit.paragraph_start_cursor.is_empty() { + unit.unit_start_cursor.clone() + } else { + unit.paragraph_start_cursor.clone() + } } -fn section_heading_path(document: &Document, sections_by_id: &HashMap, section: &DocumentSection) -> Vec { - let mut path = Vec::new(); - let mut current = Some(section.id); - while let Some(section_id) = current { - let Some(section) = sections_by_id.get(§ion_id).copied() else { - break; - }; - if let Some(heading) = section - .heading_paragraph - .and_then(|id| paragraph_index_for_id(document, id)) - .map(|ix| paragraph_text_trimmed(document, ix)) - .filter(|text| !text.is_empty()) - { - path.push(heading); - } - current = section.parent_id; +fn end_cursor_for_index(unit: &SearchUnitChunk) -> Vec { + if unit.paragraph_end_cursor.is_empty() { + unit.unit_end_cursor.clone() + } else { + unit.paragraph_end_cursor.clone() } - path.reverse(); - path } -fn paragraph_range_text(document: &Document, start: usize, end: usize) -> String { - let mut text = String::new(); - for paragraph_ix in start..end { - if !text.is_empty() { - text.push('\n'); - } - text.push_str(¶graph_text_trimmed(document, paragraph_ix)); +fn preview_paragraph_from_text(text: &str) -> InputParagraph { + InputParagraph { + style: flowstate_document::ParagraphStyle::Normal, + runs: vec![InputRun { + text: text.to_string(), + styles: flowstate_document::RunStyles::default(), + }], } - text.trim().to_owned() } -fn input_paragraphs_from_document_range(document: &Document, start: usize, end: usize) -> Vec { +fn input_paragraphs_from_document_range(document: &DocumentProjection, start: usize, end: usize) -> Vec { (start..end.min(document.paragraphs.len())) .map(|paragraph_ix| input_paragraph_from_document_range(document, paragraph_ix, 0..paragraph_text_len(&document.paragraphs[paragraph_ix]))) .filter(|paragraph| paragraph.runs.iter().any(|run| !run.text.is_empty())) .collect() } -fn input_paragraph_from_document_range(document: &Document, paragraph_ix: usize, range: std::ops::Range) -> InputParagraph { +fn input_paragraph_from_document_range(document: &DocumentProjection, paragraph_ix: usize, range: std::ops::Range) -> InputParagraph { let paragraph = &document.paragraphs[paragraph_ix]; let paragraph_range = paragraph_byte_range(document, paragraph_ix); let start = range.start.min(paragraph_text_len(paragraph)); @@ -1042,31 +1094,6 @@ fn input_paragraph_from_document_range(document: &Document, paragraph_ix: usize, } } -fn paragraph_text_trimmed(document: &Document, paragraph_ix: usize) -> String { - document_text_slice(document, paragraph_byte_range(document, paragraph_ix)) - .trim() - .to_owned() -} - -fn cite_for_range(document: &Document, start: usize, end: usize) -> Option { - for paragraph_ix in start..end { - let paragraph = document.paragraphs.get(paragraph_ix)?; - let paragraph_range = paragraph_byte_range(document, paragraph_ix); - let mut offset = 0_usize; - for run in ¶graph.runs { - if run.styles.semantic == flowstate_document::SEMANTIC_CITE { - let text = document_text_slice(document, paragraph_range.start + offset..paragraph_range.start + offset + run.len); - let text = text.trim(); - if !text.is_empty() { - return Some(text.to_owned()); - } - } - offset += run.len; - } - } - None -} - fn first_non_empty_line(text: &str) -> Option { text .lines() @@ -1076,7 +1103,17 @@ fn first_non_empty_line(text: &str) -> Option { } fn preview_text(text: &str, max_chars: usize) -> String { - let normalized = text.split_whitespace().collect::>().join(" "); + // §perf: build the whitespace-normalized string in a single pass, avoiding the + // intermediate `Vec<&str>` allocation from `.collect().join(" ")`. Output is + // identical: `split_whitespace` drops all whitespace runs and we insert a + // single space between words with no leading/trailing space. + let mut normalized = String::with_capacity(text.len()); + for word in text.split_whitespace() { + if !normalized.is_empty() { + normalized.push(' '); + } + normalized.push_str(word); + } if normalized.chars().count() <= max_chars { return normalized; } @@ -1162,8 +1199,30 @@ fn modified_ns(metadata: &fs::Metadata) -> u64 { .expect("nanosecond timestamp is clamped to u64::MAX") } -fn fingerprint(size_bytes: u64, modified_ns: u64) -> String { - format!("{size_bytes}:{modified_ns}") +fn fingerprint(size_bytes: u64, modified_ns: u64, kind: FileKind, path: &Path) -> Result { + let mut fingerprint = format!("{size_bytes}:{modified_ns}"); + if kind == FileKind::Db8 + && let Some((frontier, unit_count)) = cached_search_metadata(path)? + { + fingerprint.push(':'); + fingerprint.push_str(&frontier); + fingerprint.push(':'); + fingerprint.push_str(&unit_count.to_string()); + } + Ok(fingerprint) +} + +fn cached_search_metadata(path: &Path) -> Result> { + let Some(units) = + DocumentPackage::read_cached_search_units(path).with_context(|| format!("reading cached Flowstate search units {}", path.display()))? + else { + return Ok(None); + }; + let frontier = units + .first() + .map(|unit| hex_bytes(&unit.frontier)) + .unwrap_or_default(); + Ok(Some((frontier, units.len()))) } fn stable_file_id(root: &Path, path: &Path) -> String { @@ -1266,3 +1325,15 @@ fn emit_tree_dir(relative_dir: &Path, depth: usize, context: &mut TreeEmitContex } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unknown_search_unit_kind_is_preserved() { + let kind = SearchUnitKind::from_str("mystery_kind").expect("unknown kind should be preserved"); + assert_eq!(kind.as_str(), "mystery_kind"); + assert!(matches!(kind, SearchUnitKind::Unknown(value) if value == "mystery_kind")); + } +} diff --git a/crates/flowstate/Cargo.toml b/crates/flowstate/Cargo.toml index 7368610c..8f4518dd 100644 --- a/crates/flowstate/Cargo.toml +++ b/crates/flowstate/Cargo.toml @@ -10,25 +10,39 @@ workspace = true [dependencies] anyhow = { workspace = true } +async-channel = { workspace = true } clap = { workspace = true } dirs = { workspace = true } fff-search = { workspace = true } flowstate-document = { workspace = true } flowstate-docx = { workspace = true } +flowstate-collab = { workspace = true } flowstate-flow = { workspace = true } flowstate-tub = { workspace = true } +futures-util = "0.3" gpui = { workspace = true } gpui-component = { workspace = true } gpui-component-assets = { workspace = true } hotpath = { workspace = true } +iroh = { workspace = true } +loro = { workspace = true } memchr = "2.8.1" mimalloc = { workspace = true } +notify = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } toml = { workspace = true } uuid = { workspace = true } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["json"] } +url = { workspace = true } +atomicwrites = "0.4.4" +blake3 = "1.8.5" +flowstate-fidelity.workspace = true +chrono = { version = "0.4.45", default-features = false, features = ["clock"] } [features] hotpath = [ @@ -36,6 +50,7 @@ hotpath = [ "flowstate-document/hotpath", "flowstate-docx/hotpath", "flowstate-flow/hotpath", + "flowstate-collab/hotpath", ] hotpath-cpu = [ "hotpath", @@ -43,6 +58,7 @@ hotpath-cpu = [ "flowstate-document/hotpath-cpu", "flowstate-docx/hotpath-cpu", "flowstate-flow/hotpath-cpu", + "flowstate-collab/hotpath-cpu", ] hotpath-alloc = [ "hotpath", @@ -50,11 +66,15 @@ hotpath-alloc = [ "flowstate-document/hotpath-alloc", "flowstate-docx/hotpath-alloc", "flowstate-flow/hotpath-alloc", + "flowstate-collab/hotpath-alloc", ] -profiler = ["hotpath", "hotpath-alloc"] +profiler = ["hotpath", "hotpath-alloc", "hotpath-cpu"] [target.'cfg(target_os = "macos")'.dependencies] gpui-symbols = { workspace = true } [target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies] cosmic-text = { workspace = true } + +[dev-dependencies] +gpui = { workspace = true, features = ["test-support"] } diff --git a/crates/flowstate/assets/linux/applications/dev.flowstate.Flowstate.desktop b/crates/flowstate/assets/linux/applications/dev.flowstate.Flowstate.desktop new file mode 100644 index 00000000..3f13489f --- /dev/null +++ b/crates/flowstate/assets/linux/applications/dev.flowstate.Flowstate.desktop @@ -0,0 +1,10 @@ +[Desktop Entry] +Type=Application +Name=Flowstate +Comment=A rich-text editor for debate documents +Exec=flowstate %U +Icon=dev.flowstate.Flowstate +Terminal=false +Categories=Office;WordProcessor; +MimeType=application/x-flowstate;x-scheme-handler/flowstate; +StartupWMClass=dev.flowstate.Flowstate diff --git a/crates/flowstate/assets/linux/icons/hicolor/scalable/apps/dev.flowstate.Flowstate.svg b/crates/flowstate/assets/linux/icons/hicolor/scalable/apps/dev.flowstate.Flowstate.svg new file mode 100644 index 00000000..aa4d9e74 --- /dev/null +++ b/crates/flowstate/assets/linux/icons/hicolor/scalable/apps/dev.flowstate.Flowstate.svg @@ -0,0 +1,4 @@ + + + + diff --git a/crates/flowstate/assets/logo/flowstate-app-icon-light.svg b/crates/flowstate/assets/logo/flowstate-app-icon-light.svg new file mode 100644 index 00000000..5cff7642 --- /dev/null +++ b/crates/flowstate/assets/logo/flowstate-app-icon-light.svg @@ -0,0 +1,4 @@ + + + + diff --git a/crates/flowstate/assets/logo/flowstate-app-icon.svg b/crates/flowstate/assets/logo/flowstate-app-icon.svg new file mode 100644 index 00000000..aa4d9e74 --- /dev/null +++ b/crates/flowstate/assets/logo/flowstate-app-icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/crates/flowstate/assets/logo/flowstate-mark-black.svg b/crates/flowstate/assets/logo/flowstate-mark-black.svg new file mode 100644 index 00000000..d4bff538 --- /dev/null +++ b/crates/flowstate/assets/logo/flowstate-mark-black.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/flowstate/assets/logo/flowstate-mark-white.svg b/crates/flowstate/assets/logo/flowstate-mark-white.svg new file mode 100644 index 00000000..5ff15fbb --- /dev/null +++ b/crates/flowstate/assets/logo/flowstate-mark-white.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/flowstate/assets/logo/flowstate-mark.svg b/crates/flowstate/assets/logo/flowstate-mark.svg new file mode 100644 index 00000000..9917eb9f --- /dev/null +++ b/crates/flowstate/assets/logo/flowstate-mark.svg @@ -0,0 +1,3 @@ + + + diff --git a/crates/flowstate/assets/macos/Info.plist b/crates/flowstate/assets/macos/Info.plist new file mode 100644 index 00000000..6d3aef7a --- /dev/null +++ b/crates/flowstate/assets/macos/Info.plist @@ -0,0 +1,69 @@ + + + + + CFBundleIdentifier + dev.flowstate.Flowstate + CFBundleName + Flowstate + NSBluetoothAlwaysUsageDescription + Flowstate uses Bluetooth to find nearby collaborators you have already verified and allowed for this document. + CFBundleURLTypes + + + CFBundleURLName + Flowstate links + CFBundleURLSchemes + flowstate + + + CFBundleDocumentTypes + + + CFBundleTypeName + Flowstate Debate Document + CFBundleTypeRole + Editor + LSItemContentTypes + dev.flowstate.document + + + CFBundleTypeName + Flowstate Collaboration Invite + CFBundleTypeRole + Viewer + LSItemContentTypes + dev.flowstate.invite + + + UTExportedTypeDeclarations + + + UTTypeIdentifier + dev.flowstate.document + UTTypeDescription + Flowstate Debate Document + UTTypeConformsTo + public.data + UTTypeTagSpecification + + public.filename-extension + db8 + + + + UTTypeIdentifier + dev.flowstate.invite + UTTypeDescription + Flowstate Collaboration Invite + UTTypeConformsTo + public.plain-text + UTTypeTagSpecification + + public.filename-extension + flowinvite + + + + + diff --git a/crates/flowstate/assets/windows/Package.appxmanifest b/crates/flowstate/assets/windows/Package.appxmanifest new file mode 100644 index 00000000..c05ef27a --- /dev/null +++ b/crates/flowstate/assets/windows/Package.appxmanifest @@ -0,0 +1,58 @@ + + + + + Flowstate + Flowstate + Assets\StoreLogo.png + + + + + + + + + + + + + + + Flowstate link + + + + + .db8 + + + + + .flowinvite + + + + + + + + + + diff --git a/crates/flowstate/src/app.rs b/crates/flowstate/src/app.rs index 32b58681..b64129a4 100644 --- a/crates/flowstate/src/app.rs +++ b/crates/flowstate/src/app.rs @@ -17,7 +17,7 @@ use gpui_component::{ActiveTheme as _, Icon, IconName, Sizable as _, StyledExt a use crate::app_settings::{load_app_settings, load_keymap}; use crate::commands::register_keymap; use crate::rich_text_element::{ - Document, DocumentExportAdapter, DocumentExportFormat, RichTextEditor, demo_document, set_document_export_adapter, write_db8, + DocumentExportAdapter, DocumentExportFormat, DocumentProjection, RichTextEditor, demo_document, set_document_export_adapter, }; use crate::workspace::open_workspace_window; @@ -38,7 +38,7 @@ pub struct RichTextEditorView { #[hotpath::measure_all] impl RichTextEditorView { /// Create a new editor entity from a loaded document. - pub fn new(document: Document, document_path: Option, cx: &mut Context) -> Self { + pub fn new(document: DocumentProjection, document_path: Option, cx: &mut Context) -> Self { let editor = cx.new(|cx| RichTextEditor::new_with_path(document, document_path, cx)); Self { editor } } @@ -72,6 +72,9 @@ impl Render for RichTextEditorView { #[hotpath::measure] pub fn register_rich_text_editor_keybindings(cx: &mut App) { register_keymap(cx, &load_keymap()); + // Global fidelity marker: stamps "the bug is happening now" + the recent event + // ring into the diagnostics stream. No-op unless FLOWSTATE_TRACE_FIDELITY is on. + cx.bind_keys([KeyBinding::new("ctrl-alt-m", crate::commands::FidelityMarkAction, None)]); } #[hotpath::measure] @@ -286,13 +289,15 @@ impl Focusable for FlowPromptRenderer { /// can call the same maintenance path as the standalone binary. #[hotpath::measure] pub fn write_demo_document() -> anyhow::Result<()> { - write_db8("data/demo.db8", &demo_document())?; + let document = demo_document(); + let mut runtime = flowstate_collab::crdt_runtime::CrdtRuntime::from_document_projection(&document, "Flowstate Demo")?; + let _ = runtime.checkpoint_package("Flowstate Demo", Some("data/demo.db8".into()))?; Ok(()) } -struct FlowstateDocumentExportAdapter; +struct FlowstateFlowtextAdapter; -impl DocumentExportAdapter for FlowstateDocumentExportAdapter { +impl DocumentExportAdapter for FlowstateFlowtextAdapter { fn send_output_directory(&self, source_path: Option<&Path>, recovery_path: Option<&Path>) -> Option { if crate::app_settings::load_send_to_document_directory() { source_path @@ -304,36 +309,92 @@ impl DocumentExportAdapter for FlowstateDocumentExportAdapter { } } - fn write_document_export(&self, output_path: &Path, document: &Document, format: DocumentExportFormat) -> io::Result<()> { + fn write_document_export(&self, output_path: &Path, document: &DocumentProjection, format: DocumentExportFormat) -> io::Result<()> { match format { - DocumentExportFormat::Native => write_db8(output_path, document), - DocumentExportFormat::NativeWithExtension(_) => write_db8(output_path, document), + DocumentExportFormat::Native | DocumentExportFormat::NativeWithExtension(_) | DocumentExportFormat::Pdf => Err(io::Error::new( + io::ErrorKind::Unsupported, + "native Flowstate and PDF source export require the document CRDT runtime", + )), DocumentExportFormat::Docx => crate::docx_conversion::write_docx(output_path, document), - DocumentExportFormat::Pdf => crate::docx_conversion::write_pdf(output_path, document), } } } #[hotpath::measure] fn install_flowtext_adapters() { - let _ = set_document_export_adapter(Arc::new(FlowstateDocumentExportAdapter)); + let adapter = Arc::new(FlowstateFlowtextAdapter); + let _ = set_document_export_adapter(adapter); } /// Run the rich text processor by itself for focused component development. #[hotpath::measure] -pub fn run_standalone(document_path: Option) { - Application::new() - .with_assets(AppAssets) - .run(|cx: &mut App| { - gpui_component::init(cx); - init_theme_registry(cx); - apply_saved_theme(cx); - register_rich_text_editor_keybindings(cx); - install_prompt_renderer(cx); - install_flowtext_adapters(); - open_workspace_window(document_path, cx); - cx.activate(true); - }); +pub fn run_standalone(mut document_path: Option) { + let initial_invite = document_path + .as_ref() + .filter(|path| { + path + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("flowinvite")) + }) + .and_then(|path| std::fs::read_to_string(path).ok()); + if initial_invite.is_some() { + document_path = None; + } + let (open_url_tx, open_url_rx) = async_channel::unbounded(); + let initial_url_tx = open_url_tx.clone(); + let application = Application::new().with_assets(AppAssets); + application.on_open_urls(move |urls| { + for url in urls { + let _ = open_url_tx.try_send(url); + } + }); + application.run(move |cx: &mut App| { + gpui_component::init(cx); + init_theme_registry(cx); + apply_saved_theme(cx); + register_rich_text_editor_keybindings(cx); + install_prompt_renderer(cx); + install_flowtext_adapters(); + let workspace = open_workspace_window(document_path, cx); + if let Some(invite) = initial_invite { + let _ = initial_url_tx.try_send(invite); + } + cx.spawn(async move |cx| { + while let Ok(url) = open_url_rx.recv().await { + let handled = cx + .update(|cx| crate::collab::dropbox_oauth::route_callback(&url, cx)) + .unwrap_or(false); + if handled { + continue; + } + let result = flowstate_collab::ticket::SessionTicket::decode_text(&url); + let _ = cx.update(|cx| { + let Some(window_handle) = cx.active_window() else { + tracing::warn!("received collaboration deep link without an active window"); + return; + }; + let _ = window_handle.update(cx, |_, window, cx| match result { + Ok(ticket) => { + let _ = workspace.update(cx, |workspace, cx| workspace.join_collaboration_session(ticket, window, cx)); + cx.activate(true); + }, + Err(error) => { + let detail = format!("This Flowstate collaboration link is invalid or damaged: {error}"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Invite couldn't be opened", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + }, + }); + }); + } + }) + .detach(); + cx.activate(true); + }); } struct AppAssets; @@ -370,6 +431,11 @@ impl AssetSource for AppAssets { "icons/send-horizontal.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/icons/send-horizontal.svg")))), "icons/pdf.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/icons/pdf.svg")))), "icons/docx.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/icons/docx.svg")))), + "logo/flowstate-app-icon.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/logo/flowstate-app-icon.svg")))), + "logo/flowstate-app-icon-light.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/logo/flowstate-app-icon-light.svg")))), + "logo/flowstate-mark.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/logo/flowstate-mark.svg")))), + "logo/flowstate-mark-white.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/logo/flowstate-mark-white.svg")))), + "logo/flowstate-mark-black.svg" => Ok(Some(Cow::Borrowed(include_bytes!("../assets/logo/flowstate-mark-black.svg")))), _ => gpui_component_assets::Assets.load(path), } } @@ -461,6 +527,21 @@ impl AssetSource for AppAssets { if "icons/docx.svg".starts_with(path) { assets.push("icons/docx.svg".into()); } + if "logo/flowstate-app-icon.svg".starts_with(path) { + assets.push("logo/flowstate-app-icon.svg".into()); + } + if "logo/flowstate-app-icon-light.svg".starts_with(path) { + assets.push("logo/flowstate-app-icon-light.svg".into()); + } + if "logo/flowstate-mark.svg".starts_with(path) { + assets.push("logo/flowstate-mark.svg".into()); + } + if "logo/flowstate-mark-white.svg".starts_with(path) { + assets.push("logo/flowstate-mark-white.svg".into()); + } + if "logo/flowstate-mark-black.svg".starts_with(path) { + assets.push("logo/flowstate-mark-black.svg".into()); + } Ok(assets) } } diff --git a/crates/flowstate/src/app_settings/io.rs b/crates/flowstate/src/app_settings/io.rs index a5b36eb0..f433bb91 100644 --- a/crates/flowstate/src/app_settings/io.rs +++ b/crates/flowstate/src/app_settings/io.rs @@ -1,6 +1,126 @@ +use std::{ + collections::HashMap, + path::Path, + sync::{Arc, Mutex, OnceLock}, +}; + +use notify::{Config as NotifyConfig, RecommendedWatcher, RecursiveMode, Watcher}; + +type CommandKeyCache = HashMap>; + +#[derive(Clone)] +struct CachedAppSettings { + path: PathBuf, + settings: AppSettings, + effective_keymap: crate::commands::Keymap, + keys_by_command: CommandKeyCache, +} + +// Arc'd cache entries: every settings/keybinding accessor previously DEEP +// CLONED the whole cached struct (keymap, per-command key vectors, recents, +// theme — ~13KB) per lookup; ribbon tooltips alone made 21k such lookups per +// session (274MB). An Arc bump replaces all of that. +static APP_SETTINGS_CACHE: OnceLock>>> = OnceLock::new(); +static APP_SETTINGS_WATCHER: OnceLock>> = OnceLock::new(); + +fn app_settings_cache() -> &'static Mutex>> { + APP_SETTINGS_CACHE.get_or_init(|| Mutex::new(None)) +} + +fn invalidate_app_settings_cache() { + if let Ok(mut cache) = app_settings_cache().lock() { + *cache = None; + } +} + +fn start_app_settings_watcher(path: &std::path::Path) { + let Some(parent) = path.parent().map(PathBuf::from) else { + return; + }; + if let Err(error) = fs::create_dir_all(&parent) { + tracing::warn!(error = %error, path = %parent.display(), "failed to create app settings directory for watcher"); + return; + } + let watcher_slot = APP_SETTINGS_WATCHER.get_or_init(|| Mutex::new(None)); + let Ok(mut watcher_slot) = watcher_slot.lock() else { + return; + }; + if watcher_slot.is_some() { + return; + } + + let watcher = RecommendedWatcher::new( + move |event: notify::Result| { + if event.is_ok() { + invalidate_app_settings_cache(); + } + }, + NotifyConfig::default(), + ); + let Ok(mut watcher) = watcher else { + tracing::warn!(path = %parent.display(), "failed to create app settings watcher"); + return; + }; + if let Err(error) = watcher.watch(&parent, RecursiveMode::NonRecursive) { + tracing::warn!(error = %error, path = %parent.display(), "failed to watch app settings directory"); + return; + } + *watcher_slot = Some(watcher); +} + +fn effective_keymap_for(settings: &AppSettings) -> crate::commands::Keymap { + if settings.keymap.is_empty() { + crate::commands::Keymap::defaults() + } else { + crate::commands::Keymap { + entries: settings.keymap.clone(), + } + } +} + +fn keys_by_command_for(keymap: &crate::commands::Keymap) -> CommandKeyCache { + let mut keys = CommandKeyCache::new(); + for entry in &keymap.entries { + keys + .entry(entry.command) + .or_default() + .push(entry.key.clone()); + } + keys +} + +fn cached_settings_from(settings: AppSettings, path: PathBuf) -> CachedAppSettings { + let effective_keymap = effective_keymap_for(&settings); + let keys_by_command = keys_by_command_for(&effective_keymap); + CachedAppSettings { + path, + settings, + effective_keymap, + keys_by_command, + } +} + #[hotpath::measure] pub fn load_app_settings() -> AppSettings { - load_app_settings_from_path(settings_path()).unwrap_or_default() + load_cached_app_settings().settings.clone() +} + +fn load_cached_app_settings() -> Arc { + let path = settings_path(); + start_app_settings_watcher(&path); + + if let Ok(cache) = app_settings_cache().lock() + && let Some(cached) = cache.as_ref().filter(|cached| cached.path == path) + { + return Arc::clone(cached); + } + + let settings = load_app_settings_from_path(path.clone()).unwrap_or_default(); + let cached = Arc::new(cached_settings_from(settings, path)); + if let Ok(mut cache) = app_settings_cache().lock() { + *cache = Some(Arc::clone(&cached)); + } + cached } fn load_app_settings_from_path(path: PathBuf) -> io::Result { @@ -58,11 +178,249 @@ pub fn load_keymap_entries() -> Vec { } pub fn load_keymap() -> crate::commands::Keymap { - let entries = load_keymap_entries(); - if entries.is_empty() { - crate::commands::Keymap::defaults() + load_cached_app_settings().effective_keymap.clone() +} + +pub fn load_dropbox_collaboration() -> Option<(flowstate_collab::dropbox::DropboxCredentials, String)> { + let settings = load_app_settings().dropbox_collaboration; + if !settings.enabled || settings.access_token.is_empty() { + return None; + } + Some(( + flowstate_collab::dropbox::DropboxCredentials { + app_key: settings.app_key, + access_token: settings.access_token, + refresh_token: settings.refresh_token, + access_token_expires_at: settings.access_token_expires_at, + }, + settings.root, + )) +} + +pub fn load_dropbox_document_binding(path: &Path) -> Option { + let canonical = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); + load_app_settings() + .dropbox_documents + .into_iter() + .find(|binding| { + binding + .local_path + .canonicalize() + .unwrap_or_else(|_| binding.local_path.clone()) + == canonical + }) +} + +#[hotpath::measure] +pub fn load_keys_for_command(command: crate::commands::CommandId) -> Vec { + load_cached_app_settings() + .keys_by_command + .get(&command) + .cloned() + .unwrap_or_default() +} + +#[hotpath::measure] +pub fn load_first_key_for_command(command: crate::commands::CommandId) -> Option { + load_cached_app_settings() + .keys_by_command + .get(&command) + .and_then(|keys| keys.first().cloned()) +} + +/// §15/§31: load the stable per-install durable author identity, minting and +/// persisting a fresh one the first time it is requested. +/// +/// The returned `(user_id, display_name)` is bound to a live document runtime +/// via `DocIoHandle::set_author_identity` so revisions record their +/// author and `users_by_id` is populated. Persisting is best-effort: a write +/// failure is logged but never fatal (the id regenerates on the next launch). +pub fn load_local_user_identity() -> (u128, Option) { + let profile = load_local_user_profile(); + (profile.user_id, Some(profile.display_name)) +} + +pub fn load_local_identity_secret() -> Option { + // Ensure first-run identity generation has happened before reading the seed. + let _ = load_local_user_profile(); + load_app_settings() + .local_identity_signing_secret + .as_deref() + .and_then(|secret| flowstate_collab::identity::PortableIdentitySecret::from_hex(secret).ok()) +} + +pub fn load_local_signed_profile() -> Option { + let profile = load_local_user_profile(); + let settings = load_app_settings(); + let secret = settings + .local_identity_signing_secret + .as_deref() + .and_then(|secret| flowstate_collab::identity::PortableIdentitySecret::from_hex(secret).ok())?; + let avatar_digest = profile + .avatar_path + .as_deref() + .and_then(|path| fs::read(path).ok()) + .map(|bytes| *blake3::hash(&bytes).as_bytes()); + Some(secret.sign_profile( + settings.local_profile_sequence.max(1), + profile.display_name, + profile.color_rgb, + avatar_digest, + )) +} + +/// Load the one active portable person profile and ensure its per-device and +/// signing identities exist. These writes are silent metadata persistence and +/// never mark an open document dirty. +pub fn load_local_user_profile() -> LocalUserProfile { + let mut settings = load_app_settings(); + let mut changed = false; + if settings.local_user_id == 0 { + settings.local_user_id = uuid::Uuid::new_v4().as_u128(); + changed = true; + } + if settings.local_device_id == 0 { + settings.local_device_id = uuid::Uuid::new_v4().as_u128(); + changed = true; + } + if settings + .local_user_display_name + .as_deref() + .is_none_or(str::is_empty) + { + settings.local_user_display_name = os_username().or_else(|| Some("Flowstate user".into())); + changed = true; + } + if settings.local_user_color_rgb.is_none() { + let digest = blake3::hash(&settings.local_user_id.to_le_bytes()); + settings.local_user_color_rgb = + Some(flowstate_collab::ids::PALETTE[usize::from(digest.as_bytes()[0]) % flowstate_collab::ids::PALETTE.len()]); + changed = true; + } + if settings + .local_identity_signing_secret + .as_deref() + .and_then(|secret| flowstate_collab::identity::PortableIdentitySecret::from_hex(secret).ok()) + .is_none() + { + settings.local_identity_signing_secret = Some(flowstate_collab::identity::PortableIdentitySecret::generate().to_hex()); + settings.local_profile_sequence = settings.local_profile_sequence.max(1); + changed = true; + } + let secret = settings + .local_identity_signing_secret + .as_deref() + .unwrap_or_default(); + let fingerprint = flowstate_collab::identity::PortableIdentitySecret::from_hex(secret) + .map(|secret| secret.public().to_string()) + .unwrap_or_else(|_| blake3::hash(secret.as_bytes()).to_hex().to_string()); + let profile = LocalUserProfile { + user_id: settings.local_user_id, + device_id: settings.local_device_id, + display_name: settings + .local_user_display_name + .clone() + .unwrap_or_else(|| "Flowstate user".into()), + color_rgb: settings.local_user_color_rgb.unwrap_or(0x5b8def) & 0x00ff_ffff, + avatar_path: settings.local_user_avatar_path.clone(), + identity_fingerprint: fingerprint, + }; + if changed && let Err(error) = save_app_settings(settings) { + tracing::warn!(error = %error, "persisting generated local collaboration profile failed"); + } + profile +} + +/// Best-effort OS account name used as the default author display name. Uses +/// only the standard environment so no extra dependency is introduced. +fn os_username() -> Option { + std::env::var("USER") + .or_else(|_| std::env::var("USERNAME")) + .ok() + .map(|name| name.trim().to_owned()) + .filter(|name| !name.is_empty()) +} + +/// Evaluate standing access for ambient discovery. A TOFU contact is never +/// enough: the contact must be safety-code verified, discovery must be active, +/// and the document must fall inside an explicit person override or squad +/// default without matching an exclusion. +pub fn standing_access_for_path(identity_key: &str, document_path: &Path) -> StandingAccess { + standing_access_in_settings(&load_app_settings(), identity_key, document_path) +} + +pub fn trusted_identity_keys_for_path(document_path: &Path) -> Vec { + let settings = load_app_settings(); + settings + .trusted_collaborators + .iter() + .filter(|contact| standing_access_in_settings(&settings, &contact.identity_key, document_path) == StandingAccess::Allowed) + .filter_map(|contact| contact.identity_key.parse().ok()) + .collect() +} + +fn standing_access_in_settings(settings: &AppSettings, identity_key: &str, document_path: &Path) -> StandingAccess { + if settings.collaboration_discovery_paused { + return StandingAccess::DiscoveryPaused; + } + let Some(contact) = settings + .trusted_collaborators + .iter() + .find(|contact| contact.identity_key == identity_key) + else { + return StandingAccess::UnknownIdentity; + }; + if !contact.verified { + return StandingAccess::VerificationRequired; + } + + let squad_scopes = settings + .collaboration_squads + .iter() + .filter(|squad| { + squad + .member_identity_keys + .iter() + .any(|member| member == identity_key) + }) + .flat_map(|squad| squad.default_scopes.iter()); + let all_scopes = contact + .scopes + .iter() + .chain(squad_scopes) + .collect::>(); + if all_scopes + .iter() + .any(|scope| matches!(scope, CollaborationScope::Exclusion(path) if document_path.starts_with(path))) + { + return StandingAccess::OutOfScope; + } + let person_has_positive_override = contact + .scopes + .iter() + .any(|scope| !matches!(scope, CollaborationScope::Exclusion(_))); + let allowed = all_scopes.iter().any(|scope| match scope { + CollaborationScope::Document(path) => path == document_path, + CollaborationScope::Folder(path) => document_path.starts_with(path), + CollaborationScope::Global => true, + CollaborationScope::Exclusion(_) => false, + }); + if person_has_positive_override { + let person_allowed = contact.scopes.iter().any(|scope| match scope { + CollaborationScope::Document(path) => path == document_path, + CollaborationScope::Folder(path) => document_path.starts_with(path), + CollaborationScope::Global => true, + CollaborationScope::Exclusion(_) => false, + }); + if person_allowed { + StandingAccess::Allowed + } else { + StandingAccess::OutOfScope + } + } else if allowed { + StandingAccess::Allowed } else { - crate::commands::Keymap { entries } + StandingAccess::OutOfScope } } @@ -133,9 +491,145 @@ pub fn save_keymap_entries(keymap: Vec) -> io::Res save_app_settings(settings) } +pub fn save_dropbox_collaboration(credentials: flowstate_collab::dropbox::DropboxCredentials, root: String, enabled: bool) -> io::Result<()> { + let mut settings = load_app_settings(); + settings.dropbox_collaboration = DropboxCollaborationSettings { + enabled, + root, + app_key: credentials.app_key, + access_token: credentials.access_token, + refresh_token: credentials.refresh_token, + access_token_expires_at: credentials.access_token_expires_at, + }; + save_app_settings(settings) +} + +pub fn save_collaboration_discovery_options(paused: bool, bluetooth_enabled: bool) -> io::Result<()> { + let mut settings = load_app_settings(); + settings.collaboration_discovery_paused = paused; + settings.bluetooth_collaboration_discovery_enabled = bluetooth_enabled; + save_app_settings(settings) +} + +pub fn save_dropbox_connection_draft(app_key: String, root: String) -> io::Result<()> { + let mut settings = load_app_settings(); + settings.dropbox_collaboration.app_key = app_key; + settings.dropbox_collaboration.root = root; + save_app_settings(settings) +} + +pub fn disconnect_dropbox_collaboration() -> io::Result<()> { + let mut settings = load_app_settings(); + settings.dropbox_collaboration.enabled = false; + settings.dropbox_collaboration.access_token.clear(); + settings.dropbox_collaboration.refresh_token = None; + settings.dropbox_collaboration.access_token_expires_at = None; + save_app_settings(settings) +} + +pub fn save_local_collaboration_profile(display_name: String, color_rgb: u32, avatar_path: Option) -> io::Result<()> { + let mut settings = load_app_settings(); + settings.local_user_display_name = Some(display_name); + settings.local_user_color_rgb = Some(color_rgb & 0x00ff_ffff); + settings.local_user_avatar_path = avatar_path; + settings.local_profile_sequence = settings.local_profile_sequence.max(1).saturating_add(1); + save_app_settings(settings) +} + +pub fn save_trusted_collaborator(collaborator: TrustedCollaborator) -> io::Result<()> { + let mut settings = load_app_settings(); + if let Some(existing) = settings + .trusted_collaborators + .iter_mut() + .find(|existing| existing.identity_key == collaborator.identity_key) + { + *existing = collaborator; + } else { + settings.trusted_collaborators.push(collaborator); + } + save_app_settings(settings) +} + +pub fn remove_trusted_collaborator(identity_key: &str) -> io::Result { + let mut settings = load_app_settings(); + let before = settings.trusted_collaborators.len(); + settings + .trusted_collaborators + .retain(|collaborator| collaborator.identity_key != identity_key); + for squad in &mut settings.collaboration_squads { + squad + .member_identity_keys + .retain(|member| member != identity_key); + } + let removed = before != settings.trusted_collaborators.len(); + if removed { + save_app_settings(settings)?; + } + Ok(removed) +} + +pub fn save_collaboration_squad(squad: CollaborationSquad) -> io::Result<()> { + let mut settings = load_app_settings(); + if let Some(existing) = settings + .collaboration_squads + .iter_mut() + .find(|existing| existing.id == squad.id) + { + *existing = squad; + } else { + settings.collaboration_squads.push(squad); + } + save_app_settings(settings) +} + +pub fn remove_collaboration_squad(id: &str) -> io::Result { + let mut settings = load_app_settings(); + let before = settings.collaboration_squads.len(); + settings.collaboration_squads.retain(|squad| squad.id != id); + let removed = before != settings.collaboration_squads.len(); + if removed { + save_app_settings(settings)?; + } + Ok(removed) +} + +pub fn save_dropbox_document_binding(binding: DropboxDocumentBinding) -> io::Result<()> { + let mut settings = load_app_settings(); + if let Some(existing) = settings + .dropbox_documents + .iter_mut() + .find(|existing| existing.local_path == binding.local_path) + { + *existing = binding; + } else { + settings.dropbox_documents.push(binding); + } + save_app_settings(settings) +} + +pub fn remove_dropbox_document_binding(path: &Path) -> io::Result { + let mut settings = load_app_settings(); + let before = settings.dropbox_documents.len(); + settings + .dropbox_documents + .retain(|binding| binding.local_path != path); + let removed = settings.dropbox_documents.len() != before; + if removed { + save_app_settings(settings)?; + } + Ok(removed) +} + #[hotpath::measure] pub fn save_app_settings(settings: AppSettings) -> io::Result<()> { - save_app_settings_to_path(&settings, settings_path()) + let path = settings_path(); + start_app_settings_watcher(&path); + save_app_settings_to_path(&settings, path.clone())?; + let cached = Arc::new(cached_settings_from(settings, path)); + if let Ok(mut cache) = app_settings_cache().lock() { + *cache = Some(cached); + } + Ok(()) } fn save_app_settings_to_path(settings: &AppSettings, path: PathBuf) -> io::Result<()> { @@ -143,5 +637,19 @@ fn save_app_settings_to_path(settings: &AppSettings, path: PathBuf) -> io::Resul fs::create_dir_all(parent)?; } let text = toml::to_string_pretty(settings).map_err(io::Error::other)?; - fs::write(path, text) + fs::write(&path, text)?; + restrict_to_owner(&path) +} + +/// The settings file carries the identity signing seed and Dropbox tokens, +/// so it must never be group/world readable. +#[cfg(unix)] +fn restrict_to_owner(path: &Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(path, fs::Permissions::from_mode(0o600)) +} + +#[cfg(not(unix))] +fn restrict_to_owner(_path: &Path) -> io::Result<()> { + Ok(()) } diff --git a/crates/flowstate/src/app_settings/path.rs b/crates/flowstate/src/app_settings/path.rs index dbd0b0fc..43deb7fa 100644 --- a/crates/flowstate/src/app_settings/path.rs +++ b/crates/flowstate/src/app_settings/path.rs @@ -1,12 +1,20 @@ +/// Sandbox override for every on-disk settings artifact. Headless tests set +/// this (and `FLOWSTATE_DATA_DIR`) to a temp dir so constructing a Workspace +/// never touches — or mints — the real user profile in `~/.config/flowstate`. +fn config_dir_override() -> Option { + std::env::var_os("FLOWSTATE_CONFIG_DIR").map(PathBuf::from) +} + #[hotpath::measure] fn settings_path() -> PathBuf { - config_dir() - .unwrap_or("./".into()) + config_dir_override() + .unwrap_or_else(|| config_dir().unwrap_or("./".into())) .join::("flowstate/settings.toml".into()) } pub fn flowstate_data_dir() -> PathBuf { - data_dir() - .unwrap_or("./".into()) + std::env::var_os("FLOWSTATE_DATA_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| data_dir().unwrap_or("./".into())) .join::("flowstate".into()) } diff --git a/crates/flowstate/src/app_settings/schema.rs b/crates/flowstate/src/app_settings/schema.rs index cc06a46d..783a36f8 100644 --- a/crates/flowstate/src/app_settings/schema.rs +++ b/crates/flowstate/src/app_settings/schema.rs @@ -14,13 +14,132 @@ use dirs::{config_dir, data_dir}; #[serde(default)] pub struct AppSettings { pub theme_name: Option, + // §15/§31: a stable per-install durable author identity. Serialized as a + // string because a v4 UUID rendered as `u128` overflows TOML's signed 64-bit + // integer range. A value of `0` means "not generated yet"; + // `load_local_user_identity` mints one and persists it on first read. Kept + // ahead of the table-valued fields below so TOML serialization stays valid. + #[serde(with = "local_user_id_serde")] + pub local_user_id: u128, + pub local_user_display_name: Option, + /// Stable profile metadata shared by presence, revisions, and discovery. + pub local_user_color_rgb: Option, + pub local_user_avatar_path: Option, + #[serde(with = "local_user_id_serde")] + pub local_device_id: u128, + /// Plaintext-at-rest pending the platform keychain security review. + pub local_identity_signing_secret: Option, + /// Monotonic signed-profile revision. Incremented whenever public profile + /// metadata changes so peers can reject stale rename/avatar updates. + pub local_profile_sequence: u64, + pub collaboration_discovery_paused: bool, + /// Nearby BLE rendezvous is opt-in because enabling it asks the operating + /// system for radio/privacy permission. + pub bluetooth_collaboration_discovery_enabled: bool, + pub dropbox_collaboration: DropboxCollaborationSettings, + pub dropbox_documents: Vec, pub document_theme: Option, pub editor: EditorSettings, pub toolkit: ToolkitSettings, pub recent_documents: Vec, + pub trusted_collaborators: Vec, + pub collaboration_squads: Vec, pub keymap: Vec, } +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default)] +pub struct DropboxCollaborationSettings { + pub enabled: bool, + /// Dropbox folder containing the document, or empty for the app root. + pub root: String, + pub app_key: String, + /// Plaintext-at-rest pending the same platform-keychain review as the local + /// identity secret. Never included in diagnostics exports. + pub access_token: String, + pub refresh_token: Option, + pub access_token_expires_at: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default)] +pub struct DropboxDocumentBinding { + pub local_path: PathBuf, + pub remote_path: String, + /// Last Dropbox revision successfully downloaded or uploaded. `None` makes + /// the first upload create-only rather than overwriting unknown cloud data. + pub revision: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default)] +pub struct TrustedCollaborator { + pub identity_key: String, + pub display_name: String, + pub avatar_path: Option, + pub color_rgb: Option, + pub verified: bool, + pub scopes: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(tag = "kind", content = "path", rename_all = "snake_case")] +pub enum CollaborationScope { + Document(PathBuf), + Folder(PathBuf), + Global, + Exclusion(PathBuf), +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default)] +pub struct CollaborationSquad { + pub id: String, + pub name: String, + pub member_identity_keys: Vec, + pub default_scopes: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LocalUserProfile { + pub user_id: u128, + pub device_id: u128, + pub display_name: String, + pub color_rgb: u32, + pub avatar_path: Option, + pub identity_fingerprint: String, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StandingAccess { + Allowed, + DiscoveryPaused, + UnknownIdentity, + VerificationRequired, + OutOfScope, +} + +/// Serde adapter that stores `local_user_id` as a string. TOML integers are +/// signed 64-bit, so a 128-bit identity cannot round-trip as a native integer. +mod local_user_id_serde { + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(value: &u128, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&value.to_string()) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let raw = String::deserialize(deserializer)?; + raw.parse::().map_err(serde::de::Error::custom) + } +} + #[derive(Clone, Deserialize, Serialize)] #[serde(default)] pub struct EditorSettings { diff --git a/crates/flowstate/src/app_settings/tests.rs b/crates/flowstate/src/app_settings/tests.rs index 60cf9e95..81778c49 100644 --- a/crates/flowstate/src/app_settings/tests.rs +++ b/crates/flowstate/src/app_settings/tests.rs @@ -43,4 +43,54 @@ smart_word_selection = false assert_eq!(settings.theme_name.as_deref(), Some("sunset")); assert!(!settings.editor.smart_word_selection); } + + #[test] + fn dropbox_document_bindings_round_trip() { + let binding = DropboxDocumentBinding { + local_path: PathBuf::from("/briefs/aff.db8"), + remote_path: "/Team/aff.db8".into(), + revision: Some("015abc".into()), + }; + let mut settings = AppSettings::default(); + settings.dropbox_documents.push(binding.clone()); + let text = toml::to_string(&settings).unwrap(); + let decoded: AppSettings = toml::from_str(&text).unwrap(); + assert_eq!(decoded.dropbox_documents.len(), 1); + assert_eq!(decoded.dropbox_documents[0].local_path, binding.local_path); + assert_eq!(decoded.dropbox_documents[0].remote_path, binding.remote_path); + assert_eq!(decoded.dropbox_documents[0].revision, binding.revision); + } + + #[test] + fn standing_access_requires_verification_scope_and_honors_exclusions() { + let mut settings = AppSettings::default(); + settings.trusted_collaborators.push(TrustedCollaborator { + identity_key: "alex".into(), + display_name: "Alex".into(), + verified: true, + ..TrustedCollaborator::default() + }); + settings.collaboration_squads.push(CollaborationSquad { + id: "debate".into(), + name: "Debate squad".into(), + member_identity_keys: vec!["alex".into()], + default_scopes: vec![ + CollaborationScope::Folder(PathBuf::from("/team")), + CollaborationScope::Exclusion(PathBuf::from("/team/private")), + ], + }); + assert_eq!( + standing_access_in_settings(&settings, "alex", std::path::Path::new("/team/case.db8")), + StandingAccess::Allowed + ); + assert_eq!( + standing_access_in_settings(&settings, "alex", std::path::Path::new("/team/private/notes.db8")), + StandingAccess::OutOfScope + ); + settings.trusted_collaborators[0].verified = false; + assert_eq!( + standing_access_in_settings(&settings, "alex", std::path::Path::new("/team/case.db8")), + StandingAccess::VerificationRequired + ); + } } diff --git a/crates/flowstate/src/collab/asset_transfer.rs b/crates/flowstate/src/collab/asset_transfer.rs new file mode 100644 index 00000000..1eee4813 --- /dev/null +++ b/crates/flowstate/src/collab/asset_transfer.rs @@ -0,0 +1,198 @@ +use std::sync::Arc; + +use anyhow::{Result, ensure}; +use flowstate_collab::{crdt_runtime::RuntimeAssetMetadata, net::NetCommand}; +use gpui::{Context, SharedString}; + +use crate::rich_text_element::{AssetId, AssetRecord}; + +use super::CollabSession; + +pub(super) fn schedule_missing_assets( + session: &mut CollabSession, + preferred_peer: Option, + cx: &mut Context, +) { + let Some(editor) = session.editor.clone() else { + tracing::trace!(session = %session.session, preferred_peer = ?preferred_peer, "skipping collaboration asset scan because editor is missing"); + return; + }; + let Some(runtime) = session.runtime.clone() else { + tracing::trace!(session = %session.session, preferred_peer = ?preferred_peer, "skipping collaboration asset scan because Loro doc is missing"); + return; + }; + let session_id = session.session; + cx.spawn(async move |session, cx| { + let result = runtime.asset_metadata().await; + let _ = session.update(cx, |session, cx| match result { + Ok(assets) => schedule_missing_assets_from_metadata(session, editor, preferred_peer, assets, cx), + Err(error) => { + tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "collaboration asset scan failed"); + }, + }); + }) + .detach(); +} + +fn schedule_missing_assets_from_metadata( + session: &mut CollabSession, + editor: gpui::Entity, + preferred_peer: Option, + assets: Vec, + cx: &mut Context, +) { + let assets = assets + .into_iter() + .map(ImageAssetMeta::from) + .collect::>(); + tracing::trace!(session = %session.session, assets = assets.len(), "scanned collaboration image assets"); + if assets.is_empty() { + return; + } + + let missing_assets = { + let editor = editor.read(cx); + assets + .iter() + .filter(|meta| { + let id = AssetId(meta.asset_id); + editor + .document() + .assets + .assets + .get(&id) + .is_none_or(AssetRecord::is_loading_placeholder) + }) + .cloned() + .collect::>() + }; + tracing::debug!(session = %session.session, assets = assets.len(), missing_assets = missing_assets.len(), "checked collaboration image assets for missing bytes"); + if missing_assets.is_empty() { + return; + } + + let placeholders = missing_assets + .iter() + .map(|meta| (AssetId(meta.asset_id), meta.placeholder_record())) + .collect::>(); + tracing::debug!(session = %session.session, placeholders = placeholders.len(), "queueing collaboration asset placeholder records"); + session.queue_asset_records(placeholders, cx); + + let candidates = session.pull_candidates(preferred_peer); + if candidates.is_empty() { + tracing::warn!(session = %session.session, missing_assets = missing_assets.len(), "cannot pull collaboration assets because no candidate peers are available"); + return; + } + + for meta in missing_assets { + let id = AssetId(meta.asset_id); + if !session.asset_pulls_in_flight.insert(id) { + tracing::trace!(session = %session.session, ?id, "collaboration asset pull already in flight"); + continue; + } + tracing::debug!(session = %session.session, ?id, bytes = meta.byte_len, candidate_count = candidates.len(), "scheduling collaboration asset pull"); + start_asset_pull(session, candidates.clone(), meta, cx); + } +} + +fn start_asset_pull( + session: &mut CollabSession, + candidates: Vec, + meta: ImageAssetMeta, + cx: &mut Context, +) { + let (reply_tx, reply_rx) = async_channel::bounded(1); + let id = AssetId(meta.asset_id); + let session_id = session.session; + let candidate_count = candidates.len(); + if let Err(error) = session.net_tx.try_send(NetCommand::PullAsset { + session: session.session, + candidates, + asset: meta.asset_id, + reply: reply_tx, + }) { + tracing::warn!(session = %session_id, ?id, candidate_count, error = %error, "queueing collaboration asset pull failed"); + session.asset_pulls_in_flight.remove(&id); + return; + } + tracing::debug!(session = %session_id, ?id, candidate_count, expected_bytes = meta.byte_len, "queued collaboration asset pull"); + + cx.spawn(async move |session, cx| { + let result = reply_rx.recv().await; + let _ = session.update(cx, |session, cx| { + session.asset_pulls_in_flight.remove(&id); + match result { + Ok(Ok(bytes)) => match meta.record_from_bytes(bytes.bytes) { + Ok(record) => { + tracing::debug!(session = %session_id, ?id, bytes = record.bytes.len(), "collaboration asset pull succeeded"); + session.queue_asset_records(vec![(id, record)], cx); + }, + Err(error) => tracing::warn!(session = %session_id, ?id, error = %format_args!("{error:#}"), "rejected fetched collaboration asset"), + }, + Ok(Err(error)) => tracing::warn!(session = %session_id, ?id, error = %format_args!("{error:#}"), "collaboration asset pull failed"), + Err(error) => tracing::warn!(session = %session_id, ?id, error = %error, "collaboration asset pull channel closed"), + } + }); + }) + .detach(); +} + +#[derive(Clone, Debug)] +struct ImageAssetMeta { + asset_id: u128, + mime: String, + original_name: Option, + content_hash: [u8; 32], + byte_len: u64, +} + +impl From for ImageAssetMeta { + fn from(value: RuntimeAssetMetadata) -> Self { + Self { + asset_id: value.asset_id, + mime: value.mime_type, + original_name: value.original_name, + content_hash: value.content_hash, + byte_len: value.byte_length, + } + } +} + +impl ImageAssetMeta { + fn placeholder_record(&self) -> AssetRecord { + AssetRecord { + id: AssetId(self.asset_id), + mime_type: SharedString::from(self.mime.clone()), + original_name: self.original_name.clone().map(SharedString::from), + content_hash: local_cache_hash(&self.content_hash), + bytes: Arc::new(Vec::new()), + } + } + + fn record_from_bytes(&self, bytes: Vec) -> Result { + tracing::trace!( + asset = self.asset_id, + expected_bytes = self.byte_len, + received_bytes = bytes.len(), + "validating fetched collaboration asset bytes" + ); + ensure!(bytes.len() as u64 == self.byte_len, "asset byte length mismatch"); + ensure!(blake3::hash(&bytes).as_bytes() == &self.content_hash, "asset BLAKE3 digest mismatch"); + let content_hash = local_cache_hash(&self.content_hash); + Ok(AssetRecord { + id: AssetId(self.asset_id), + mime_type: SharedString::from(self.mime.clone()), + original_name: self.original_name.clone().map(SharedString::from), + content_hash, + bytes: Arc::new(bytes), + }) + } +} + +fn local_cache_hash(content_hash: &[u8; 32]) -> u64 { + u64::from_le_bytes( + content_hash[..8] + .try_into() + .expect("BLAKE3 prefix is exactly eight bytes"), + ) +} diff --git a/crates/flowstate/src/collab/discovery_runtime.rs b/crates/flowstate/src/collab/discovery_runtime.rs new file mode 100644 index 00000000..70fcb35d --- /dev/null +++ b/crates/flowstate/src/collab/discovery_runtime.rs @@ -0,0 +1,220 @@ +use std::{borrow::BorrowMut, collections::HashMap, sync::Arc, time::Duration}; + +use flowstate_collab::{ + SessionId, + bluetooth::NativeBluetoothBackend, + discovery::{DiscoveryAdvertisement, DiscoveryTransport, RendezvousBackend}, + dropbox::{DropboxClient, DropboxRendezvousBackend}, + identity::{PortableIdentitySecret, SignedProfile}, +}; +use futures_util::{FutureExt as _, future::Either}; +use gpui::{App, Timer}; +use iroh::EndpointAddr; + +use crate::app_settings::{load_app_settings, load_dropbox_collaboration}; + +const DISCOVERY_REFRESH: Duration = Duration::from_secs(30); +const ADVERTISEMENT_LIFETIME_SECS: u64 = 90; +const BLUETOOTH_SCAN_DURATION: Duration = Duration::from_secs(4); + +#[derive(Clone)] +pub(super) struct DiscoveryRuntime { + commands: async_channel::Sender, +} + +#[derive(Clone)] +pub(super) struct DiscoveryPublication { + pub secret: PortableIdentitySecret, + pub device_id: u128, + pub document_fingerprint: [u8; 32], + pub session: SessionId, + pub endpoint: EndpointAddr, + pub profile: SignedProfile, +} + +enum DiscoveryCommand { + Upsert(Box), + Remove { + session: SessionId, + }, + Scan { + document_fingerprint: [u8; 32], + reply: async_channel::Sender, + }, + Shutdown, +} + +pub struct DiscoveryScanResult { + pub advertisements: Vec, + pub failures: Vec<(DiscoveryTransport, String)>, + pub active_transports: Vec, + pub paused: bool, +} + +impl DiscoveryRuntime { + pub fn start(cx: &mut C) -> Self + where + C: BorrowMut, + { + let settings = load_app_settings(); + let dropbox = load_dropbox_collaboration(); + let bluetooth_enabled = settings.bluetooth_collaboration_discovery_enabled && !settings.collaboration_discovery_paused; + let discovery_paused = settings.collaboration_discovery_paused; + let (commands, receiver) = async_channel::unbounded(); + cx.borrow_mut() + .spawn(async move |_| run_discovery_actor(receiver, dropbox, bluetooth_enabled, discovery_paused)) + .detach(); + Self { commands } + } + + pub fn upsert(&self, publication: DiscoveryPublication) { + if let Err(error) = self + .commands + .try_send(DiscoveryCommand::Upsert(Box::new(publication))) + { + tracing::warn!(%error, "queueing collaboration discovery publication failed"); + } + } + + pub fn remove(&self, session: SessionId) { + if let Err(error) = self.commands.try_send(DiscoveryCommand::Remove { session }) { + tracing::warn!(%error, %session, "queueing collaboration discovery removal failed"); + } + } + + pub(super) fn scan(&self, document_fingerprint: [u8; 32]) -> async_channel::Receiver { + let (reply, receiver) = async_channel::bounded(1); + if let Err(error) = self + .commands + .try_send(DiscoveryCommand::Scan { document_fingerprint, reply }) + { + tracing::warn!(%error, "queueing collaboration discovery scan failed"); + } + receiver + } + + pub fn shutdown(&self) { + let _ = self.commands.try_send(DiscoveryCommand::Shutdown); + } +} + +async fn run_discovery_actor( + receiver: async_channel::Receiver, + dropbox: Option<(flowstate_collab::dropbox::DropboxCredentials, String)>, + bluetooth_enabled: bool, + discovery_paused: bool, +) { + let mut backends: Vec> = Vec::new(); + if !discovery_paused { + if let Some((credentials, root)) = dropbox { + backends.push(Arc::new(DropboxRendezvousBackend::new(DropboxClient::new(credentials), root))); + } + if bluetooth_enabled { + match NativeBluetoothBackend::new(BLUETOOTH_SCAN_DURATION).await { + Ok(backend) => backends.push(Arc::new(backend)), + Err(error) => tracing::warn!(%error, "native Bluetooth collaboration discovery is unavailable"), + } + } + } + + let mut publications: HashMap = HashMap::new(); + let mut bluetooth_cursor = 0_usize; + loop { + let command = receiver.recv().boxed(); + let refresh = Timer::after(DISCOVERY_REFRESH).boxed(); + match futures_util::future::select(command, refresh).await { + Either::Left((Ok(DiscoveryCommand::Upsert(publication)), _)) => { + let session = publication.session; + publications.insert(session, *publication); + publish_session(&backends, publications.get(&session).expect("inserted publication")).await; + }, + Either::Left((Ok(DiscoveryCommand::Remove { session }), _)) => { + if let Some(publication) = publications.remove(&session) { + clear_publication(&backends, &publication).await; + } + }, + Either::Left((Ok(DiscoveryCommand::Scan { document_fingerprint, reply }), _)) => { + let mut advertisements = Vec::new(); + let mut failures = Vec::new(); + for backend in &backends { + match backend.scan(document_fingerprint).await { + Ok(mut found) => advertisements.append(&mut found), + Err(error) => failures.push((backend.transport(), error.to_string())), + } + } + let _ = reply + .send(DiscoveryScanResult { + advertisements, + failures, + active_transports: backends.iter().map(|backend| backend.transport()).collect(), + paused: discovery_paused, + }) + .await; + }, + Either::Left((Ok(DiscoveryCommand::Shutdown) | Err(_), _)) => break, + Either::Right((_instant, _)) => { + for backend in &backends { + match backend.transport() { + DiscoveryTransport::Dropbox => { + for publication in publications.values() { + publish_backend(backend, publication).await; + } + }, + DiscoveryTransport::Bluetooth if !publications.is_empty() => { + let index = bluetooth_cursor % publications.len(); + if let Some(publication) = publications.values().nth(index) { + publish_backend(backend, publication).await; + } + bluetooth_cursor = bluetooth_cursor.wrapping_add(1); + }, + DiscoveryTransport::Bluetooth => {}, + } + } + }, + } + } + + for publication in publications.values() { + clear_publication(&backends, publication).await; + } +} + +async fn publish_session(backends: &[Arc], publication: &DiscoveryPublication) { + for backend in backends { + publish_backend(backend, publication).await; + } +} + +async fn publish_backend(backend: &Arc, publication: &DiscoveryPublication) { + let advertisement = DiscoveryAdvertisement::issue( + &publication.secret, + publication.device_id, + publication.document_fingerprint, + publication.session, + publication.endpoint.clone(), + unix_now().saturating_add(ADVERTISEMENT_LIFETIME_SECS), + publication.profile.clone(), + ); + if let Err(error) = backend.publish(advertisement).await { + tracing::warn!(%error, session = %publication.session, transport = ?backend.transport(), "publishing collaboration discovery advertisement failed"); + } +} + +async fn clear_publication(backends: &[Arc], publication: &DiscoveryPublication) { + let identity = publication.secret.public(); + for backend in backends { + if let Err(error) = backend + .clear(identity, publication.device_id, publication.document_fingerprint) + .await + { + tracing::warn!(%error, session = %publication.session, transport = ?backend.transport(), "clearing collaboration discovery advertisement failed"); + } + } +} + +fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_secs() +} diff --git a/crates/flowstate/src/collab/dropbox_checkpoint.rs b/crates/flowstate/src/collab/dropbox_checkpoint.rs new file mode 100644 index 00000000..ed0591fd --- /dev/null +++ b/crates/flowstate/src/collab/dropbox_checkpoint.rs @@ -0,0 +1,83 @@ +use std::{io, path::Path}; + +use flowstate_collab::{ + doc_io::DocIoHandle, + dropbox::{DropboxClient, DropboxWriteError}, +}; +use loro::ExportMode; + +use crate::app_settings::{ + load_dropbox_collaboration, load_dropbox_document_binding, save_dropbox_collaboration, save_dropbox_document_binding, +}; + +/// Mirror a successfully assembled local checkpoint to its explicitly bound +/// Dropbox path. Unbound documents remain strictly local. +pub async fn sync_bound_checkpoint(local_path: &Path, title: String, io_handle: &DocIoHandle, package: Vec) -> io::Result<()> { + let Some(mut binding) = load_dropbox_document_binding(local_path) else { + return Ok(()); + }; + let Some((credentials, root)) = load_dropbox_collaboration() else { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "document is bound to Dropbox, but Dropbox collaboration is disconnected", + )); + }; + let client = DropboxClient::new(credentials); + let metadata = match client + .put_checkpoint(&binding.remote_path, package.clone(), binding.revision.as_deref()) + .await + { + Ok(metadata) => metadata, + Err(DropboxWriteError::Conflict { .. }) => { + // Trust boundary: whoever can write the bound Dropbox path is a squad + // collaborator, so their checkpoint merges without any further + // authentication. The document-id gate below only prevents accidental + // cross-document clobbering, not a malicious folder member. + let remote = client + .download(&binding.remote_path) + .await + .map_err(|error| io::Error::other(format!("Dropbox conflict download failed: {error:#}")))?; + let local_package = flowstate_document::DocumentPackage::from_bytes(&package)?; + let remote_package = flowstate_document::DocumentPackage::from_bytes(&remote.bytes)?; + if local_package.manifest.document_id != remote_package.manifest.document_id { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Dropbox path contains a different Flowstate document; it was not overwritten", + )); + } + let local_doc = local_package.load_loro_doc()?; + let remote_doc = remote_package.load_loro_doc()?; + let update = remote_doc + .export(ExportMode::updates(&local_doc.oplog_vv())) + .map_err(io::Error::other)?; + if !update.is_empty() { + io_handle + .import_remote_update(update) + .await + .map_err(|error| io::Error::other(format!("merging Dropbox changes failed: {error:#}")))?; + } + io_handle + .checkpoint_package(title.clone(), Some(local_path.to_path_buf())) + .await + .map_err(|error| io::Error::other(format!("saving merged Dropbox checkpoint failed: {error:#}")))?; + let merged = io_handle + .package_bytes(title) + .await + .map_err(|error| io::Error::other(format!("assembling merged Dropbox checkpoint failed: {error:#}")))?; + client + .put_checkpoint(&binding.remote_path, merged, Some(&remote.metadata.rev)) + .await + .map_err(|error| io::Error::other(format!("Dropbox changed again during conflict recovery: {:#}", error.into_anyhow())))? + }, + Err(DropboxWriteError::Other(error)) => { + return Err(io::Error::other(format!("saved locally, but Dropbox upload failed: {error:#}"))); + }, + }; + binding.revision = Some(metadata.rev); + save_dropbox_document_binding(binding)?; + + // Persist a token refreshed during this request. The client never logs or + // returns the bearer outside this settings update. + let refreshed = client.credentials().await; + save_dropbox_collaboration(refreshed, root, true) +} diff --git a/crates/flowstate/src/collab/dropbox_oauth.rs b/crates/flowstate/src/collab/dropbox_oauth.rs new file mode 100644 index 00000000..bdd81b6a --- /dev/null +++ b/crates/flowstate/src/collab/dropbox_oauth.rs @@ -0,0 +1,103 @@ +use std::sync::{Mutex, OnceLock}; + +use anyhow::{Context as _, Result, bail}; +use flowstate_collab::dropbox::DropboxPkceFlow; +use gpui::{App, PromptButton, PromptLevel}; +use url::Url; + +use crate::app_settings::{load_app_settings, save_dropbox_collaboration}; + +const REDIRECT_URI: &str = "flowstate://oauth/dropbox"; + +fn pending_flow() -> &'static Mutex> { + static PENDING: OnceLock>> = OnceLock::new(); + PENDING.get_or_init(|| Mutex::new(None)) +} + +/// Begin Dropbox desktop OAuth using PKCE. The verifier remains process-local +/// and the callback state is checked before any token is accepted. +pub fn begin(cx: &mut App) -> Result<()> { + let settings = load_app_settings().dropbox_collaboration; + let flow = DropboxPkceFlow::begin(settings.app_key, REDIRECT_URI)?; + let authorization_url = flow.authorization_url.to_string(); + *pending_flow() + .lock() + .expect("Dropbox OAuth state mutex poisoned") = Some(flow); + cx.open_url(&authorization_url); + Ok(()) +} + +/// Route a custom URL if it is the Dropbox OAuth callback. Returns `true` +/// whenever the URL belongs to this route, including malformed callbacks, so +/// it can never fall through to collaboration-ticket decoding. +pub fn route_callback(url: &str, cx: &mut App) -> bool { + let Ok(callback) = Url::parse(url) else { return false }; + if callback.scheme() != "flowstate" || callback.host_str() != Some("oauth") || callback.path() != "/dropbox" { + return false; + } + + let flow = pending_flow() + .lock() + .expect("Dropbox OAuth state mutex poisoned") + .take(); + let root = load_app_settings().dropbox_collaboration.root; + cx.spawn(async move |cx| { + let result = async { + let flow = flow.context("No Dropbox connection was started in this Flowstate process")?; + let credentials = flow.exchange(&callback).await?; + save_dropbox_collaboration(credentials, root, true).context("saving Dropbox connection")?; + Ok::<_, anyhow::Error>(()) + } + .await; + + let _ = cx.update(|cx| { + if result.is_ok() { + crate::collab::reconfigure_discovery(cx); + } + show_result(result, cx); + }); + }) + .detach(); + true +} + +fn show_result(result: Result<()>, cx: &mut App) { + let Some(window_handle) = cx.active_window() else { + if let Err(error) = result { + tracing::error!(error = %format_args!("{error:#}"), "Dropbox authorization failed without an active window"); + } + return; + }; + let _ = window_handle.update(cx, |_, window, cx| match result { + Ok(()) => { + std::mem::drop(window.prompt( + PromptLevel::Info, + "Dropbox connected", + Some("Flowstate can now use Dropbox for explicitly linked documents and collaboration discovery."), + &[PromptButton::ok("Ok")], + cx, + )); + }, + Err(error) => { + let detail = format!("Dropbox could not be connected: {error:#}"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Dropbox connection failed", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + }, + }); +} + +pub fn cancel_pending() -> Result<()> { + let removed = pending_flow() + .lock() + .map_err(|_| anyhow::anyhow!("Dropbox OAuth state mutex poisoned"))? + .take(); + if removed.is_none() { + bail!("No Dropbox connection is pending"); + } + Ok(()) +} diff --git a/crates/flowstate/src/collab/manager.rs b/crates/flowstate/src/collab/manager.rs new file mode 100644 index 00000000..e5960779 --- /dev/null +++ b/crates/flowstate/src/collab/manager.rs @@ -0,0 +1,622 @@ +use std::{ + borrow::BorrowMut, + collections::HashMap, + path::PathBuf, + time::{Duration, SystemTime}, +}; + +use anyhow::{Context as _, Result, anyhow, ensure}; +use async_channel::Receiver; +use flowstate_collab::{ + SessionId, + discovery::{DiscoveryAdmissionRequest, DiscoveryAdvertisement, document_fingerprint}, + doc_io::DocIoHandle, + ids::PeerId, + net::{ + NetCommand, TicketSeed, + runtime::{self, CommandSender}, + }, + ticket::SessionTicket, +}; +use gpui::{App, AppContext, BorrowAppContext, Context, Entity, Global}; +use uuid::Uuid; + +use crate::app_settings::{load_local_identity_secret, load_local_signed_profile, load_local_user_profile, trusted_identity_keys_for_path}; +use crate::rich_text_element::RichTextEditor; + +use super::{ + discovery_runtime::{DiscoveryPublication, DiscoveryRuntime}, + session::{CollabSession, DetachReason, JoinedDocument, SessionPhase, SessionRosterEntry}, +}; + +pub struct JoinRequest { + pub session: SessionId, + pub completed: Receiver>, +} + +#[derive(Default)] +pub struct CollabManager { + runtime: Option, + pub(super) sessions_by_id: HashMap>, + session_by_panel: HashMap, + own_endpoint_id: Option, + discovery: Option, + discovery_documents: HashMap, + discovery_publications: HashMap, + discovery_paths: HashMap, + discovery_titles: HashMap, + pub(super) endpoint_online: bool, + pub(super) event_pump_started: bool, +} + +impl Global for CollabManager {} + +#[derive(Clone)] +struct CollabRuntime { + commands: CommandSender, +} + +impl CollabManager { + pub fn init(cx: &mut C) + where + C: BorrowAppContext, + { + cx.update_default_global::(|_, _| {}); + } + + pub(super) fn shutdown_runtime(&mut self) { + tracing::info!( + open_sessions = self.sessions_by_id.len(), + runtime_running = self.runtime.is_some(), + "shutting down collaboration runtime" + ); + if let Some(runtime) = self.runtime.take() + && let Err(error) = runtime.commands.try_send(NetCommand::Shutdown) + { + tracing::warn!(error = %error, "queueing collaboration runtime shutdown failed"); + } + self.sessions_by_id.clear(); + self.session_by_panel.clear(); + self.own_endpoint_id = None; + if let Some(discovery) = self.discovery.take() { + discovery.shutdown(); + } + self.discovery_documents.clear(); + self.discovery_publications.clear(); + self.discovery_paths.clear(); + self.discovery_titles.clear(); + self.event_pump_started = false; + self.endpoint_online = false; + tracing::debug!("collaboration runtime shutdown state cleared"); + } + + pub fn start_session_for_panel( + &mut self, + panel_id: Uuid, + editor: Entity, + title: String, + document_io: DocIoHandle, + cx: &mut Context, + ) -> Result + where + T: 'static, + { + if let Some(session) = self.session_by_panel.get(&panel_id).copied() { + tracing::debug!(%panel_id, %session, "collaboration session already exists for panel"); + return Ok(session); + } + + let commands = self.ensure_runtime(cx)?; + let session = SessionId::new(); + let canonical_document_id = editor.read(cx).document().ids.document_id; + let document_path = editor.read(cx).document_path().cloned(); + self + .discovery_documents + .insert(session, document_fingerprint(canonical_document_id)); + tracing::info!( + %panel_id, + %session, + title = %title, + "starting local collaboration session", + ); + if let Some(path) = document_path { + self.discovery_paths.insert(session, path); + } + self.discovery_titles.insert(session, title.clone()); + let collab = CollabSession::from_local_runtime(session, panel_id, editor, title, document_io, commands.clone()); + let direct_handler = collab.direct_handler(); + let entity = cx.new(|_| collab); + entity.update(cx, |session, cx| session.attach(cx)); + + self.register_session(entity.clone(), cx); + if let Err(error) = commands.try_send(NetCommand::RegisterDirectHandler { + session, + handler: direct_handler, + }) { + tracing::error!(%panel_id, %session, error = %error, "registering collaboration direct handler failed"); + Self::detach_entity( + &entity, + DetachReason::Fatal(format!("registering collaboration direct handler failed: {error}")), + cx, + ); + self.unregister_session(session); + return Err(error).context("registering collaboration direct handler failed"); + } + tracing::debug!(%panel_id, %session, "registered collaboration direct handler for local session"); + self.configure_standing_access(session); + + let (reply_tx, reply_rx) = async_channel::bounded(1); + if let Err(error) = commands.try_send(NetCommand::CreateSession { session, reply: reply_tx }) { + tracing::error!(%panel_id, %session, error = %error, "queueing collaboration create-session command failed"); + Self::detach_entity( + &entity, + DetachReason::Fatal(format!("creating collaboration network session failed: {error}")), + cx, + ); + self.unregister_session(session); + return Err(error).context("creating collaboration network session failed"); + } + tracing::debug!(%panel_id, %session, "queued collaboration create-session command"); + Self::finish_create_session(entity, reply_rx, cx); + Ok(session) + } + + pub fn join_session(&mut self, ticket: SessionTicket, cx: &mut Context) -> Result + where + T: 'static, + { + tracing::info!(session = %ticket.session, bootstrap_count = ticket.bootstrap.len(), title = %ticket.title, "joining collaboration session from ticket"); + if !ticket.is_supported_version() { + tracing::warn!(session = %ticket.session, "unsupported collaboration protocol version in ticket"); + } + ensure!(ticket.is_supported_version(), "unsupported collaboration protocol version"); + // FS-080: reject a tampered, foreign-signed, or expired invite before we + // dial anyone. The owner re-verifies the epoch at the handshake, which a + // dishonest joiner cannot skip. + ticket + .verify_for_join() + .context("collaboration invite is invalid")?; + let commands = self.ensure_runtime(cx)?; + commands + .try_send(NetCommand::EnsureUp) + .context("starting collaboration networking failed")?; + ensure!( + !ticket + .bootstrap + .iter() + .any(|addr| self.own_endpoint_id == Some(addr.id)), + "That's your own invite — open the Share dialog instead." + ); + if self.unregister_detached_session(ticket.session, cx) { + tracing::info!(session = %ticket.session, "discarded detached collaboration session before rejoin"); + } + if let Some(existing) = self.sessions_by_id.get(&ticket.session) { + let phase = existing.read(cx).phase().clone(); + tracing::warn!(session = %ticket.session, ?phase, "collaboration session is already open"); + return Err(anyhow!("collaboration session is already open")); + } + + let session = ticket.session; + let admission = ticket.admission.clone(); + let bootstrap = ticket.bootstrap.clone(); + let preferred_peer = bootstrap + .first() + .map(|addr| addr.id) + .ok_or_else(|| anyhow!("collaboration invite has no reachable participants"))?; + let collab = CollabSession::joining(session, ticket.title.clone(), commands.clone(), bootstrap.clone(), admission.clone()); + let entity = cx.new(|_| collab); + self.register_session(entity.clone(), cx); + let neighbor_rx = entity.update(cx, |session, cx| session.prepare_join_neighbor_wait(cx)); + + if let Err(error) = commands.try_send(NetCommand::JoinSession { + session, + bootstrap, + admission, + }) { + tracing::error!(%session, error = %error, "queueing collaboration join-session command failed"); + entity.update(cx, |session, cx| { + session.detach( + DetachReason::JoinFailed(format!("joining collaboration network session failed: {error}")), + cx, + ); + }); + self.unregister_session(session); + return Err(error).context("joining collaboration network session failed"); + } + tracing::debug!(%session, peer = %preferred_peer, "queued collaboration join-session command"); + let completed = entity.update(cx, |session, cx| session.begin_join_bootstrap(preferred_peer, neighbor_rx, cx)); + Ok(JoinRequest { session, completed }) + } + + pub fn attach_joined_session( + &mut self, + session_id: SessionId, + panel_id: Uuid, + editor: Entity, + cx: &mut Context, + ) -> Result<()> + where + T: 'static, + { + tracing::info!(%session_id, %panel_id, "attaching joined collaboration session to editor"); + let commands = self.ensure_runtime(cx)?; + let session = self + .sessions_by_id + .get(&session_id) + .cloned() + .ok_or_else(|| anyhow!("collaboration session is not registered"))?; + let direct_handler = session.read(cx).direct_handler(); + if let Err(error) = commands.try_send(NetCommand::RegisterDirectHandler { + session: session_id, + handler: direct_handler, + }) { + tracing::error!(%session_id, %panel_id, error = %error, "registering collaboration direct handler for joined session failed"); + Self::detach_entity( + &session, + DetachReason::Fatal(format!("registering collaboration direct handler failed: {error}")), + cx, + ); + self.unregister_session(session_id); + return Err(error).context("registering collaboration direct handler failed"); + } + tracing::debug!(%session_id, %panel_id, "registered collaboration direct handler for joined session"); + if let Err(error) = Self::establish_joined_peer(session.clone(), commands, cx) { + Self::detach_entity( + &session, + DetachReason::Fatal(format!("collaboration endpoint address unavailable: {error:#}")), + cx, + ); + self.unregister_session(session_id); + return Err(error); + } + if let Err(error) = session.update(cx, |session, cx| session.attach_joined_editor(panel_id, editor, cx)) { + Self::detach_entity( + &session, + DetachReason::Fatal(format!("attaching joined collaboration editor failed: {error:#}")), + cx, + ); + self.unregister_session(session_id); + return Err(error); + } + self.session_by_panel.insert(panel_id, session_id); + Ok(()) + } + + pub fn leave_session_for_panel(&mut self, panel_id: Uuid, cx: &mut Context) -> bool + where + T: 'static, + { + let Some(session_id) = self.session_by_panel.get(&panel_id).copied() else { + tracing::debug!(%panel_id, "leave requested for panel without collaboration session"); + return false; + }; + let Some(session) = self.sessions_by_id.get(&session_id).cloned() else { + tracing::warn!(%panel_id, %session_id, "leave requested for missing registered collaboration session"); + self.unregister_session(session_id); + return false; + }; + + tracing::info!(%panel_id, %session_id, "leaving collaboration session for panel"); + let _ = session.update(cx, |session, cx| session.detach(DetachReason::UserLeft, cx)); + self.unregister_session(session_id); + true + } + + pub fn request_ticket_for_panel(&mut self, panel_id: Uuid, cx: &mut Context) -> Option>> + where + T: 'static, + { + let session_id = self.session_by_panel.get(&panel_id).copied()?; + let session = self.sessions_by_id.get(&session_id)?.clone(); + let title = session.read(cx).title().to_string(); + let commands = self.ensure_runtime(cx).ok()?; + let (ticket_tx, ticket_rx) = async_channel::bounded(1); + let (reply_tx, reply_rx) = async_channel::bounded(1); + if let Err(error) = commands.try_send(NetCommand::MintTicket { + session: session_id, + reply: reply_tx, + }) { + tracing::error!(%panel_id, %session_id, error = %error, "queueing collaboration mint-ticket command failed"); + return None; + } + tracing::debug!(%panel_id, %session_id, "requested collaboration invite"); + + cx.spawn(async move |_, cx| { + let ticket = match reply_rx.recv().await { + Ok(Ok(minted)) => { + tracing::info!(%session_id, inviter = %minted.inviter.id, "created collaboration share ticket"); + let own_endpoint_id = minted.inviter.id; + let _ = cx.update_global::(|manager, _| manager.own_endpoint_id = Some(own_endpoint_id)); + Ok(SessionTicket::new(session_id, vec![minted.inviter], title, minted.admission)) + }, + Ok(Err(error)) => { + tracing::error!(%session_id, error = %format_args!("{error:#}"), "minting collaboration invite failed"); + Err(anyhow!("minting collaboration invite failed: {error:#}")) + }, + Err(error) => { + tracing::error!(%session_id, error = %error, "collaboration invite reply channel closed"); + Err(anyhow!("collaboration endpoint address unavailable: {error}")) + }, + }; + let _ = ticket_tx.send(ticket).await; + }) + .detach(); + Some(ticket_rx) + } + + pub fn session_for_panel(&self, panel_id: Uuid) -> Option> { + self + .session_by_panel + .get(&panel_id) + .and_then(|session| self.sessions_by_id.get(session)) + .cloned() + } + + pub fn phase_for_panel(&self, panel_id: Uuid, cx: &App) -> Option { + self + .session_for_panel(panel_id) + .map(|session| session.read(cx).phase().clone()) + } + + pub fn phase_for_session(&self, session_id: SessionId, cx: &App) -> Option { + self + .sessions_by_id + .get(&session_id) + .map(|session| session.read(cx).phase().clone()) + } + + pub fn session_for_id(&self, session_id: SessionId) -> Option> { + self.sessions_by_id.get(&session_id).cloned() + } + + pub fn roster_for_panel(&self, panel_id: Uuid, cx: &App) -> Vec { + self + .session_for_panel(panel_id) + .map_or_else(Vec::new, |session| session.read(cx).roster()) + } + + pub fn scan_document_discovery( + &mut self, + document_id: u128, + cx: &mut Context, + ) -> async_channel::Receiver + where + T: 'static, + { + if self.discovery.is_none() { + self.discovery = Some(DiscoveryRuntime::start(cx)); + } + self + .discovery + .as_ref() + .expect("discovery runtime initialized") + .scan(document_fingerprint(document_id)) + } + + fn register_session(&mut self, session: Entity, cx: &mut Context) { + let id = session.read(cx).session_id(); + if let Some(panel_id) = session.read(cx).panel_id() { + self.session_by_panel.insert(panel_id, id); + tracing::debug!(%id, %panel_id, "registered collaboration session panel mapping"); + } + self.sessions_by_id.insert(id, session); + tracing::debug!(%id, open_sessions = self.sessions_by_id.len(), "registered collaboration session"); + } + + pub(super) fn unregister_session(&mut self, session: SessionId) { + if let Some(discovery) = &self.discovery { + discovery.remove(session); + } + self.discovery_documents.remove(&session); + self.discovery_publications.remove(&session); + self.discovery_paths.remove(&session); + self.discovery_titles.remove(&session); + let removed = self.sessions_by_id.remove(&session).is_some(); + self.session_by_panel.retain(|_, active| *active != session); + tracing::debug!(%session, removed, open_sessions = self.sessions_by_id.len(), "unregistered collaboration session"); + } + + fn unregister_detached_session(&mut self, session: SessionId, cx: &mut Context) -> bool + where + T: 'static, + { + let detached = self + .sessions_by_id + .get(&session) + .is_some_and(|existing| matches!(existing.read(cx).phase(), SessionPhase::Detached(_))); + if detached { + self.unregister_session(session); + } + detached + } + + fn ensure_runtime(&mut self, cx: &mut Context) -> Result + where + T: 'static, + { + if let Some(runtime) = &self.runtime { + tracing::trace!("collaboration runtime already available"); + return Ok(runtime.commands.clone()); + } + + tracing::info!("starting collaboration runtime for manager"); + let (commands, events) = runtime::start()?; + self.runtime = Some(CollabRuntime { commands: commands.clone() }); + self.discovery = Some(DiscoveryRuntime::start(cx)); + self.start_event_pump(events, cx); + Ok(commands) + } + + fn finish_create_session(session: Entity, reply_rx: Receiver>, cx: &mut Context) + where + T: 'static, + { + let session_id = session.read(cx).session_id(); + cx.spawn(async move |_, cx| match reply_rx.recv().await { + Ok(Ok(seed)) => { + tracing::info!(%session_id, inviter = %seed.inviter.id, "collaboration create-session completed"); + let own_endpoint_id = seed.inviter.id; + let _ = cx.update_global::(|manager, _| manager.own_endpoint_id = Some(own_endpoint_id)); + let _ = session.update(cx, |session, cx| { + session.set_admission(seed.admission); + session.establish_local_peer(&seed.inviter.id, cx); + }); + let _ = cx.update_global::(|manager, _| manager.publish_discovery(session_id, seed.inviter)); + }, + Ok(Err(error)) => { + tracing::error!(%session_id, error = %format_args!("{error:#}"), "collaboration create-session failed"); + let _ = session.update(cx, |session, cx| { + session.detach(DetachReason::Fatal(format!("creating collaboration session failed: {error:#}")), cx); + }); + }, + Err(error) => { + tracing::error!(%session_id, error = %error, "collaboration create-session reply channel closed"); + let _ = session.update(cx, |session, cx| { + session.detach(DetachReason::Fatal(format!("creating collaboration session failed: {error}")), cx); + }); + }, + }) + .detach(); + } + + fn publish_discovery(&mut self, session: SessionId, endpoint: iroh::EndpointAddr) { + let Some(runtime) = &self.discovery else { return }; + let Some(document_fingerprint) = self.discovery_documents.get(&session).copied() else { + tracing::debug!(%session, "skipping collaboration discovery for a session without a canonical document identity"); + return; + }; + let Some(secret) = load_local_identity_secret() else { + tracing::warn!(%session, "skipping collaboration discovery because the local signing identity is unavailable"); + return; + }; + let Some(profile) = load_local_signed_profile() else { + tracing::warn!(%session, "skipping collaboration discovery because the signed local profile is unavailable"); + return; + }; + let local = load_local_user_profile(); + let publication = DiscoveryPublication { + secret, + device_id: local.device_id, + document_fingerprint, + session, + endpoint, + profile, + }; + runtime.upsert(publication.clone()); + self.discovery_publications.insert(session, publication); + } + + pub fn reconfigure_discovery(&mut self, cx: &mut C) + where + C: BorrowMut, + { + if let Some(discovery) = self.discovery.take() { + discovery.shutdown(); + } + if self.runtime.is_none() { + return; + } + let discovery = DiscoveryRuntime::start(cx); + for publication in self.discovery_publications.values().cloned() { + discovery.upsert(publication); + } + self.discovery = Some(discovery); + for session in self.sessions_by_id.keys().copied() { + self.configure_standing_access(session); + } + } + + fn configure_standing_access(&self, session: SessionId) { + let Some(runtime) = &self.runtime else { return }; + let Some(document_fingerprint) = self.discovery_documents.get(&session).copied() else { + return; + }; + let Some(path) = self.discovery_paths.get(&session) else { return }; + let title = self + .discovery_titles + .get(&session) + .cloned() + .unwrap_or_else(|| "Shared document".into()); + let identities = trusted_identity_keys_for_path(path); + if let Err(error) = runtime + .commands + .try_send(NetCommand::ConfigureStandingAccess { + session, + document_fingerprint, + title, + identities, + }) + { + tracing::warn!(%error, %session, "queueing standing collaboration access failed"); + } + } + + pub fn request_discovered_ticket(&mut self, advertisement: DiscoveryAdvertisement, cx: &mut Context) -> Receiver> + where + T: 'static, + { + let (reply, receiver) = async_channel::bounded(1); + let result = (|| { + let commands = self.ensure_runtime(cx)?; + let secret = load_local_identity_secret().context("local collaboration identity is unavailable")?; + let mut nonce = [0_u8; 32]; + nonce[..16].copy_from_slice(Uuid::new_v4().as_bytes()); + nonce[16..].copy_from_slice(Uuid::new_v4().as_bytes()); + let expires = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or(Duration::ZERO) + .as_secs() + .saturating_add(30); + let request = DiscoveryAdmissionRequest::issue(&secret, advertisement.session, advertisement.document_fingerprint, nonce, expires); + commands + .try_send(NetCommand::RequestDiscoveredTicket { + advertisement: Box::new(advertisement), + request: Box::new(request), + reply: reply.clone(), + }) + .context("queueing trusted-peer admission request") + })(); + if let Err(error) = result { + let _ = reply.try_send(Err(error)); + } + receiver + } + + fn establish_joined_peer(session: Entity, commands: CommandSender, cx: &mut Context) -> Result<()> + where + T: 'static, + { + let session_id = session.read(cx).session_id(); + let (reply_tx, reply_rx) = async_channel::bounded(1); + if let Err(error) = commands.try_send(NetCommand::MintTicketAddr { reply: reply_tx }) { + tracing::error!(%session_id, error = %error, "queueing collaboration endpoint address command failed"); + return Err(error).context("requesting collaboration endpoint address failed"); + } + tracing::debug!(%session_id, "requested collaboration endpoint address for joined session"); + cx.spawn(async move |_, cx| match reply_rx.recv().await { + Ok(addr) => { + tracing::info!(%session_id, peer = %addr.id, "collaboration joined session local peer established"); + let own_endpoint_id = addr.id; + let _ = cx.update_global::(|manager, _| manager.own_endpoint_id = Some(own_endpoint_id)); + let _ = session.update(cx, |session, cx| session.establish_local_peer(&addr.id, cx)); + }, + Err(error) => { + tracing::error!(%session_id, error = %error, "collaboration endpoint address unavailable for joined session"); + let _ = session.update(cx, |session, cx| { + session.detach(DetachReason::Fatal(format!("collaboration endpoint address unavailable: {error}")), cx); + }); + }, + }) + .detach(); + Ok(()) + } + + fn detach_entity(entity: &Entity, reason: DetachReason, cx: &mut Context) + where + T: 'static, + { + entity.update(cx, |session, cx| { + session.detach(reason, cx); + }); + } +} diff --git a/crates/flowstate/src/collab/mod.rs b/crates/flowstate/src/collab/mod.rs new file mode 100644 index 00000000..26c81357 --- /dev/null +++ b/crates/flowstate/src/collab/mod.rs @@ -0,0 +1,156 @@ +mod discovery_runtime; +pub(crate) mod dropbox_checkpoint; +pub mod dropbox_oauth; +mod manager; +pub(crate) mod notify; +mod presence_view; +mod pump; +mod session; +pub mod share_dialog; +mod shutdown; +pub mod status; + +use std::{borrow::BorrowMut, sync::Arc}; + +use anyhow::Result; +use async_channel::Receiver; +use flowstate_collab::{SessionId, doc_io::DocIoHandle, local_write::LocalDocHandle, ticket::SessionTicket}; +use gpui::{App, BorrowAppContext, Context, Entity, ReadGlobal}; +use uuid::Uuid; + +use crate::rich_text_element::RichTextEditor; + +pub use discovery_runtime::DiscoveryScanResult; +pub use manager::{CollabManager, JoinRequest}; +pub use session::{ + Attachment, CollabSession, Connectivity, DetachReason, JoinStage, JoinedDocument, SessionNotice, SessionPhase, SessionRosterEntry, +}; +pub use shutdown::shutdown; + +pub fn init(cx: &mut C) +where + C: BorrowAppContext, +{ + CollabManager::init(cx); +} + +pub fn start_session_for_panel( + panel_id: Uuid, + editor: Entity, + title: String, + io: DocIoHandle, + cx: &mut Context, +) -> Result +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.start_session_for_panel(panel_id, editor, title, io, cx)) +} + +/// JOIN handoff (Loro-first spec §3): take the document services the joining +/// session constructed from the initial snapshot — the write authority for the +/// editor and the I/O handle for saves/exports. One-shot: the write authority +/// moves out of the session (the session is transport-only, invariant 5); +/// subsequent calls return `None`. +pub fn take_joined_document_services_for_session(session_id: SessionId, cx: &mut Context) -> Option<(Arc, DocIoHandle)> +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| { + let session = manager.session_for_id(session_id)?; + session.update(cx, |session, _| session.take_joined_document_services()) + }) +} + +pub fn join_session(ticket: SessionTicket, cx: &mut Context) -> Result +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.join_session(ticket, cx)) +} + +pub fn attach_joined_session(session_id: SessionId, panel_id: Uuid, editor: Entity, cx: &mut Context) -> Result<()> +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.attach_joined_session(session_id, panel_id, editor, cx)) +} + +pub fn leave_session_for_panel(panel_id: Uuid, cx: &mut Context) -> bool +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.leave_session_for_panel(panel_id, cx)) +} + +pub fn request_ticket_for_panel(panel_id: Uuid, cx: &mut Context) -> Option>> +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.request_ticket_for_panel(panel_id, cx)) +} + +pub fn phase_for_panel(panel_id: Uuid, cx: &App) -> Option { + CollabManager::global(cx).phase_for_panel(panel_id, cx) +} + +pub fn phase_for_session(session_id: SessionId, cx: &App) -> Option { + CollabManager::global(cx).phase_for_session(session_id, cx) +} + +pub fn session_for_panel(panel_id: Uuid, cx: &App) -> Option> { + CollabManager::global(cx).session_for_panel(panel_id) +} + +pub fn session_for_id(session_id: SessionId, cx: &App) -> Option> { + CollabManager::global(cx).session_for_id(session_id) +} + +pub fn roster_for_panel(panel_id: Uuid, cx: &App) -> Vec { + CollabManager::global(cx).roster_for_panel(panel_id, cx) +} + +pub fn scan_document_discovery(document_id: u128, cx: &mut Context) -> Receiver +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.scan_document_discovery(document_id, cx)) +} + +pub fn reconfigure_discovery(cx: &mut C) +where + C: BorrowAppContext + BorrowMut, +{ + cx.update_default_global::(|manager, cx| manager.reconfigure_discovery(cx)); +} + +pub fn request_discovered_ticket( + advertisement: flowstate_collab::discovery::DiscoveryAdvertisement, + cx: &mut Context, +) -> Receiver> +where + T: 'static, +{ + cx.update_default_global::(|manager, cx| manager.request_discovered_ticket(advertisement, cx)) +} + +/// Notify the collaboration session for `panel_id` that the document's runtime +/// was just checkpointed (saved) outside the session. +/// +/// A save records a named revision into Loro via the runtime's save hook, which +/// advances the canonical version vector but cannot route the resulting +/// `LocalUpdate` events back to the session (the save hook future has no GPUI +/// context). This refreshes the session's cached runtime version vector and +/// re-publishes an anti-entropy digest so peers converge on the new revision op +/// promptly. Returns `false` (a harmless no-op) when the panel is not part of an +/// active collaboration session, so solo-document saves are unaffected. +pub fn refresh_after_external_checkpoint(panel_id: Uuid, cx: &mut Context) -> bool +where + T: 'static, +{ + let Some(session) = session_for_panel(panel_id, cx) else { + return false; + }; + session.update(cx, |session, cx| session.refresh_after_external_checkpoint(cx)); + true +} diff --git a/crates/flowstate/src/collab/notify.rs b/crates/flowstate/src/collab/notify.rs new file mode 100644 index 00000000..da604276 --- /dev/null +++ b/crates/flowstate/src/collab/notify.rs @@ -0,0 +1,42 @@ +use gpui::{App, PromptButton, PromptLevel, Window}; +use gpui_component::{WindowExt as _, notification::Notification}; + +use super::SessionNotice; + +pub fn show_session_notice(notice: &SessionNotice, window: &mut Window, cx: &mut App) { + match notice { + SessionNotice::PeerJoined(name) => push_info(collaborator_message(name, "joined"), window, cx), + SessionNotice::PeerLeft(name) => push_info(collaborator_message(name, "left"), window, cx), + SessionNotice::LeftSession => push_info("Left session - this copy is now local.", window, cx), + SessionNotice::Disconnected(detail) => { + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Collaboration disconnected", + Some(detail.as_str()), + &[PromptButton::ok("Ok")], + cx, + )); + }, + SessionNotice::ViewRebuilt => { + #[cfg(debug_assertions)] + push_info("Collaboration view rebuilt from remote state.", window, cx); + }, + SessionNotice::IncompatibleVersion(peer) => push_warning(format!("{peer} is using an incompatible collaboration version."), window, cx), + } +} + +fn push_info(message: impl Into, window: &mut Window, cx: &mut App) { + window.push_notification(Notification::info(message).title("Collaboration"), cx); +} + +fn push_warning(message: impl Into, window: &mut Window, cx: &mut App) { + window.push_notification(Notification::warning(message).title("Collaboration"), cx); +} + +fn collaborator_message(name: &str, action: &str) -> String { + if name.trim().is_empty() { + format!("A collaborator {action}") + } else { + format!("{name} {action}") + } +} diff --git a/crates/flowstate/src/collab/presence_view.rs b/crates/flowstate/src/collab/presence_view.rs new file mode 100644 index 00000000..513a47a7 --- /dev/null +++ b/crates/flowstate/src/collab/presence_view.rs @@ -0,0 +1,29 @@ +use std::path::PathBuf; + +use flowstate_collab::SessionId; + +pub fn collaboration_recovery_path(session: SessionId, title: &str) -> PathBuf { + let dir = std::env::temp_dir().join("flowstate-collab-recovery"); + if let Err(error) = std::fs::create_dir_all(&dir) { + tracing::warn!("creating collaboration recovery directory failed ({}): {error}", dir.display()); + } + let session_hex = session.to_string(); + let prefix = session_hex.get(..16).unwrap_or(&session_hex); + dir.join(format!("{prefix}-{}.db8", sanitized_recovery_title(title))) +} + +fn sanitized_recovery_title(title: &str) -> String { + let mut out = String::new(); + for ch in title.chars() { + if out.len() >= 48 { + break; + } + if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.') { + out.push(ch); + } else if ch.is_whitespace() && !out.ends_with('_') { + out.push('_'); + } + } + let trimmed = out.trim_matches(['_', '.']).to_string(); + if trimmed.is_empty() { "shared-document".to_string() } else { trimmed } +} diff --git a/crates/flowstate/src/collab/pump.rs b/crates/flowstate/src/collab/pump.rs new file mode 100644 index 00000000..29dbaf14 --- /dev/null +++ b/crates/flowstate/src/collab/pump.rs @@ -0,0 +1,93 @@ +use flowstate_collab::{ + SessionId, + net::{NetEvent, runtime}, +}; +use gpui::{App, Context}; + +use super::{ + manager::CollabManager, + session::{CollabSession, DetachReason}, +}; + +impl CollabManager { + pub(super) fn start_event_pump(&mut self, events: runtime::EventReceiver, cx: &mut Context) + where + T: 'static, + { + if self.event_pump_started { + tracing::trace!("collaboration network event pump already started"); + return; + } + self.event_pump_started = true; + tracing::debug!("starting collaboration network event pump"); + cx.spawn(async move |_, cx| { + while let Ok(event) = events.recv().await { + if cx + .update_global::(|manager, cx| manager.handle_net_event(event, cx)) + .is_err() + { + tracing::warn!("collaboration network event pump could not update manager; stopping"); + break; + } + } + tracing::debug!("collaboration network event pump stopped"); + }) + .detach(); + } + + fn handle_net_event(&mut self, event: NetEvent, cx: &mut App) { + match event { + NetEvent::EndpointOnline(online) => { + tracing::info!(online, "collaboration endpoint status changed"); + self.endpoint_online = online; + for session in self.sessions_by_id.values() { + session.update(cx, |session, cx| session.set_endpoint_online(online, cx)); + } + }, + NetEvent::Gossip { session, from, msg } => { + tracing::trace!(%session, from = %from, gossip_kind = msg.kind(), gossip_payload_bytes = msg.payload_len(), "routing collaboration gossip event"); + self.update_session(session, cx, |session, cx| session.handle_gossip(from, msg, cx)); + }, + NetEvent::IncompatibleVersion { session, peer } => { + tracing::warn!(%session, peer = %peer, "routing collaboration incompatible-version event"); + self.update_session(session, cx, |session, cx| session.handle_incompatible_version(peer, cx)); + }, + NetEvent::NeighborUp { session, peer } => { + tracing::debug!(%session, peer = %peer, "routing collaboration neighbor-up event"); + self.update_session(session, cx, |session, cx| session.neighbor_up(peer, cx)); + }, + NetEvent::NeighborDown { session, peer } => { + tracing::debug!(%session, peer = %peer, "routing collaboration neighbor-down event"); + self.update_session(session, cx, |session, cx| session.neighbor_down(peer, cx)); + }, + NetEvent::GossipLagged { session } => { + tracing::warn!(%session, "routing collaboration gossip-lagged event"); + self.update_session(session, cx, |session, cx| session.handle_gossip_lagged(cx)); + }, + NetEvent::SubscribeFailed { session, error } => { + tracing::error!(%session, error = %error, "collaboration gossip subscription failed"); + let detached = self.update_session(session, cx, |session, cx| session.detach(DetachReason::Fatal(error), cx)); + if detached.unwrap_or(false) { + self.unregister_session(session); + } + }, + } + } + + fn update_session( + &mut self, + session: SessionId, + cx: &mut App, + update: impl FnOnce(&mut CollabSession, &mut Context) -> R, + ) -> Option { + self + .sessions_by_id + .get(&session) + .cloned() + .map(|session| session.update(cx, update)) + .or_else(|| { + tracing::warn!(%session, "collaboration event targeted missing session"); + None + }) + } +} diff --git a/crates/flowstate/src/collab/session.rs b/crates/flowstate/src/collab/session.rs new file mode 100644 index 00000000..a8f03cb3 --- /dev/null +++ b/crates/flowstate/src/collab/session.rs @@ -0,0 +1,1236 @@ +use std::{ + collections::HashSet, + sync::Arc, + time::{Duration, Instant}, +}; + +use anyhow::{Context as _, Result, anyhow, bail}; +use flowstate_collab::{ + SessionAdmission, SessionId, + crdt_runtime::{CrdtRuntime, RuntimeEvent}, + doc_io::DocIoHandle, + ids::PeerId, + local_write::{LocalDocHandle, LocalWriteConfig}, + net::{ + NetCommand, PeerAddr, PublishPayload, + anti_entropy::{AntiEntropyState, GapAction}, + direct::{DirectServeRequest, DirectSessionHandler}, + runtime::CommandSender, + }, + presence::PresenceStore, + proto_gossip::GossipMsg, +}; +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use gpui::{Context, Entity, EventEmitter, Subscription, Timer}; +use loro::{LoroDoc, Subscription as LoroSubscription}; +use uuid::Uuid; + +use crate::app_settings::{load_document_theme, load_local_user_identity, load_local_user_profile}; +use crate::rich_text_element::{AssetId, AssetRecord, CollaborationRole, DocumentProjection, EditorEvent, RichTextEditor}; + +use super::presence_view; + +#[path = "asset_transfer.rs"] +mod asset_transfer; +#[path = "session_io.rs"] +mod session_io; +#[path = "session_presence.rs"] +mod session_presence; +#[path = "session_timers.rs"] +mod session_timers; + +#[derive(Clone, Debug)] +pub enum SessionPhase { + Creating, + Joining(JoinStage), + Attached(Attachment), + Detached(DetachReason), +} + +#[derive(Clone, Debug)] +pub struct Attachment { + pub connectivity: Connectivity, + pub peers_present: usize, +} + +#[derive(Clone, Debug)] +pub enum Connectivity { + Online, + Offline { since: Instant, retries: u32 }, +} + +#[derive(Clone, Debug)] +pub enum JoinStage { + Resolving, + Subscribing, + FetchingSnapshot { got: u64, total: Option }, + Building, +} + +#[derive(Clone, Debug)] +pub enum DetachReason { + UserLeft, + JoinFailed(String), + Fatal(String), +} + +#[derive(Clone, Debug)] +pub enum SessionNotice { + PeerJoined(String), + PeerLeft(String), + LeftSession, + Disconnected(String), + ViewRebuilt, + IncompatibleVersion(String), +} + +impl EventEmitter for CollabSession {} + +pub(super) enum JoinNeighborSignal { + NeighborUp, + TimedOut, +} + +pub struct JoinedDocument { + pub session: SessionId, + pub title: String, + pub document: DocumentProjection, +} + +#[derive(Clone, Debug)] +pub struct SessionRosterEntry { + pub name: String, + pub color_rgb: u32, + pub is_self: bool, +} + +pub struct CollabSession { + session: SessionId, + title: String, + phase: SessionPhase, + // Loro-first (spec §3, invariant 5): the session is TRANSPORT-ONLY. It holds + // the document I/O service handle to import remote updates and drain the + // publish queue — never a write path into the document. + runtime: Option, + // JOIN handoff slot (spec §3 join gate): the write authority constructed by + // `finish_join_snapshot`, held ONLY until the workspace takes it via + // `take_joined_document_services`. The session never calls into it. + join_authority: Option>, + runtime_vv: Vec, + editor: Option>, + panel_id: Option, + // UI-only asset cache records fetched after Loro metadata arrives. + pending_asset_records: Vec<(AssetId, AssetRecord)>, + pending_remote_updates: Vec>, + pending_remote_update_bytes: usize, + pending_remote_updates_overflowed: bool, + presence: Option, + net_tx: CommandSender, + direct_tx: async_channel::Sender, + direct_rx: async_channel::Receiver, + editor_subscriptions: Vec, + loro_subscriptions: Vec, + neighbors: HashSet, + bootstrap_addrs: Vec, + // Symmetric live-session admission, retained only in memory for reconnects + // and participant-created invitations. + admission: Option, + // The role this endpoint holds in the session; gates local writes/undo and is + // pushed onto the editor at attach so `can_write_collaboration()` takes effect. + collaboration_role: CollaborationRole, + asset_pulls_in_flight: HashSet, + anti_entropy: AntiEntropyState, + direct_pump_started: bool, + presence_refresh_pending: bool, + presence_refresh_generation: u64, + external_caret_refresh_pending: bool, + external_caret_refresh_generation: u64, + comment_annotation_refresh_pending: bool, + comment_annotation_refresh_generation: u64, + timers_started: bool, + endpoint_online: bool, + zero_neighbors_since: Option, + inbound_since_last_digest: bool, + quiet_digest_rounds: u8, + next_recovery_at: Option, + awaiting_recovery_neighbor_until: Option, + known_peers: HashSet, + probe_pending: bool, + last_probe_failed: bool, + join_neighbor_tx: Option>, + incompatible_version_peers: HashSet, + last_document_activity: Instant, + publish_pump_pending: bool, +} + +const DIRECT_REQUEST_CHANNEL_CAPACITY: usize = 32; +const JOIN_FIRST_NEIGHBOR_TIMEOUT: Duration = Duration::from_secs(15); +const JOIN_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(30); +/// Debounce for draining the doc I/O publish queue after local commits (spec +/// §6 publication rule: an update is publishable once its intent released the +/// write gate; the session only batches the network broadcast). +const LOCAL_UPDATE_PUBLISH_DEBOUNCE_MS: u64 = 24; + +impl CollabSession { + pub fn from_local_runtime( + session: SessionId, + panel_id: Uuid, + editor: Entity, + title: String, + io: DocIoHandle, + net_tx: CommandSender, + ) -> Self { + // Direct serving is network-bound and should backpressure rather than grow without limit. + let (direct_tx, direct_rx) = async_channel::bounded(DIRECT_REQUEST_CHANNEL_CAPACITY); + let now = Instant::now(); + tracing::info!( + %session, + %panel_id, + title = %title, + "attached local collaboration session to document I/O service", + ); + + Self { + session, + title, + phase: SessionPhase::Creating, + runtime: Some(io), + join_authority: None, + runtime_vv: Vec::new(), + editor: Some(editor), + panel_id: Some(panel_id), + pending_asset_records: Vec::new(), + pending_remote_updates: Vec::new(), + pending_remote_update_bytes: 0, + pending_remote_updates_overflowed: false, + presence: None, + net_tx, + direct_tx, + direct_rx, + editor_subscriptions: Vec::new(), + loro_subscriptions: Vec::new(), + neighbors: HashSet::new(), + bootstrap_addrs: Vec::new(), + admission: None, + collaboration_role: CollaborationRole::Editor, + asset_pulls_in_flight: HashSet::new(), + anti_entropy: AntiEntropyState::new(session, now), + direct_pump_started: false, + presence_refresh_pending: false, + presence_refresh_generation: 0, + external_caret_refresh_pending: false, + external_caret_refresh_generation: 0, + comment_annotation_refresh_pending: false, + comment_annotation_refresh_generation: 0, + timers_started: false, + endpoint_online: true, + zero_neighbors_since: Some(now), + inbound_since_last_digest: false, + quiet_digest_rounds: 0, + next_recovery_at: None, + awaiting_recovery_neighbor_until: None, + known_peers: HashSet::new(), + probe_pending: false, + last_probe_failed: false, + join_neighbor_tx: None, + incompatible_version_peers: HashSet::new(), + last_document_activity: now, + publish_pump_pending: false, + } + } + + pub fn joining(session: SessionId, title: String, net_tx: CommandSender, bootstrap_addrs: Vec, admission: SessionAdmission) -> Self { + let collaboration_role = CollaborationRole::Editor; + let (direct_tx, direct_rx) = async_channel::bounded(DIRECT_REQUEST_CHANNEL_CAPACITY); + let now = Instant::now(); + tracing::info!(%session, title = %title, bootstrap_count = bootstrap_addrs.len(), "created joining collaboration session state"); + Self { + session, + title, + phase: SessionPhase::Joining(JoinStage::Resolving), + runtime: None, + join_authority: None, + runtime_vv: Vec::new(), + editor: None, + panel_id: None, + pending_asset_records: Vec::new(), + pending_remote_updates: Vec::new(), + pending_remote_update_bytes: 0, + pending_remote_updates_overflowed: false, + presence: None, + net_tx, + direct_tx, + direct_rx, + editor_subscriptions: Vec::new(), + loro_subscriptions: Vec::new(), + neighbors: HashSet::new(), + bootstrap_addrs, + admission: Some(admission), + collaboration_role, + asset_pulls_in_flight: HashSet::new(), + anti_entropy: AntiEntropyState::new(session, now), + direct_pump_started: false, + presence_refresh_pending: false, + presence_refresh_generation: 0, + external_caret_refresh_pending: false, + external_caret_refresh_generation: 0, + comment_annotation_refresh_pending: false, + comment_annotation_refresh_generation: 0, + timers_started: false, + endpoint_online: true, + zero_neighbors_since: Some(now), + inbound_since_last_digest: false, + quiet_digest_rounds: 0, + next_recovery_at: None, + awaiting_recovery_neighbor_until: None, + known_peers: HashSet::new(), + probe_pending: false, + last_probe_failed: false, + join_neighbor_tx: None, + incompatible_version_peers: HashSet::new(), + last_document_activity: now, + publish_pump_pending: false, + } + } + + pub fn session_id(&self) -> SessionId { + self.session + } + + pub fn panel_id(&self) -> Option { + self.panel_id + } + + pub fn title(&self) -> &str { + &self.title + } + + pub(super) fn set_admission(&mut self, admission: SessionAdmission) { + self.admission = Some(admission); + } + + pub fn phase(&self) -> &SessionPhase { + &self.phase + } + + pub fn roster(&self) -> Vec { + let Some(presence) = &self.presence else { + return Vec::new(); + }; + let self_key = presence.self_key().to_string(); + presence + .roster() + .into_iter() + .map(|entry| SessionRosterEntry { + is_self: entry.key == self_key, + name: entry.name, + color_rgb: entry.color_rgb, + }) + .collect() + } + + pub fn direct_handler(&self) -> DirectSessionHandler { + // Hand the direct-serve path the document I/O service handle so + // snapshot/update pulls are answered through the gate-disciplined I/O + // thread (spec I-9a: raw ungated doc reads are outlawed — they can + // force-commit mid-intent state). Absent a runtime yet, it falls back to + // the session-served request channel. + DirectSessionHandler::new(self.direct_tx.clone(), self.runtime.clone()) + } + + pub(super) fn pull_candidates(&self, preferred: Option) -> Vec { + let mut candidates = Vec::with_capacity(self.neighbors.len() + usize::from(preferred.is_some())); + if let Some(peer) = preferred { + candidates.push(peer); + } + candidates.extend( + self + .neighbors + .iter() + .copied() + .filter(|peer| Some(*peer) != preferred), + ); + candidates + } + + pub fn set_join_stage(&mut self, stage: JoinStage, cx: &mut Context) { + tracing::debug!(session = %self.session, ?stage, "collaboration join stage changed"); + self.phase = SessionPhase::Joining(stage); + cx.notify(); + } + + pub(super) fn prepare_join_neighbor_wait(&mut self, cx: &mut Context) -> async_channel::Receiver { + let (neighbor_tx, neighbor_rx) = async_channel::bounded(1); + self.join_neighbor_tx = Some(neighbor_tx.clone()); + self.phase = SessionPhase::Joining(JoinStage::Subscribing); + cx.spawn(async move |_, _| { + Timer::after(JOIN_FIRST_NEIGHBOR_TIMEOUT).await; + let _ = neighbor_tx.try_send(JoinNeighborSignal::TimedOut); + }) + .detach(); + cx.notify(); + neighbor_rx + } + + pub(super) fn begin_join_bootstrap( + &mut self, + inviter: PeerId, + neighbor_rx: async_channel::Receiver, + cx: &mut Context, + ) -> async_channel::Receiver> { + let (result_tx, result_rx) = async_channel::bounded(1); + let session_id = self.session; + + cx.spawn(async move |session, cx| { + match neighbor_rx.recv().await { + Ok(JoinNeighborSignal::NeighborUp) => {}, + Ok(JoinNeighborSignal::TimedOut) | Err(_) => { + let detail = "Couldn't reach anyone in this session. Make sure the inviter is online and the invite is current, then try again.".to_string(); + let _ = session.update(cx, |session, cx| { + session.detach(DetachReason::JoinFailed(detail.clone()), cx); + }); + let _ = result_tx.try_send(Err(anyhow!(detail))); + return; + }, + } + + let reply_rx = match session.update(cx, |session, cx| session.start_join_snapshot_pull(inviter, result_tx.clone(), cx)) { + Ok(Ok(reply_rx)) => reply_rx, + Ok(Err(_)) => return, + Err(error) => { + let detail = format!("collaboration join session disappeared: {error}"); + let _ = result_tx.try_send(Err(anyhow!(detail))); + return; + }, + }; + + let joined = match reply_rx.recv().await { + Ok(Ok(bytes)) => { + tracing::info!(session = %session_id, snapshot_bytes = bytes.len(), "collaboration join snapshot pulled"); + session + .update(cx, |session, cx| match session.finish_join_snapshot(&bytes, cx) { + Ok(joined) => Ok(joined), + Err(error) => { + tracing::error!(session = %session.session, error = %format_args!("{error:#}"), "building collaboration document from snapshot failed"); + let detail = format!("building collaboration document failed: {error:#}"); + session.detach(DetachReason::JoinFailed(detail), cx); + Err(error.context("building collaboration document failed")) + }, + }) + .unwrap_or_else(|error| { + tracing::error!(error = %error, "collaboration join session disappeared while building snapshot"); + Err(anyhow!("collaboration join session disappeared: {error}")) + }) + }, + Ok(Err(error)) => { + tracing::error!(session = %session_id, error = %format_args!("{error:#}"), "pulling collaboration join snapshot failed"); + let detail = format!("pulling collaboration snapshot failed: {error:#}"); + let _ = session.update(cx, |session, cx| { + session.detach(DetachReason::JoinFailed(detail.clone()), cx); + }); + Err(anyhow!(detail)) + }, + Err(error) => { + tracing::error!(session = %session_id, error = %error, "collaboration join snapshot reply channel closed"); + let detail = format!("collaboration snapshot reply channel closed: {error}"); + let _ = session.update(cx, |session, cx| { + session.detach(DetachReason::JoinFailed(detail.clone()), cx); + }); + Err(anyhow!(detail)) + }, + }; + let _ = result_tx.try_send(joined); + }) + .detach(); + + result_rx + } + + fn start_join_snapshot_pull( + &mut self, + inviter: PeerId, + result_tx: async_channel::Sender>, + cx: &mut Context, + ) -> Result>>> { + self.join_neighbor_tx = None; + self.phase = SessionPhase::Joining(JoinStage::FetchingSnapshot { got: 0, total: None }); + cx.notify(); + + let (reply_tx, reply_rx) = async_channel::bounded(1); + let (progress_tx, progress_rx) = async_channel::bounded(8); + let candidates = self.pull_candidates(Some(inviter)); + tracing::info!(session = %self.session, inviter = %inviter, candidate_count = candidates.len(), "requesting collaboration join snapshot"); + if let Err(error) = self.net_tx.try_send(NetCommand::PullSnapshot { + session: self.session, + candidates, + progress: Some(progress_tx), + reply: reply_tx, + }) { + tracing::error!(session = %self.session, inviter = %inviter, error = %error, "queueing collaboration join snapshot pull failed"); + let detail = format!("requesting collaboration snapshot failed: {error}"); + self.detach(DetachReason::JoinFailed(detail.clone()), cx); + let _ = result_tx.try_send(Err(anyhow!(detail.clone()))); + bail!(detail); + } + + cx.spawn(async move |session, cx| { + while let Ok(progress) = progress_rx.recv().await { + if session + .update(cx, |session, cx| { + if matches!(session.phase, SessionPhase::Joining(JoinStage::FetchingSnapshot { .. })) { + session.phase = SessionPhase::Joining(JoinStage::FetchingSnapshot { + got: progress.got, + total: Some(progress.total), + }); + cx.notify(); + } + }) + .is_err() + { + break; + } + } + }) + .detach(); + + cx.spawn(async move |session, cx| { + Timer::after(JOIN_SNAPSHOT_TIMEOUT).await; + let detail = "Joining this session timed out while fetching the document snapshot. Check your connection and try again.".to_string(); + let timed_out = session + .update(cx, |session, cx| { + if matches!(session.phase, SessionPhase::Joining(JoinStage::FetchingSnapshot { .. })) { + session.detach(DetachReason::JoinFailed(detail.clone()), cx); + true + } else { + false + } + }) + .unwrap_or(false); + if timed_out { + let _ = result_tx.try_send(Err(anyhow!(detail))); + } + }) + .detach(); + + Ok(reply_rx) + } + + pub fn attach_joined_editor(&mut self, panel_id: Uuid, editor: Entity, cx: &mut Context) -> Result<()> { + if self.runtime.is_none() { + tracing::warn!(session = %self.session, %panel_id, "cannot attach joined editor before snapshot load finishes"); + bail!("collaboration snapshot has not finished loading"); + } + + tracing::info!( + session = %self.session, + %panel_id, + pending_remote_updates = self.pending_remote_updates.len(), + pending_asset_records = self.pending_asset_records.len(), + "attaching joined collaboration editor", + ); + self.panel_id = Some(panel_id); + self.editor = Some(editor); + self.attach(cx); + + let pending_updates = std::mem::take(&mut self.pending_remote_updates); + self.pending_remote_update_bytes = 0; + for update in pending_updates { + // §perf: each queued update is owned; move it in to avoid re-copying. + self.import_update_bytes_owned(update, cx)?; + } + if std::mem::take(&mut self.pending_remote_updates_overflowed) + && let Some(from) = self.pull_candidates(None).first().copied() + { + tracing::info!(session = %self.session, from = %from, "starting collaboration resync pull after pre-attach queue overflow"); + // Also go through the dedup so this recovery pull registers the slot its + // `finish_pull` will later clear (never clearing a digest pull's slot). + if let GapAction::Pull { from, our_vv } = self + .anti_entropy + .begin_pull(from, self.runtime_vv.clone(), std::time::Instant::now()) + { + self.start_update_pull(from, our_vv, cx); + } + } + self.flush_pending_asset_records(cx); + asset_transfer::schedule_missing_assets(self, None, cx); + self.publish_digest(); + cx.notify(); + tracing::info!(session = %self.session, %panel_id, "joined collaboration editor attached"); + Ok(()) + } + + pub fn attach(&mut self, cx: &mut Context) { + tracing::debug!(session = %self.session, phase = ?self.phase, "attaching collaboration session hooks"); + self.attach_editor_hooks(cx); + self.attach_direct_request_pump(cx); + self.attach_timers(cx); + self.refresh_runtime_version_vector(cx); + self.refresh_comment_annotations(cx); + // Spec §6: pump once immediately so intents committed before the session + // started (the publish queue is filled by gate-held local commits + // regardless of any session existing) broadcast right away. + self.pump_publish(cx); + } + + /// JOIN handoff (spec §3): the workspace takes ownership of the document + /// services built by `finish_join_snapshot` — the write authority moves out + /// (the session must never hold a write path once attached) and the I/O + /// handle is shared. Returns `None` before the snapshot import completes or + /// after the services were already taken. + pub fn take_joined_document_services(&mut self) -> Option<(Arc, DocIoHandle)> { + let io = self.runtime.clone()?; + let authority = self.join_authority.take()?; + Some((authority, io)) + } + + /// Single fidelity-instrumented write point for the cached runtime version + /// vector. Logs every update; when `enforce_monotonic` is set (op-application + /// paths, where the vector can only advance) it also checks the new vector + /// dominates or equals the previous one. Strictly additive: the vector is + /// assigned unconditionally, so behavior is identical when tracing is off. + fn update_runtime_vv(&mut self, new_vv: Vec, source: &'static str, enforce_monotonic: bool) { + if fidelity::enabled() { + fidelity::event(FidelityClass::Frontier, "runtime-vv-update", || { + format!( + "session={} source={source} old_bytes={} new_bytes={}", + self.session, + self.runtime_vv.len(), + new_vv.len(), + ) + }); + if enforce_monotonic + && !self.runtime_vv.is_empty() + && !new_vv.is_empty() + && let (Ok(old), Ok(new)) = (loro::VersionVector::decode(&self.runtime_vv), loro::VersionVector::decode(&new_vv)) + { + let relation = new.partial_cmp(&old); + fidelity::check( + matches!(relation, Some(std::cmp::Ordering::Greater | std::cmp::Ordering::Equal)), + FidelityClass::Frontier, + "runtime-vv-regression", + || { + format!( + "session={} source={source} relation={relation:?} old_bytes={} new_bytes={}", + self.session, + self.runtime_vv.len(), + new_vv.len(), + ) + }, + ); + } + } + self.runtime_vv = new_vv; + } + + fn refresh_runtime_version_vector(&mut self, cx: &mut Context) { + let Some(runtime) = self.runtime.clone() else { + return; + }; + let session_id = self.session; + cx.spawn(async move |session, cx| { + let result = runtime.oplog_version_vector().await; + let _ = session.update(cx, |session, _| match result { + Ok(vv) => { + // Authoritative re-read of the oplog vector: enforcing monotonicity + // here would false-positive against benign async reordering with a + // concurrent op-applying commit, so log the update without the check. + session.update_runtime_vv(vv, "oplog-refresh", false); + session.publish_digest(); + }, + Err(error) => { + tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "reading collaboration runtime version vector failed"); + }, + }); + }) + .detach(); + } + + /// Re-synchronize anti-entropy state after a save/checkpoint that was applied + /// to the shared document runtime *outside* this session. + /// + /// Saving a document runs the editor's native save hook, which calls + /// `DocIoHandle::checkpoint_package`. Recording that named revision is a + /// real Loro mutation: it advances the canonical frontier/version-vector. The + /// hook returns the resulting `RuntimeEvent::LocalUpdate` bytes, but the save + /// hook future runs without any GPUI context (see `NativeSaveHook`), so it + /// cannot route them back into this session. As a result our cached + /// `runtime_vv` would go stale and peers would not learn about the new + /// revision op until the next periodic digest (~10s). + /// + /// Re-reading the authoritative oplog version vector and re-publishing a + /// digest fixes the staleness and lets anti-entropy converge immediately: + /// peers see we are ahead (`SenderHasMissingOps`) and pull the revision op on + /// the spot. We intentionally do not re-publish update bytes or re-apply a + /// projection here — the checkpoint changed no visible content, and the + /// runtime remains the single Loro owner. + pub(super) fn refresh_after_external_checkpoint(&mut self, cx: &mut Context) { + if self.runtime.is_none() { + tracing::trace!(session = %self.session, "ignoring external checkpoint refresh because the session has no runtime"); + return; + } + tracing::debug!(session = %self.session, "refreshing collaboration anti-entropy state after an external document checkpoint"); + self.refresh_runtime_version_vector(cx); + } + + pub fn establish_local_peer(&mut self, peer: &flowstate_collab::ids::PeerId, cx: &mut Context) { + if self.presence.is_none() { + tracing::info!(session = %self.session, peer = %peer, "establishing local collaboration peer presence"); + let profile = load_local_user_profile(); + let presence = PresenceStore::new_with_color(peer, profile.color_rgb); + let session = self.session; + let net_tx = self.net_tx.clone(); + self + .loro_subscriptions + .push(presence.subscribe_local_updates(move |bytes| { + let bytes_len = bytes.len(); + if let Err(error) = net_tx.try_send(NetCommand::Publish { + session, + payload: PublishPayload::Presence(bytes.clone()), + }) { + tracing::warn!(%session, bytes = bytes_len, error = %error, "queueing collaboration presence publish failed"); + } else { + tracing::trace!(%session, bytes = bytes_len, "queued collaboration presence publish from local update"); + } + true + })); + self.presence = Some(presence); + } else { + tracing::debug!(session = %self.session, peer = %peer, "local collaboration peer presence already established"); + } + if let (Some(editor), Some(presence)) = (self.editor.clone(), self.presence.as_ref()) { + editor.update(cx, |editor, cx| editor.set_own_collaboration_caret_color(Some(presence.self_color()), cx)); + } + self.refresh_own_presence(cx); + self.endpoint_online = true; + self.phase = SessionPhase::Attached(Attachment { + connectivity: Connectivity::Online, + peers_present: self.peers_present(), + }); + self.publish_presence_snapshot(); + self.publish_digest(); + asset_transfer::schedule_missing_assets(self, None, cx); + cx.notify(); + tracing::info!(session = %self.session, peer = %peer, peers_present = self.peers_present(), "local collaboration peer established"); + } + + fn finish_join_snapshot(&mut self, snapshot: &[u8], cx: &mut Context) -> Result { + tracing::info!(session = %self.session, snapshot_bytes = snapshot.len(), "building collaboration document from join snapshot"); + if matches!(self.phase, SessionPhase::Detached(_)) { + bail!("collaboration join is no longer active"); + } + let total = snapshot.len() as u64; + self.phase = SessionPhase::Joining(JoinStage::FetchingSnapshot { + got: total, + total: Some(total), + }); + cx.notify(); + self.phase = SessionPhase::Joining(JoinStage::Building); + cx.notify(); + + let doc = LoroDoc::new(); + flowstate_document::loro_schema::configure_text_styles(&doc); + doc + .import_with(snapshot, "remote") + .context("importing collaboration snapshot failed")?; + let runtime = CrdtRuntime::from_doc(doc, None, None).context("creating joined collaboration CRDT runtime")?; + let mut document = runtime + .projection_snapshot() + .context("projecting joined Loro-native document")?; + // Loro-first services (spec §3): wrap the imported core in the write gate. + // The session keeps only the transport-side I/O handle; the write + // authority is parked for the workspace handoff (join gate: the editor + // cannot receive it before this point, i.e. before the initial snapshot + // import completed). + let (authority, gate) = LocalDocHandle::new(runtime, LocalWriteConfig::default()); + let io = DocIoHandle::spawn(gate).context("starting joined collaboration document I/O service")?; + // §15/§31: bind this joiner's durable author identity to the joined + // document so their revisions record an author and `users_by_id` is + // populated. The user-registration op converges to peers via + // anti-entropy. Fire-and-forget and non-fatal: a failure must not break + // the join. `create_document_panel` receives this document through the + // `Attachment` source, which deliberately skips re-binding to avoid a + // redundant second call. + let identity_io = io.clone(); + cx.spawn(async move |session, cx| { + let (user_id, display_name) = cx + .background_executor() + .spawn(async { load_local_user_identity() }) + .await; + match identity_io.set_author_identity(user_id, display_name).await { + Ok(events) => { + let applied = session.update(cx, |session, cx| session.apply_runtime_events(events, false, cx)); + if let Ok(Err(error)) = applied { + tracing::warn!(error = %format_args!("{error:#}"), "publishing durable author identity update failed"); + } + }, + Err(error) => { + tracing::warn!(error = %format_args!("{error:#}"), "binding durable author identity to joined collaboration document failed"); + }, + } + }) + .detach(); + document.theme = load_document_theme(); + tracing::info!( + session = %self.session, + paragraphs = document.paragraphs.len(), + blocks = document.blocks.len(), + assets = document.assets.assets.len(), + "built collaboration document from join snapshot", + ); + + self.runtime = Some(io); + self.join_authority = Some(Arc::new(authority)); + fidelity::event(FidelityClass::Frontier, "runtime-vv-reset", || { + format!("session={} source=join-snapshot prior_bytes={}", self.session, self.runtime_vv.len()) + }); + self.runtime_vv.clear(); + Ok(JoinedDocument { + session: self.session, + title: format!("{} (shared)", self.title), + document, + }) + } + + pub fn detach(&mut self, reason: DetachReason, cx: &mut Context) -> bool { + if matches!(self.phase, SessionPhase::Detached(_)) { + tracing::debug!(session = %self.session, ?reason, "collaboration session already detached"); + return false; + } + + tracing::warn!(session = %self.session, ?reason, phase = ?self.phase, "detaching collaboration session"); + let user_left = matches!(reason, DetachReason::UserLeft); + let fatal_detail = match &reason { + DetachReason::Fatal(detail) => Some(detail.clone()), + DetachReason::UserLeft | DetachReason::JoinFailed(_) => None, + }; + if let Some(presence) = &self.presence { + presence.delete_self(); + self.publish_presence_bytes(presence.encode_self()); + } + if let Err(error) = self + .net_tx + .try_send(NetCommand::LeaveSession { session: self.session }) + { + tracing::warn!(session = %self.session, error = %error, "queueing collaboration leave-session command failed during detach"); + } + self.flush_pending_asset_records(cx); + + // Loro-first (invariant 5): leaving a session stops transport only. The + // document's write authority, gate, and I/O service are untouched — the + // editor keeps editing through the identical path; nothing to reset on it + // beyond collaboration presentation state. + if let Some(editor) = self.editor.clone() { + editor.update(cx, |editor, cx| { + editor.set_recovery_path(None, cx); + editor.set_collaboration_role(None, cx); + editor.set_own_collaboration_caret_color(None, cx); + editor.set_external_carets(Vec::new(), cx); + }); + } + + self.editor_subscriptions.clear(); + self.loro_subscriptions.clear(); + self.presence = None; + self.runtime = None; + self.join_authority = None; + self.runtime_vv.clear(); + self.pending_asset_records.clear(); + self.pending_remote_updates.clear(); + self.pending_remote_update_bytes = 0; + self.pending_remote_updates_overflowed = false; + self.publish_pump_pending = false; + self.neighbors.clear(); + self.asset_pulls_in_flight.clear(); + self.zero_neighbors_since = Some(Instant::now()); + self.inbound_since_last_digest = false; + self.quiet_digest_rounds = 0; + self.next_recovery_at = None; + self.awaiting_recovery_neighbor_until = None; + self.probe_pending = false; + self.last_probe_failed = false; + self.join_neighbor_tx = None; + self.presence_refresh_pending = false; + self.presence_refresh_generation = self.presence_refresh_generation.wrapping_add(1); + self.external_caret_refresh_pending = false; + self.external_caret_refresh_generation = self.external_caret_refresh_generation.wrapping_add(1); + self.phase = SessionPhase::Detached(reason); + if user_left { + cx.emit(SessionNotice::LeftSession); + } else if let Some(detail) = fatal_detail { + cx.emit(SessionNotice::Disconnected(detail)); + } + cx.notify(); + tracing::info!(session = %self.session, "collaboration session detached and cleaned up"); + true + } + + /// Drain the doc I/O publish queue (committed local Loro updates) and + /// broadcast them (spec §6 publication rule). The session never writes + /// document content — the updates were committed by the editor's write + /// authority under the gate; this is transport only (invariant 5). + pub(super) fn pump_publish(&mut self, cx: &mut Context) { + if matches!(self.phase, SessionPhase::Detached(_)) { + tracing::trace!(session = %self.session, phase = ?self.phase, "skipping local update publish pump for detached session"); + return; + } + let Some(io) = self.runtime.clone() else { + tracing::trace!(session = %self.session, "skipping local update publish pump because the document I/O service is missing"); + return; + }; + let session_id = self.session; + cx.spawn(async move |session, cx| { + match io.pump_publish().await { + Ok(events) => { + if events.is_empty() { + return; + } + let published = events.len(); + let _ = session.update(cx, |session, cx| { + fidelity::event(FidelityClass::Reconcile, "publish-pump", || { + format!("session={session_id} events={published}") + }); + if let Err(error) = session.apply_runtime_events(events, false, cx) { + tracing::error!(session = %session_id, error = %format_args!("{error:#}"), "publishing committed local collaboration updates failed"); + return; + } + session.refresh_external_carets(cx); + }); + }, + Err(error) => { + tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "pumping collaboration local update publish queue failed"); + }, + } + }) + .detach(); + } + + pub(super) fn apply_runtime_events(&mut self, events: Vec, apply_projection: bool, cx: &mut Context) -> Result<()> { + // §hang-watchdog: the main-thread handler for each incoming collab update. + let apply_started = std::time::Instant::now(); + let event_count = events.len(); + for event in events { + fidelity::event(FidelityClass::Projection, "apply-runtime-event", || { + let kind = match &event { + RuntimeEvent::LocalUpdate { .. } => "local-update", + RuntimeEvent::RemoteUpdateApplied { .. } => "remote-update-applied", + RuntimeEvent::RevisionOpened { .. } => "revision-opened", + RuntimeEvent::RevisionForked { .. } => "revision-forked", + RuntimeEvent::SelectionRestored { .. } => "selection-restored", + RuntimeEvent::ProjectionUpdated { .. } => "projection-updated", + RuntimeEvent::ProjectionPatched { .. } => "projection-patched", + RuntimeEvent::HistoryRebaseRequired { .. } => "history-rebase-required", + }; + format!("session={} kind={kind} apply_projection={apply_projection}", self.session) + }); + match event { + RuntimeEvent::LocalUpdate { bytes, version_vector, .. } => { + self.update_runtime_vv(version_vector, "local-update", true); + self.publish_update_bytes(bytes); + }, + // §A12.1.3 slice 4: a below-root remote payload was merged into the + // PACKAGE (durable, full history) but this shallow session's + // in-memory doc cannot hold it — the document must be reopened to + // show the merged state. Rare by construction; surfaced loudly. + // Follow-up (design Q4): auto-reopen instead of warn-only. + RuntimeEvent::HistoryRebaseRequired { merged_frontier } => { + tracing::warn!( + session = %self.session, + merged_frontier_len = merged_frontier.len(), + "offline collaborator history merged on disk; reopen the document to see it" + ); + }, + RuntimeEvent::RemoteUpdateApplied { pending, version_vector, .. } => { + self.update_runtime_vv(version_vector, "remote-update-applied", true); + if let Some(pending) = pending { + tracing::debug!( + session = %self.session, + pending_ranges = pending.iter().count(), + "remote collaboration update has pending Loro dependencies; requesting anti-entropy pull immediately", + ); + if let Some(from) = self.pull_candidates(None).first().copied() { + // Route through `begin_pull` (the same dedup the digest path uses) + // rather than calling `start_update_pull` directly: a burst of pending + // imports (each carrying the SAME frozen `runtime_vv` while the import + // stays buffered) must not fire a storm of identical pulls at one peer. + // begin_pull admits at most one in-flight pull per peer (with a + // deadline), and registers the slot the later `finish_pull` clears. + match self + .anti_entropy + .begin_pull(from, self.runtime_vv.clone(), std::time::Instant::now()) + { + GapAction::Pull { from, our_vv } => self.start_update_pull(from, our_vv, cx), + _ => { + tracing::debug!(session = %self.session, from = %from, "pending-dependency pull skipped; one is already in flight for this peer"); + }, + } + } else { + tracing::warn!(session = %self.session, "cannot pull pending Loro dependencies because no collaboration peers are available"); + } + } + }, + RuntimeEvent::ProjectionUpdated { + document, version_vector, .. + } if apply_projection => { + self.update_runtime_vv(version_vector, "projection-updated", true); + self.apply_runtime_projection(*document, cx)?; + }, + RuntimeEvent::ProjectionPatched { batch, version_vector, .. } if apply_projection => { + self.update_runtime_vv(version_vector, "projection-patched", true); + self.apply_runtime_patches(batch, cx)?; + }, + RuntimeEvent::RevisionOpened { document, .. } if apply_projection => { + self.apply_runtime_projection(*document, cx)?; + }, + RuntimeEvent::SelectionRestored { selection } if apply_projection => { + if let Some(editor) = self.editor.clone() { + editor.update(cx, |editor, cx| editor.restore_runtime_selection(selection, cx)); + } + }, + RuntimeEvent::RevisionForked { .. } + | RuntimeEvent::ProjectionUpdated { .. } + | RuntimeEvent::ProjectionPatched { .. } + | RuntimeEvent::RevisionOpened { .. } + | RuntimeEvent::SelectionRestored { .. } => {}, + } + } + if event_count > 0 { + self.refresh_comment_annotations(cx); + } + let apply_ms = apply_started.elapsed().as_millis(); + if apply_ms > 150 { + tracing::warn!("slow collab apply_runtime_events (hang watchdog): {apply_ms}ms, events={event_count}"); + } + Ok(()) + } + + fn apply_runtime_patches(&mut self, batch: flowstate_document::ProjectionPatchBatch, cx: &mut Context) -> Result<()> { + let Some(editor) = self.editor.clone() else { + return Ok(()); + }; + fidelity::event(FidelityClass::Projection, "apply-projection-patches", || { + format!( + "session={} patches={} base_frontier_bytes={} new_frontier_bytes={}", + self.session, + batch.patches.len(), + batch.base_frontier.len(), + batch.new_frontier.len(), + ) + }); + // Field fix 2026-07-07: remote batches reach the editor exclusively via + // the core's ORDERED projection stream — this event is only the pump. The + // payload in the event is intentionally unused (ordering lives in the + // stream, not the delivery channel). + let _ = &batch; + editor.update(cx, |editor, cx| editor.sync_projection_from_authority(cx)); + self.last_document_activity = Instant::now(); + self.refresh_external_carets(cx); + Ok(()) + } + + fn apply_runtime_projection(&mut self, mut document: DocumentProjection, cx: &mut Context) -> Result<()> { + let Some(editor) = self.editor.clone() else { + return Ok(()); + }; + fidelity::event(FidelityClass::Projection, "apply-projection", || { + format!( + "session={} paragraphs={} blocks={} frontier_bytes={}", + self.session, + document.paragraphs.len(), + document.blocks.len(), + document.frontier.len(), + ) + }); + // Field fix 2026-07-07: full replaces also ride the ordered stream; this + // event is only the pump (the sync preserves UI-cached asset bytes). + let _ = &mut document; + editor.update(cx, |editor, cx| editor.sync_projection_from_authority(cx)); + self.last_document_activity = Instant::now(); + self.refresh_external_carets(cx); + Ok(()) + } + + fn publish_update_bytes(&self, bytes: Vec) { + if bytes.is_empty() { + tracing::trace!(session = %self.session, "skipping empty collaboration update publish"); + return; + } + let bytes_len = bytes.len(); + if let Err(error) = self.net_tx.try_send(NetCommand::Publish { + session: self.session, + payload: PublishPayload::Update(bytes), + }) { + tracing::warn!(session = %self.session, bytes = bytes_len, error = %error, "queueing collaboration update publish failed"); + } else { + tracing::trace!(session = %self.session, bytes = bytes_len, "queued collaboration update publish from CRDT runtime event"); + } + } + + pub fn handle_gossip(&mut self, from: flowstate_collab::ids::PeerId, msg: GossipMsg, cx: &mut Context) { + let gossip_kind = msg.kind(); + let gossip_payload_bytes = msg.payload_len(); + tracing::trace!(session = %self.session, from = %from, gossip_kind, gossip_payload_bytes, "handling collaboration gossip message"); + self.known_peers.insert(from); + self.note_inbound_traffic(cx); + let result = match msg { + // §perf: the gossip payload is owned; move it in to avoid a full-update memcpy. + GossipMsg::Update(bytes) => self + .import_update_bytes_owned(bytes, cx) + .map(|()| asset_transfer::schedule_missing_assets(self, Some(from), cx)), + GossipMsg::UpdateAvailable { blob, len } => { + tracing::debug!(session = %self.session, from = %from, ?blob, bytes = len, "collaboration update available via direct blob pull"); + self.pull_blob(from, blob, cx); + Ok(()) + }, + GossipMsg::Presence(bytes) => { + self.apply_presence(from, &bytes, cx); + Ok(()) + }, + GossipMsg::Digest { session, vv } => self.handle_digest(from, session, &vv, cx), + }; + if let Err(error) = result { + tracing::error!(session = %self.session, from = %from, gossip_kind, error = %format_args!("{error:#}"), "collaboration gossip handling failed"); + self.detach(DetachReason::Fatal(format!("collaboration update failed: {error:#}")), cx); + } + } + + pub fn neighbor_up(&mut self, peer: flowstate_collab::ids::PeerId, cx: &mut Context) { + let inserted = self.neighbors.insert(peer); + self.known_peers.insert(peer); + self.zero_neighbors_since = None; + self.last_probe_failed = false; + if let Some(join_neighbor_tx) = self.join_neighbor_tx.take() { + let _ = join_neighbor_tx.try_send(JoinNeighborSignal::NeighborUp); + } + self.anti_entropy.on_neighbor_up(); + self.mark_online(cx); + self.publish_presence_snapshot(); + self.publish_digest(); + asset_transfer::schedule_missing_assets(self, Some(peer), cx); + cx.notify(); + tracing::info!(session = %self.session, peer = %peer, inserted, neighbors = self.neighbors.len(), "collaboration neighbor up"); + } + + pub fn neighbor_down(&mut self, peer: flowstate_collab::ids::PeerId, cx: &mut Context) { + let removed = self.neighbors.remove(&peer); + if self.neighbors.is_empty() && self.zero_neighbors_since.is_none() { + self.zero_neighbors_since = Some(Instant::now()); + } + self.evaluate_connectivity(cx); + cx.notify(); + tracing::info!(session = %self.session, peer = %peer, removed, neighbors = self.neighbors.len(), "collaboration neighbor down"); + } + + pub fn handle_gossip_lagged(&mut self, cx: &mut Context) { + tracing::warn!(session = %self.session, neighbors = self.neighbors.len(), "collaboration gossip lagged; scheduling recovery"); + self.publish_digest(); + let peer = self.neighbors.iter().next().copied(); + let vv = self.runtime_vv.clone(); + let action = self.anti_entropy.on_lagged(peer, vv, Instant::now()); + self.handle_gap_action(action, cx); + } + + pub fn set_endpoint_online(&mut self, online: bool, cx: &mut Context) { + tracing::info!(session = %self.session, online, previous_online = self.endpoint_online, "collaboration endpoint online state applied to session"); + self.endpoint_online = online; + if online { + self.last_probe_failed = false; + } + if online { + if self.peers_present() == 0 || !self.neighbors.is_empty() { + self.mark_online(cx); + } + } else { + self.evaluate_connectivity(cx); + } + cx.notify(); + } + + pub fn handle_incompatible_version(&mut self, peer: PeerId, cx: &mut Context) { + if self.incompatible_version_peers.insert(peer) { + cx.emit(SessionNotice::IncompatibleVersion(peer.to_string())); + } + } + + fn attach_editor_hooks(&mut self, cx: &mut Context) { + if !self.editor_subscriptions.is_empty() { + tracing::trace!(session = %self.session, "collaboration editor hooks already attached"); + return; + } + let Some(editor) = self.editor.clone() else { + tracing::warn!(session = %self.session, "cannot attach collaboration editor hooks because editor is missing"); + return; + }; + + tracing::debug!(session = %self.session, "attaching collaboration editor hooks"); + editor.update(cx, |editor, cx| { + if editor.document_path().is_none() { + editor.set_recovery_path(Some(presence_view::collaboration_recovery_path(self.session, &self.title)), cx); + tracing::debug!(session = %self.session, "set collaboration recovery path for untitled document"); + } + // FS-080: publish the session role so the editor's `can_write_collaboration()` + // gate blocks local edits/undo for a Viewer. Detach clears it. + // Loro-first: no capture flags, no undo redirect — the editor's write + // authority is the only write path and undo executes through it. + editor.set_collaboration_role(Some(self.collaboration_role), cx); + }); + + // Spec §6 publication rule: every editor change means intents may have + // committed under the gate; debounce, then drain the publish queue. + self + .editor_subscriptions + .push(cx.observe(&editor, |session, _, cx| { + session.schedule_publish_pump(cx); + session.flush_pending_asset_records(cx); + })); + + self + .editor_subscriptions + .push(cx.subscribe(&editor, |session, _, event: &EditorEvent, cx| { + match event { + EditorEvent::SelectionChanged { .. } => { + tracing::trace!(session = %session.session, "collaboration local selection changed; refreshing presence"); + session.schedule_own_presence_refresh(cx); + }, + EditorEvent::ReconciliationRecovery { + dropped_batches, + reason, + total_recoveries, + } => { + // A recovery means the editor's optimistic state diverged from the + // canonical projection and had to be rebuilt: a fidelity failure. + fidelity::violation(FidelityClass::Reconcile, "reconciliation-recovery", || { + format!( + "session={} dropped_batches={dropped_batches} total_recoveries={total_recoveries} reason={reason}", + session.session, + ) + }); + tracing::warn!( + session = %session.session, + dropped_batches, + total_recoveries, + %reason, + "editor reconciliation recovery; optimistic state diverged from the canonical projection", + ); + }, + _ => {}, + } + })); + tracing::debug!(session = %self.session, subscriptions = self.editor_subscriptions.len(), "collaboration editor hooks attached"); + } + + fn schedule_publish_pump(&mut self, cx: &mut Context) { + if self.publish_pump_pending { + return; + } + self.publish_pump_pending = true; + cx.spawn(async move |session, cx| { + Timer::after(Duration::from_millis(LOCAL_UPDATE_PUBLISH_DEBOUNCE_MS)).await; + let _ = session.update(cx, |session, cx| { + session.publish_pump_pending = false; + session.pump_publish(cx); + session.flush_pending_asset_records(cx); + }); + }) + .detach(); + } +} diff --git a/crates/flowstate/src/collab/session_io.rs b/crates/flowstate/src/collab/session_io.rs new file mode 100644 index 00000000..6fa508dc --- /dev/null +++ b/crates/flowstate/src/collab/session_io.rs @@ -0,0 +1,453 @@ +use anyhow::{Context as _, Result, anyhow}; +use flowstate_collab::{ + SessionId, + net::{ + NetCommand, PublishPayload, + anti_entropy::{GapAction, VersionVectorRelation}, + direct::DirectServeRequest, + }, + proto_direct::AssetBytes, +}; +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use gpui::Context; +use loro::VersionVector; + +use crate::rich_text_element::{AssetId, AssetRecord}; + +use super::{CollabSession, DetachReason}; + +pub(super) const MAX_PENDING_REMOTE_UPDATES: usize = 512; +pub(super) const MAX_PENDING_REMOTE_UPDATE_BYTES: usize = 64 * 1024 * 1024; + +impl CollabSession { + // §perf: borrowed entry point for the genuinely-borrowed callers; copies once into an + // owned buffer and forwards to import_update_bytes_owned. Owned-Vec callers should call + // import_update_bytes_owned directly to skip this full-update memcpy on the main thread. + pub fn import_update_bytes(&mut self, bytes: &[u8], cx: &mut Context) -> Result<()> { + self.import_update_bytes_owned(bytes.to_vec(), cx) + } + + pub fn import_update_bytes_owned(&mut self, bytes: Vec, cx: &mut Context) -> Result<()> { + if bytes.is_empty() { + tracing::trace!(session = %self.session, "skipping empty collaboration update import"); + return Ok(()); + } + if self.runtime.is_none() || self.editor.is_none() { + if self.pending_remote_updates.len() >= MAX_PENDING_REMOTE_UPDATES + || self.pending_remote_update_bytes.saturating_add(bytes.len()) > MAX_PENDING_REMOTE_UPDATE_BYTES + { + // Drop the whole pre-attach queue: partial update history is useless + // to Loro anyway, and the overflow flag forces one anti-entropy pull + // right after attach, which resynchronizes everything we dropped. + tracing::warn!( + session = %self.session, + dropped_updates = self.pending_remote_updates.len() + 1, + dropped_bytes = self.pending_remote_update_bytes + bytes.len(), + "pre-attach collaboration update queue overflowed; will resync via anti-entropy pull after attach", + ); + self.pending_remote_updates.clear(); + self.pending_remote_update_bytes = 0; + self.pending_remote_updates_overflowed = true; + return Ok(()); + } + tracing::debug!( + session = %self.session, + bytes = bytes.len(), + queued_updates = self.pending_remote_updates.len() + 1, + has_runtime = self.runtime.is_some(), + has_editor = self.editor.is_some(), + "queueing remote collaboration update until session is attached", + ); + self.pending_remote_update_bytes = self.pending_remote_update_bytes.saturating_add(bytes.len()); + self.pending_remote_updates.push(bytes); // §perf: move the owned buffer into the queue instead of copying + return Ok(()); + } + + tracing::debug!(session = %self.session, bytes = bytes.len(), "importing remote collaboration update"); + let io = self + .runtime + .clone() + .context("collaboration session has no document I/O service")?; + // §perf: bytes is already owned; hand it straight to the I/O service with no extra copy. + let bytes_len = bytes.len(); + let session_id = self.session; + cx.spawn(async move |session, cx| { + let result = io.import_remote_update(bytes).await; + match result { + Ok(events) => { + let applied = session.update(cx, |session, cx| session.apply_runtime_events(events, true, cx)); + let projection_error = match applied { + Ok(Ok(())) => None, + Ok(Err(error)) => Some(format!("{error:#}")), + Err(error) => { + tracing::debug!(session = %session_id, %error, "collaboration session disappeared while applying remote update"); + return; + }, + }; + if let Some(detail) = projection_error { + // Applying a remote patch batch to THE projection failed; repair + // via the spec §6 fallback: canonical `projection_snapshot()` + + // full install. Still a fidelity violation worth counting. + fidelity::violation(FidelityClass::Reconcile, "remote-projection-repair", || { + format!("session={session_id} bytes={bytes_len} error={detail}") + }); + tracing::warn!(session = %session_id, bytes = bytes_len, error = %detail, "remote patch application failed; repairing from canonical runtime snapshot"); + match io.projection_snapshot().await { + Ok(document) => { + let _ = session.update(cx, |session, cx| { + if let Err(error) = session.apply_runtime_projection(document, cx) { + session.detach( + DetachReason::Fatal(format!("applying canonical collaboration repair failed: {error:#}")), + cx, + ); + } + }); + }, + Err(error) => { + let _ = session.update(cx, |session, cx| { + session.detach( + DetachReason::Fatal(format!("fetching canonical collaboration repair failed: {error:#}")), + cx, + ); + }); + }, + } + } else { + tracing::debug!(session = %session_id, bytes = bytes_len, "remote collaboration update imported and projected"); + } + }, + Err(error) => { + // A malformed or unimportable update is dropped, not session-fatal: + // real content re-arrives via digest-driven anti-entropy pulls, and a + // dead I/O service surfaces through the publish pump instead. + tracing::error!(session = %session_id, bytes = bytes_len, error = %format_args!("{error:#}"), "dropped unimportable remote collaboration update"); + }, + } + }) + .detach(); + Ok(()) + } + + pub(super) fn attach_direct_request_pump(&mut self, cx: &mut Context) { + if self.direct_pump_started { + tracing::trace!(session = %self.session, "collaboration direct request pump already started"); + return; + } + self.direct_pump_started = true; + let requests = self.direct_rx.clone(); + let session_id = self.session; + tracing::debug!(session = %session_id, "starting collaboration direct request pump"); + cx.spawn(async move |session, cx| { + while let Ok(request) = requests.recv().await { + tracing::trace!(session = %session_id, request_kind = direct_serve_request_kind(&request), "received collaboration direct serve request"); + if session + .update(cx, |session, cx| session.handle_direct_request(request, cx)) + .is_err() + { + tracing::debug!(session = %session_id, "collaboration direct request pump session disappeared"); + break; + } + } + tracing::debug!(session = %session_id, "collaboration direct request pump stopped"); + }) + .detach(); + } + + pub(super) fn handle_digest( + &mut self, + from: flowstate_collab::ids::PeerId, + digest_session: SessionId, + vv: &[u8], + cx: &mut Context, + ) -> Result<()> { + if self.runtime.is_none() { + tracing::debug!(session = %self.session, from = %from, digest_session = %digest_session, vv_bytes = vv.len(), "ignored collaboration digest because Loro doc is missing"); + return Ok(()); + } + let sender_vv = match VersionVector::decode(vv).context("decoding collaboration digest failed") { + Ok(sender_vv) => sender_vv, + Err(error) => { + // A malformed digest is the sender's defect, never grounds to detach + // this healthy session; drop the frame and keep serving other peers. + tracing::warn!(session = %self.session, from = %from, digest_session = %digest_session, vv_bytes = vv.len(), error = %format_args!("{error:#}"), "ignored undecodable collaboration digest from peer"); + return Ok(()); + }, + }; + if self.runtime_vv.is_empty() { + tracing::debug!(session = %self.session, from = %from, "ignored collaboration digest until local version vector is initialized"); + return Ok(()); + } + let our_vv = VersionVector::decode(&self.runtime_vv).context("decoding local collaboration version vector failed")?; + let relation = match sender_vv.partial_cmp(&our_vv) { + Some(std::cmp::Ordering::Equal) => VersionVectorRelation::Equal, + Some(std::cmp::Ordering::Greater) => VersionVectorRelation::SenderHasMissingOps, + Some(std::cmp::Ordering::Less) => VersionVectorRelation::WeHaveMissingOps, + None => VersionVectorRelation::Concurrent, + }; + let action = self + .anti_entropy + .consider_digest(from, digest_session, relation, self.runtime_vv.clone(), std::time::Instant::now()); + tracing::trace!( + session = %self.session, + from = %from, + digest_session = %digest_session, + vv_bytes = vv.len(), + ?relation, + action = gap_action_kind(&action), + "handled collaboration digest", + ); + self.handle_gap_action(action, cx); + Ok(()) + } + + pub(super) fn pull_blob(&mut self, from: flowstate_collab::ids::PeerId, blob: flowstate_collab::BlobId, cx: &mut Context) { + let (reply_tx, reply_rx) = async_channel::bounded(1); + let candidates = self.pull_candidates(Some(from)); + tracing::debug!(session = %self.session, from = %from, ?blob, candidate_count = candidates.len(), "requesting collaboration update blob pull"); + if let Err(error) = self.net_tx.try_send(NetCommand::PullBlob { + session: self.session, + candidates, + blob, + reply: reply_tx, + }) { + tracing::warn!(session = %self.session, from = %from, ?blob, error = %error, "queueing collaboration blob pull failed"); + return; + } + let session_id = self.session; + cx.spawn(async move |session, cx| { + let result = reply_rx.recv().await; + let _ = session.update(cx, |session, cx| match result { + Ok(Ok(bytes)) => { + tracing::debug!(session = %session_id, ?blob, bytes = bytes.len(), "collaboration blob pull succeeded"); + // §perf: bytes is owned here; move it in to avoid a full-update memcpy. + if let Err(error) = session.import_update_bytes_owned(bytes, cx) { + tracing::error!(session = %session_id, ?blob, error = %format_args!("{error:#}"), "importing pulled collaboration blob failed"); + session.detach(DetachReason::Fatal(format!("pulling collaboration blob failed: {error:#}")), cx); + } + }, + Ok(Err(error)) => tracing::warn!(session = %session_id, ?blob, error = %format_args!("{error:#}"), "collaboration blob pull failed"), + Err(error) => tracing::warn!(session = %session_id, ?blob, error = %error, "collaboration blob pull reply channel closed"), + }); + }) + .detach(); + } + + pub(super) fn publish_digest(&self) { + if self.runtime.is_some() { + let vv = self.runtime_vv.clone(); + if vv.is_empty() { + tracing::trace!(session = %self.session, "skipping collaboration digest publish until runtime version vector is initialized"); + return; + } + let vv_bytes = vv.len(); + if let Err(error) = self.net_tx.try_send(NetCommand::Publish { + session: self.session, + payload: PublishPayload::Digest { vv }, + }) { + tracing::warn!(session = %self.session, vv_bytes, error = %error, "queueing collaboration digest publish failed"); + } else { + tracing::trace!(session = %self.session, vv_bytes, "queued collaboration digest publish"); + } + } else { + tracing::trace!(session = %self.session, "skipping collaboration digest publish because Loro doc is missing"); + } + } + + pub(super) fn flush_pending_asset_records(&mut self, cx: &mut Context) -> bool { + let Some(editor) = self.editor.clone() else { + tracing::trace!(session = %self.session, pending_asset_records = self.pending_asset_records.len(), "cannot flush collaboration asset records because editor is missing"); + return false; + }; + let deferred = editor.read(cx).projection_apply_deferred(); + if self.pending_asset_records.is_empty() || deferred { + tracing::trace!( + session = %self.session, + pending_asset_records = self.pending_asset_records.len(), + deferred, + "collaboration asset record flush skipped", + ); + return false; + } + let asset_records = std::mem::take(&mut self.pending_asset_records); + tracing::debug!(session = %self.session, asset_records = asset_records.len(), "flushing collaboration asset records to editor"); + editor.update(cx, |editor, cx| { + editor.apply_synced_asset_records(&asset_records, cx); + }); + self.last_document_activity = std::time::Instant::now(); + self.refresh_external_carets(cx); + tracing::debug!(session = %self.session, asset_records = asset_records.len(), "collaboration asset records flushed to editor"); + true + } + + pub(super) fn handle_gap_action(&mut self, action: GapAction, cx: &mut Context) { + match action { + GapAction::None => {}, + GapAction::Pull { from, our_vv } => { + tracing::debug!(session = %self.session, from = %from, our_vv_bytes = our_vv.len(), "collaboration gap action requested update pull"); + self.start_update_pull(from, our_vv, cx); + }, + GapAction::LineageMismatch { from, expected, got } => { + tracing::warn!(session = %self.session, from = %from, expected = %expected, got = %got, "ignored mismatched collaboration digest"); + }, + } + } + + fn handle_direct_request(&mut self, request: DirectServeRequest, cx: &mut Context) { + tracing::trace!(session = %self.session, request_kind = direct_serve_request_kind(&request), "serving collaboration direct request from session"); + match request { + DirectServeRequest::Snapshot { reply } => { + let runtime = self.runtime.clone(); + let session_id = self.session; + cx.spawn(async move |_, _| { + let result = match runtime { + Some(runtime) => runtime.snapshot_bytes().await, + None => Err(anyhow!("collaboration session is not attached")), + }; + log_direct_serve_result(session_id, "snapshot", &result); + let _ = reply.send(result).await; + }) + .detach(); + }, + DirectServeRequest::Updates { have_vv, reply } => { + tracing::trace!(session = %self.session, have_vv_bytes = have_vv.len(), "serving collaboration updates request"); + let runtime = self.runtime.clone(); + let session_id = self.session; + cx.spawn(async move |_, _| { + let result = match runtime { + Some(runtime) => runtime.export_updates_for(have_vv).await, + None => Err(anyhow!("collaboration session is not attached")), + }; + log_direct_serve_result(session_id, "updates", &result); + let _ = reply.send(result).await; + }) + .detach(); + }, + DirectServeRequest::Asset { asset, reply } => { + let result = self.asset_bytes(asset, cx); + match &result { + Ok(bytes) => tracing::debug!(session = %self.session, asset, bytes = bytes.bytes.len(), "served collaboration asset direct request"), + Err(error) => { + tracing::warn!(session = %self.session, asset, error = %format_args!("{error:#}"), "serving collaboration asset direct request failed"); + }, + } + let _ = reply.try_send(result); + }, + } + } + + fn asset_bytes(&self, asset: u128, cx: &mut Context) -> Result { + let editor = self + .editor + .as_ref() + .context("collaboration session has no editor")?; + let bytes = editor + .read(cx) + .document() + .assets + .assets + .get(&AssetId(asset)) + .map(|record| record.bytes.as_ref().clone()) + .ok_or_else(|| anyhow!("collaboration asset {asset} is not available"))?; + tracing::debug!(session = %self.session, asset, bytes = bytes.len(), "exported collaboration asset bytes"); + Ok(AssetBytes { bytes }) + } + + pub(super) fn queue_asset_records(&mut self, mut asset_records: Vec<(AssetId, AssetRecord)>, cx: &mut Context) { + if asset_records.is_empty() { + tracing::trace!(session = %self.session, "no collaboration asset records to queue"); + return; + } + for (id, record) in &asset_records { + trace_asset_record(self.session, *id, record); + } + // Loro-first: fetched asset bytes flow to the editor's UI cache below AND + // are recorded into canonical state through the I/O service so this + // replica's own saves/packages include them. The session still never + // writes document CONTENT (invariant 5) — asset bytes are content- + // addressed sideband records. + if let Some(io) = self.runtime.clone() { + let records: Vec = asset_records + .iter() + .map(|(_, record)| record.clone()) + .collect(); + cx.spawn(async move |_, _| { + if let Err(error) = io.record_assets(records).await { + tracing::warn!(%error, "recording pulled asset bytes into canonical state failed; saves may miss them until re-pull"); + } + }) + .detach(); + } + tracing::debug!(session = %self.session, asset_records = asset_records.len(), pending_before = self.pending_asset_records.len(), "queueing collaboration asset records"); + self.pending_asset_records.append(&mut asset_records); + let flushed = self.flush_pending_asset_records(cx); + tracing::trace!(session = %self.session, pending_after = self.pending_asset_records.len(), flushed, "collaboration asset record queue updated"); + } + + pub(super) fn start_update_pull(&mut self, from: flowstate_collab::ids::PeerId, our_vv: Vec, cx: &mut Context) { + let (reply_tx, reply_rx) = async_channel::bounded(1); + let candidates = self.pull_candidates(Some(from)); + tracing::debug!(session = %self.session, from = %from, our_vv_bytes = our_vv.len(), candidate_count = candidates.len(), "requesting collaboration update pull"); + let send_result = self.net_tx.try_send(NetCommand::PullUpdates { + session: self.session, + candidates, + our_vv, + reply: reply_tx, + }); + if let Err(error) = send_result { + tracing::warn!(session = %self.session, from = %from, error = %error, "queueing collaboration update pull failed"); + self.anti_entropy.finish_pull(from); + return; + } + let session_id = self.session; + cx.spawn(async move |session, cx| { + let result = reply_rx.recv().await; + let _ = session.update(cx, |session, cx| { + session.anti_entropy.finish_pull(from); + match result { + Ok(Ok(bytes)) => { + tracing::debug!(session = %session_id, from = %from, bytes = bytes.len(), "collaboration update pull succeeded"); + if bytes.is_empty() { + tracing::trace!(session = %session_id, from = %from, "collaboration update pull returned no missing updates"); + return; + } + // §perf: bytes is owned here; move it in to avoid a full-update memcpy. + if let Err(error) = session.import_update_bytes_owned(bytes, cx) { + tracing::error!(session = %session_id, from = %from, error = %format_args!("{error:#}"), "importing pulled collaboration updates failed"); + session.detach(DetachReason::Fatal(format!("pulling collaboration updates failed: {error:#}")), cx); + } + }, + Ok(Err(error)) => tracing::warn!(session = %session_id, from = %from, error = %format_args!("{error:#}"), "collaboration update pull failed"), + Err(error) => tracing::warn!(session = %session_id, from = %from, error = %error, "collaboration update pull reply channel closed"), + } + }); + }) + .detach(); + } +} + +fn direct_serve_request_kind(request: &DirectServeRequest) -> &'static str { + match request { + DirectServeRequest::Snapshot { .. } => "snapshot", + DirectServeRequest::Updates { .. } => "updates", + DirectServeRequest::Asset { .. } => "asset", + } +} + +fn gap_action_kind(action: &GapAction) -> &'static str { + match action { + GapAction::None => "none", + GapAction::Pull { .. } => "pull", + GapAction::LineageMismatch { .. } => "lineage_mismatch", + } +} + +fn log_direct_serve_result(session: SessionId, kind: &'static str, result: &Result>) { + match result { + Ok(bytes) => tracing::debug!(%session, kind, bytes = bytes.len(), "served collaboration direct payload"), + Err(error) => tracing::warn!(%session, kind, error = %format_args!("{error:#}"), "serving collaboration direct payload failed"), + } +} + +fn trace_asset_record(session: SessionId, id: AssetId, record: &AssetRecord) { + tracing::trace!(%session, ?id, bytes = record.bytes.len(), placeholder = record.is_loading_placeholder(), "queued collaboration asset record"); +} diff --git a/crates/flowstate/src/collab/session_presence.rs b/crates/flowstate/src/collab/session_presence.rs new file mode 100644 index 00000000..9b814eed --- /dev/null +++ b/crates/flowstate/src/collab/session_presence.rs @@ -0,0 +1,384 @@ +use std::{collections::HashMap, time::Duration}; + +use flowstate_collab::{ + crdt_runtime::RuntimePresenceCaretRequest, + ids::PeerId, + presence::{PresenceState, PresenceStore}, +}; +use flowstate_fidelity::{self as fidelity, FidelityClass}; +use gpui::{Context, Timer}; + +use super::{CollabSession, SessionNotice}; + +const PRESENCE_REFRESH_DEBOUNCE: Duration = Duration::from_millis(50); +const EXTERNAL_CARET_REFRESH_DEBOUNCE: Duration = Duration::from_millis(16); + +impl CollabSession { + pub(super) fn apply_presence(&mut self, from: PeerId, bytes: &[u8], cx: &mut Context) { + if let Some(presence) = &self.presence { + tracing::trace!(session = %self.session, from = %from, bytes = bytes.len(), "applying remote collaboration presence update"); + let before = remote_roster(presence); + let before_keys = if fidelity::enabled() { + roster_key_set(presence) + } else { + std::collections::HashSet::new() + }; + if let Err(error) = presence.apply_from(&from, bytes) { + fidelity::event(FidelityClass::Presence, "presence-reject", || { + format!("session={} from={from} bytes={} error={error:#}", self.session, bytes.len()) + }); + tracing::warn!(session = %self.session, bytes = bytes.len(), error = %format_args!("{error:#}"), "collaboration presence update failed"); + return; + } + if fidelity::enabled() { + // Presence is a per-peer, self-authored signal: `apply_from` only accepts + // a frame that touches the delivering peer's own key. Any other key that + // changed across the apply means a frame bound to the wrong peer slipped + // through — an impersonation/misroute the trust gate should have dropped. + let expected_key = flowstate_collab::presence::peer_key(&from); + let after_keys = roster_key_set(presence); + let offending: Vec = after_keys + .symmetric_difference(&before_keys) + .filter(|key| **key != expected_key) + .cloned() + .collect(); + fidelity::check(offending.is_empty(), FidelityClass::Presence, "presence-key-unbound", || { + format!( + "session={} from={from} expected_key={expected_key} offending_keys={offending:?}", + self.session + ) + }); + fidelity::event(FidelityClass::Presence, "presence-apply", || { + format!( + "session={} from={from} bytes={} roster_keys={}", + self.session, + bytes.len(), + after_keys.len() + ) + }); + } + let roster_changed = self.emit_presence_roster_diff(before, cx); + let peer_count_changed = self.refresh_peer_count(); + self.evaluate_connectivity(cx); + self.refresh_external_carets(cx); + if roster_changed || peer_count_changed { + cx.notify(); + } + } else { + tracing::debug!(session = %self.session, bytes = bytes.len(), "ignored collaboration presence update before local presence was established"); + } + } + + pub(super) fn refresh_own_presence(&mut self, cx: &mut Context) { + if self.presence.is_none() { + tracing::trace!(session = %self.session, "skipping own collaboration presence refresh because store is missing"); + return; + } + let selection = self + .editor + .as_ref() + .map(|editor| editor.read(cx).selection().clone()); + let runtime = self.runtime.clone(); + let session_id = self.session; + self.presence_refresh_generation = self.presence_refresh_generation.wrapping_add(1); + let generation = self.presence_refresh_generation; + cx.spawn(async move |session, cx| { + let selection = match (runtime, selection) { + (Some(runtime), Some(selection)) => runtime.presence_selection(selection).await, + _ => Ok(None), + }; + let _ = session.update(cx, |session, cx| match selection { + Ok(selection) => { + if session.presence_refresh_generation != generation { + tracing::trace!(session = %session_id, generation, current_generation = session.presence_refresh_generation, "skipping stale own collaboration presence refresh"); + return; + } + let state = PresenceState { + name: default_presence_name(), + color_rgb: crate::app_settings::load_local_user_profile().color_rgb, + selection, + }; + tracing::trace!(session = %session_id, has_selection = state.selection.is_some(), "refreshing own collaboration presence"); + if let Some(presence) = &session.presence + && let Err(error) = presence.set_self(&state) + { + tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "collaboration presence encode failed"); + } + if session.refresh_peer_count() { + cx.notify(); + } + }, + Err(error) => { + tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "resolving own collaboration selection failed"); + }, + }); + }) + .detach(); + } + + pub(super) fn schedule_own_presence_refresh(&mut self, cx: &mut Context) { + if self.presence.is_none() || self.presence_refresh_pending { + return; + } + self.presence_refresh_pending = true; + cx.spawn(async move |session, cx| { + Timer::after(PRESENCE_REFRESH_DEBOUNCE).await; + let _ = session.update(cx, |session, cx| { + session.presence_refresh_pending = false; + session.refresh_own_presence(cx); + }); + }) + .detach(); + } + + pub(super) fn remove_outdated_presence(&mut self, cx: &mut Context) -> bool { + let Some(presence) = &self.presence else { + return false; + }; + let before = remote_roster(presence); + presence.remove_outdated(); + self.emit_presence_roster_diff(before, cx) + } + + pub(super) fn refresh_external_carets(&mut self, cx: &mut Context) { + if self.external_caret_refresh_pending { + return; + } + self.external_caret_refresh_pending = true; + cx.spawn(async move |session, cx| { + Timer::after(EXTERNAL_CARET_REFRESH_DEBOUNCE).await; + let _ = session.update(cx, |session, cx| { + session.external_caret_refresh_pending = false; + session.refresh_external_carets_now(cx); + }); + }) + .detach(); + } + + pub(super) fn refresh_comment_annotations(&mut self, cx: &mut Context) { + if self.comment_annotation_refresh_pending { + return; + } + let Some(runtime) = self.runtime.clone() else { return }; + let Some(editor) = self.editor.clone() else { return }; + self.comment_annotation_refresh_pending = true; + self.comment_annotation_refresh_generation = self.comment_annotation_refresh_generation.wrapping_add(1); + let generation = self.comment_annotation_refresh_generation; + let session_id = self.session; + cx.spawn(async move |session, cx| { + Timer::after(EXTERNAL_CARET_REFRESH_DEBOUNCE).await; + let result = runtime.comments().await; + let _ = session.update(cx, |session, cx| { + if session.comment_annotation_refresh_generation != generation { + return; + } + session.comment_annotation_refresh_pending = false; + match result { + Ok(comments) + if session + .editor + .as_ref() + .is_some_and(|current| current == &editor) => + { + let selections = comments + .into_iter() + .filter(|thread| !thread.resolved) + .filter_map(|thread| thread.anchor) + .map(|(start, end)| crate::rich_text_element::ExternalSelection { + selection: crate::rich_text_element::EditorSelection::range(start, end), + color_rgb: 0xd99a20, + }) + .collect(); + editor.update(cx, |editor, cx| editor.set_annotation_selections(selections, cx)); + }, + Ok(_) => {}, + Err(error) => tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "refreshing comment annotations failed"), + } + }); + }) + .detach(); + } + + fn refresh_external_carets_now(&mut self, cx: &mut Context) { + let Some(presence) = &self.presence else { + tracing::trace!(session = %self.session, "skipping external caret refresh because presence is missing"); + return; + }; + let Some(runtime) = self.runtime.clone() else { + tracing::trace!(session = %self.session, "skipping external caret refresh because Loro doc is missing"); + return; + }; + let Some(editor) = self.editor.clone() else { + tracing::trace!(session = %self.session, "skipping external caret refresh because editor is missing"); + return; + }; + // §perf: self_key is only used for equality; borrow it instead of allocating a String. + let self_key = presence.self_key(); + let requests: Vec = presence + .roster() + .into_iter() + .filter(|entry| entry.key != self_key) + .filter_map(|entry| { + entry + .selection + .map(|selection| RuntimePresenceCaretRequest { + selection, + color_rgb: entry.color_rgb, + }) + }) + .collect(); + let session_id = self.session; + self.external_caret_refresh_generation = self.external_caret_refresh_generation.wrapping_add(1); + let generation = self.external_caret_refresh_generation; + if requests.is_empty() { + tracing::trace!(session = %self.session, "clearing collaboration external carets because no remote selections are present"); + editor.update(cx, |editor, cx| { + editor.set_external_carets(Vec::new(), cx); + editor.set_external_selections(Vec::new(), cx); + }); + return; + } + cx.spawn(async move |session, cx| { + let result = runtime.resolve_presence_carets(requests).await; + let _ = session.update(cx, |session, cx| match result { + Ok(resolved) => { + if session.external_caret_refresh_generation != generation { + tracing::trace!(session = %session_id, generation, current_generation = session.external_caret_refresh_generation, "skipping stale collaboration external caret refresh"); + return; + } + tracing::trace!(session = %session_id, carets = resolved.carets.len(), "refreshing collaboration external carets"); + if session + .editor + .as_ref() + .is_some_and(|current| current == &editor) + { + if fidelity::enabled() { + // A resolved external caret must land inside the editor's current + // projection; a paragraph index past the end means the resolution + // targeted a document the local editor no longer reflects. + let paragraph_count = editor.read(cx).document().paragraphs.len(); + for caret in &resolved.carets { + fidelity::check( + caret.offset.paragraph < paragraph_count, + FidelityClass::Presence, + "external-caret-invalid-offset", + || { + format!( + "session={session_id} paragraph={} byte={} paragraphs={paragraph_count}", + caret.offset.paragraph, caret.offset.byte, + ) + }, + ); + } + fidelity::event(FidelityClass::Presence, "external-carets-resolved", || { + format!("session={session_id} carets={} paragraphs={paragraph_count}", resolved.carets.len()) + }); + } + editor.update(cx, |editor, cx| { + editor.set_external_selections(resolved.selections, cx); + editor.set_external_carets(resolved.carets, cx); + }); + } + }, + Err(error) => { + tracing::warn!(session = %session_id, error = %format_args!("{error:#}"), "resolving collaboration external carets failed"); + }, + }); + }) + .detach(); + } + + pub(super) fn publish_presence_snapshot(&self) { + if let Some(presence) = &self.presence { + let bytes = presence.encode_self(); + tracing::debug!(session = %self.session, bytes = bytes.len(), "publishing own collaboration presence snapshot"); + self.publish_presence_bytes(bytes); + } else { + tracing::trace!(session = %self.session, "skipping collaboration presence snapshot because store is missing"); + } + } + + pub(super) fn publish_presence_bytes(&self, bytes: Vec) { + if bytes.is_empty() { + tracing::trace!(session = %self.session, "skipping empty collaboration presence publish"); + return; + } + let byte_len = bytes.len(); + if let Err(error) = self + .net_tx + .try_send(flowstate_collab::net::NetCommand::Publish { + session: self.session, + payload: flowstate_collab::net::PublishPayload::Presence(bytes), + }) + { + tracing::warn!(session = %self.session, bytes = byte_len, error = %error, "queueing collaboration presence publish failed"); + } else { + tracing::trace!(session = %self.session, bytes = byte_len, "queued collaboration presence publish"); + } + } + + pub(super) fn refresh_peer_count(&mut self) -> bool { + let peers_present = self.peers_present(); + let session_id = self.session; + if let super::SessionPhase::Attached(attachment) = &mut self.phase { + let previous = attachment.peers_present; + attachment.peers_present = peers_present; + if previous != peers_present { + tracing::info!(session = %session_id, previous, peers_present, "collaboration peer presence count changed"); + return true; + } + } + false + } + + pub(super) fn peers_present(&self) -> usize { + self + .presence + .as_ref() + .map_or(0, |presence| presence.roster().len().saturating_sub(1)) + } + + fn emit_presence_roster_diff(&mut self, before: HashMap, cx: &mut Context) -> bool { + let Some(presence) = &self.presence else { + return false; + }; + let after = remote_roster(presence); + let changed = before != after; + for (key, name) in &after { + if !before.contains_key(key) { + cx.emit(SessionNotice::PeerJoined(name.clone())); + } + } + for (key, name) in before { + if !after.contains_key(&key) { + cx.emit(SessionNotice::PeerLeft(name)); + } + } + changed + } +} + +fn remote_roster(presence: &PresenceStore) -> HashMap { + let self_key = presence.self_key().to_string(); + presence + .roster() + .into_iter() + .filter(|entry| entry.key != self_key) + .map(|entry| (entry.key, entry.name)) + .collect() +} + +/// Full set of roster keys (including this endpoint's own). Used only by the +/// fidelity presence-binding invariant to detect a frame that mutated a key +/// other than the delivering peer's own. +fn roster_key_set(presence: &PresenceStore) -> std::collections::HashSet { + presence + .roster() + .into_iter() + .map(|entry| entry.key) + .collect() +} + +fn default_presence_name() -> String { + crate::app_settings::load_local_user_profile().display_name +} diff --git a/crates/flowstate/src/collab/session_timers.rs b/crates/flowstate/src/collab/session_timers.rs new file mode 100644 index 00000000..3f9c75d7 --- /dev/null +++ b/crates/flowstate/src/collab/session_timers.rs @@ -0,0 +1,309 @@ +use std::time::{Duration, Instant}; + +use flowstate_collab::{net::NetCommand, presence::PRESENCE_KEEPALIVE_SECS}; +use gpui::{Context, Timer}; + +use super::{Attachment, CollabSession, Connectivity, DetachReason, SessionPhase}; + +const ZERO_NEIGHBOR_OFFLINE_GRACE: Duration = Duration::from_secs(5); +const QUIET_DIGEST_ROUNDS_OFFLINE: u8 = 2; +const RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30); +const RECOVERY_NEIGHBOR_WAIT: Duration = Duration::from_secs(10); + +impl CollabSession { + pub(super) fn attach_timers(&mut self, cx: &mut Context) { + if self.timers_started { + tracing::trace!(session = %self.session, "collaboration timers already started"); + return; + } + self.timers_started = true; + tracing::debug!(session = %self.session, "starting collaboration timers"); + self.spawn_digest_timer(cx); + self.spawn_presence_timer(cx); + } + + pub(super) fn note_inbound_traffic(&mut self, cx: &mut Context) { + self.inbound_since_last_digest = true; + self.quiet_digest_rounds = 0; + tracing::trace!(session = %self.session, "noted inbound collaboration network traffic"); + if self.mark_online(cx) { + self.anti_entropy.on_reconnect(); + self.publish_digest(); + } + } + + pub(super) fn mark_online(&mut self, cx: &mut Context) -> bool { + if self + .awaiting_recovery_neighbor_until + .is_some_and(|deadline| self.peers_present() > 0 && self.neighbors.is_empty() && Instant::now() < deadline) + { + return false; + } + + let mut changed = false; + if let SessionPhase::Attached(attachment) = &mut self.phase + && matches!(attachment.connectivity, Connectivity::Offline { .. }) + { + attachment.connectivity = Connectivity::Online; + self.next_recovery_at = None; + self.awaiting_recovery_neighbor_until = None; + self.probe_pending = false; + self.last_probe_failed = false; + changed = true; + } + if changed { + tracing::info!(session = %self.session, "collaboration session marked online"); + cx.notify(); + } + changed + } + + pub(super) fn evaluate_connectivity(&mut self, cx: &mut Context) { + if !matches!(self.phase, SessionPhase::Attached(_)) { + tracing::trace!(session = %self.session, phase = ?self.phase, "skipping collaboration connectivity evaluation for inactive phase"); + return; + } + + self.refresh_peer_count(); + if self.peers_present() == 0 || !self.neighbors.is_empty() { + tracing::trace!(session = %self.session, peers_present = self.peers_present(), neighbors = self.neighbors.len(), "collaboration connectivity considered online"); + self.mark_online(cx); + return; + } + + let now = Instant::now(); + let zero_since = *self.zero_neighbors_since.get_or_insert(now); + let zero_long_enough = now.saturating_duration_since(zero_since) >= ZERO_NEIGHBOR_OFFLINE_GRACE; + let quiet_enough = self.quiet_digest_rounds >= QUIET_DIGEST_ROUNDS_OFFLINE; + tracing::trace!( + session = %self.session, + peers_present = self.peers_present(), + neighbors = self.neighbors.len(), + quiet_rounds = self.quiet_digest_rounds, + quiet_enough, + zero_long_enough, + endpoint_online = self.endpoint_online, + last_probe_failed = self.last_probe_failed, + "evaluating collaboration connectivity", + ); + if zero_long_enough && quiet_enough && (!self.endpoint_online || self.last_probe_failed || !self.start_connectivity_probe(cx)) { + self.mark_offline(now, cx); + } + } + + fn spawn_digest_timer(&mut self, cx: &mut Context) { + let session_id = self.session; + tracing::debug!(session = %session_id, "starting collaboration digest timer"); + cx.spawn(async move |session, cx| { + loop { + let delay = match session.update(cx, |session, _| session.next_digest_delay()) { + Ok(Some(delay)) => delay, + Ok(None) | Err(_) => break, + }; + Timer::after(delay).await; + match session.update(cx, |session, cx| session.run_digest_tick(cx)) { + Ok(true) => {}, + Ok(false) | Err(_) => break, + } + } + tracing::debug!(session = %session_id, "collaboration digest timer stopped"); + }) + .detach(); + } + + fn spawn_presence_timer(&mut self, cx: &mut Context) { + let session_id = self.session; + tracing::debug!(session = %session_id, "starting collaboration presence timer"); + cx.spawn(async move |session, cx| { + loop { + Timer::after(Duration::from_secs(PRESENCE_KEEPALIVE_SECS)).await; + match session.update(cx, |session, cx| session.run_presence_tick(cx)) { + Ok(true) => {}, + Ok(false) | Err(_) => break, + } + } + tracing::debug!(session = %session_id, "collaboration presence timer stopped"); + }) + .detach(); + } + + fn next_digest_delay(&self) -> Option { + self + .timer_live() + .then(|| self.anti_entropy.duration_until_digest(Instant::now())) + } + + fn run_digest_tick(&mut self, cx: &mut Context) -> bool { + if !self.timer_live() { + return false; + } + + let now = Instant::now(); + if self.anti_entropy.digest_due(now) { + if matches!(self.phase, SessionPhase::Attached(_)) { + tracing::trace!(session = %self.session, quiet_rounds = self.quiet_digest_rounds, inbound_since_last_digest = self.inbound_since_last_digest, "collaboration digest timer fired"); + self.publish_digest(); + self.finish_digest_round(now, cx); + } + self.anti_entropy.mark_digest_sent(now); + } + true + } + + fn run_presence_tick(&mut self, cx: &mut Context) -> bool { + if !self.timer_live() { + return false; + } + if self.presence.is_some() { + tracing::trace!(session = %self.session, "collaboration presence timer fired"); + let roster_changed = self.remove_outdated_presence(cx); + self.refresh_own_presence(cx); + let peer_count_changed = self.refresh_peer_count(); + self.evaluate_connectivity(cx); + self.refresh_external_carets(cx); + if roster_changed || peer_count_changed { + cx.notify(); + } + } + true + } + + fn finish_digest_round(&mut self, now: Instant, cx: &mut Context) { + if self.inbound_since_last_digest { + self.quiet_digest_rounds = 0; + } else { + self.quiet_digest_rounds = self.quiet_digest_rounds.saturating_add(1); + } + self.inbound_since_last_digest = false; + tracing::trace!(session = %self.session, quiet_rounds = self.quiet_digest_rounds, "finished collaboration digest round"); + self.evaluate_connectivity(cx); + self.run_recovery_if_due(now, cx); + } + + fn run_recovery_if_due(&mut self, now: Instant, cx: &mut Context) { + let Some(next_recovery_at) = self.next_recovery_at else { + return; + }; + if now < next_recovery_at { + return; + } + + let mut attempted = false; + if let SessionPhase::Attached(Attachment { + connectivity: Connectivity::Offline { retries, .. }, + .. + }) = &mut self.phase + { + let delay = recovery_delay(*retries); + *retries = retries.saturating_add(1); + self.next_recovery_at = Some(now + delay); + attempted = true; + } + + if attempted { + tracing::warn!(session = %self.session, bootstrap_count = self.bootstrap_addrs.len(), "attempting collaboration connectivity recovery"); + self.awaiting_recovery_neighbor_until = Some(now + RECOVERY_NEIGHBOR_WAIT); + if self.bootstrap_addrs.is_empty() { + if let Err(error) = self.net_tx.try_send(NetCommand::EnsureUp) { + tracing::warn!(session = %self.session, error = %error, "queueing collaboration ensure-up recovery failed"); + } + } else if let Some(admission) = self.admission.clone() { + if let Err(error) = self.net_tx.try_send(NetCommand::JoinSession { + session: self.session, + bootstrap: self.bootstrap_addrs.clone(), + admission, + }) { + tracing::warn!(session = %self.session, error = %error, "queueing collaboration join-session recovery failed"); + } + } else { + tracing::warn!(session = %self.session, "cannot rejoin collaboration session without stored admission"); + } + self.publish_digest(); + cx.notify(); + } + } + + fn start_connectivity_probe(&mut self, cx: &mut Context) -> bool { + if self.probe_pending { + return true; + } + if self.runtime.is_none() { + return false; + } + let candidates = self.known_peers.iter().copied().collect::>(); + if candidates.is_empty() { + return false; + } + let our_vv = self.runtime_vv.clone(); + let (reply_tx, reply_rx) = async_channel::bounded(1); + if self + .net_tx + .try_send(NetCommand::PullUpdates { + session: self.session, + candidates, + our_vv, + reply: reply_tx, + }) + .is_err() + { + return false; + } + + self.probe_pending = true; + cx.spawn(async move |session, cx| { + let result = reply_rx.recv().await; + let _ = session.update(cx, |session, cx| { + session.probe_pending = false; + match result { + Ok(Ok(bytes)) => { + session.last_probe_failed = false; + // §perf: bytes is owned here; move it in to avoid a full-update memcpy. + if !bytes.is_empty() + && let Err(error) = session.import_update_bytes_owned(bytes, cx) + { + session.detach( + DetachReason::Fatal(format!("probing collaboration peer failed to apply updates: {error:#}")), + cx, + ); + return; + } + session.note_inbound_traffic(cx); + }, + Ok(Err(_)) | Err(_) => { + session.last_probe_failed = true; + session.evaluate_connectivity(cx); + }, + } + }); + }) + .detach(); + true + } + + fn mark_offline(&mut self, now: Instant, cx: &mut Context) { + let session_id = self.session; + let quiet_rounds = self.quiet_digest_rounds; + let neighbors = self.neighbors.len(); + if let SessionPhase::Attached(attachment) = &mut self.phase + && matches!(attachment.connectivity, Connectivity::Online) + { + let peers_present = attachment.peers_present; + attachment.connectivity = Connectivity::Offline { since: now, retries: 0 }; + self.next_recovery_at = Some(now); + tracing::warn!(session = %session_id, quiet_rounds, neighbors, peers_present, "collaboration session marked offline"); + cx.notify(); + } + } + + fn timer_live(&self) -> bool { + !matches!(self.phase, SessionPhase::Detached(_)) + } +} + +fn recovery_delay(retries: u32) -> Duration { + let shift = retries.min(5); + let secs = 1_u64 + .checked_shl(shift) + .unwrap_or(RECOVERY_MAX_BACKOFF.as_secs()); + Duration::from_secs(secs).min(RECOVERY_MAX_BACKOFF) +} diff --git a/crates/flowstate/src/collab/share_dialog.rs b/crates/flowstate/src/collab/share_dialog.rs new file mode 100644 index 00000000..3065da6a --- /dev/null +++ b/crates/flowstate/src/collab/share_dialog.rs @@ -0,0 +1,649 @@ +use flowstate_collab::{ + SessionId, + discovery::{DiscoveryAdvertisement, eligible_advertisements}, + ticket::SessionTicket, +}; +use gpui::{ + AnyElement, App, Context, Entity, FocusHandle, Focusable, InteractiveElement, IntoElement, KeyDownEvent, ParentElement, Render, SharedString, + Subscription, WeakEntity, Window, div, prelude::*, px, +}; +use gpui_component::{ + ActiveTheme as _, Disableable, Sizable as _, WindowExt as _, + button::{Button, ButtonVariants as _}, + h_flex, + input::{Input, InputEvent, InputState}, + scroll::ScrollableElement, + v_flex, +}; +use uuid::Uuid; + +use crate::workspace::Workspace; + +use super::{DetachReason, SessionPhase, status}; + +#[path = "share_dialog_view.rs"] +mod share_dialog_view; +use share_dialog_view::*; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum CollabDialogMode { + Share, + Join, +} + +pub struct CollabShareDialog { + workspace: WeakEntity, + panel_id: Option, + mode: CollabDialogMode, + join_input: Entity, + joining_session: Option, + ticket_text: Option, + ticket_loading: bool, + ticket_error: Option, + copy_notice: Option, + join_error: Option, + discovery_scanning: bool, + discovery_peers: Vec, + discovery_error: Option, + discovery_status: Option, + _input_subscription: Subscription, + _join_subscription: Option, +} + +#[hotpath::measure_all] +impl CollabShareDialog { + pub fn new( + workspace: WeakEntity, + panel_id: Option, + mode: CollabDialogMode, + window: &mut Window, + cx: &mut Context, + ) -> Self { + let join_input = cx.new(|cx| InputState::new(window, cx).placeholder("Paste a Flowstate collaboration invite")); + let input_subscription = cx.subscribe(&join_input, |dialog, _, event: &InputEvent, cx| { + if let InputEvent::Change = event { + dialog.joining_session = None; + dialog._join_subscription = None; + dialog.validate_join_ticket(cx); + } + }); + + let mut dialog = Self { + workspace, + panel_id, + mode, + join_input, + joining_session: None, + ticket_text: None, + ticket_loading: false, + ticket_error: None, + copy_notice: None, + join_error: None, + discovery_scanning: false, + discovery_peers: Vec::new(), + discovery_error: None, + discovery_status: None, + _input_subscription: input_subscription, + _join_subscription: None, + }; + if mode == CollabDialogMode::Share { + dialog.refresh_ticket(cx); + } + if panel_id.is_some() { + // The dialog is constructed inside the Workspace's own update (via + // `open_collaboration_dialog_with_mode`), so the scan — which reads + // discovery context through `workspace.update` — must wait until the + // Workspace lease is released or GPUI panics on the double lease. + let handle = cx.entity().downgrade(); + cx.defer(move |cx| { + let _ = handle.update(cx, |dialog, cx| dialog.scan_trusted_peers(cx)); + }); + } + dialog + } + + pub fn focus(&self, window: &mut Window, cx: &mut Context) { + if self.mode == CollabDialogMode::Join { + self.join_input.focus_handle(cx).focus(window); + } + } + + fn close(&mut self, window: &mut Window, cx: &mut Context) { + let _ = self + .workspace + .update(cx, |workspace, cx| workspace.close_collaboration_dialog(cx)); + window.close_dialog(cx); + } + + fn set_mode(&mut self, mode: CollabDialogMode, cx: &mut Context) { + if self.mode == mode { + return; + } + self.mode = mode; + self.join_error = None; + self.copy_notice = None; + if mode == CollabDialogMode::Share { + self.refresh_ticket(cx); + } + cx.notify(); + } + + fn start_session(&mut self, cx: &mut Context) { + let Some(panel_id) = self.panel_id else { + self.ticket_error = Some("Open a document before starting a session.".into()); + cx.notify(); + return; + }; + + match self + .workspace + .update(cx, |workspace, cx| workspace.start_collaboration_on_document(panel_id, cx)) + { + Ok(Some(_)) => self.refresh_ticket(cx), + Ok(None) => { + self.ticket_error = Some("The active document could not be shared.".into()); + cx.notify(); + }, + Err(error) => { + self.ticket_error = Some(format!("The workspace is no longer available: {error}").into()); + cx.notify(); + }, + } + } + + fn refresh_ticket(&mut self, cx: &mut Context) { + let Some(panel_id) = self.panel_id else { + self.ticket_text = None; + self.ticket_loading = false; + return; + }; + let Some(ticket_rx) = crate::collab::request_ticket_for_panel(panel_id, cx) else { + self.ticket_text = None; + self.ticket_loading = false; + self.ticket_error = None; + return; + }; + + self.ticket_loading = true; + self.ticket_error = None; + self.copy_notice = None; + cx.notify(); + + cx.spawn(async move |dialog, cx| { + let result = ticket_rx.recv().await; + let encoded = match result { + Ok(Ok(ticket)) => Ok(ticket.encode_invite_link()), + Ok(Err(error)) => Err(format!("Creating collaboration invite failed: {error:#}")), + Err(error) => Err(format!("Creating collaboration invite failed: {error}")), + }; + + let _ = dialog.update(cx, |dialog, cx| { + dialog.ticket_loading = false; + match encoded { + Ok(text) => { + dialog.ticket_text = Some(text.into()); + dialog.ticket_error = None; + }, + Err(error) => { + dialog.ticket_error = Some(error.into()); + }, + } + cx.notify(); + }); + }) + .detach(); + } + + fn save_invite_file(&mut self, cx: &mut Context) { + let Some(ticket) = self.ticket_text.clone() else { return }; + let suggested = SessionTicket::decode_text(ticket.as_ref()) + .ok() + .map(|ticket| ticket.title) + .unwrap_or_else(|| "Flowstate collaboration".into()); + let safe_title = suggested + .chars() + .map(|character| { + if character.is_alphanumeric() || matches!(character, ' ' | '-' | '_') { + character + } else { + '_' + } + }) + .collect::(); + let directory = dirs::document_dir().unwrap_or_else(|| std::env::current_dir().unwrap_or_default()); + let path = cx.prompt_for_new_path(&directory, Some(&format!("{safe_title}.flowinvite"))); + cx.spawn(async move |dialog, cx| { + let result = match path.await { + Ok(Ok(Some(path))) => std::fs::write(&path, ticket.as_bytes()) + .map(|_| format!("Invite saved to {}.", path.display())) + .map_err(|error| format!("Saving invite failed: {error}")), + Ok(Ok(None)) => return, + Ok(Err(error)) => Err(format!("Opening the save dialog failed: {error}")), + Err(error) => Err(format!("The save dialog closed unexpectedly: {error}")), + }; + let _ = dialog.update(cx, |dialog, cx| { + match result { + Ok(notice) => dialog.copy_notice = Some(notice.into()), + Err(error) => dialog.ticket_error = Some(error.into()), + } + cx.notify(); + }); + }) + .detach(); + } + + fn validate_join_ticket(&mut self, cx: &mut Context) { + self.join_error = match self.parse_join_ticket(cx) { + Ok(_) => None, + Err(error) => Some(error.into()), + }; + cx.notify(); + } + + fn parse_join_ticket(&self, cx: &App) -> Result, String> { + let text = self.join_input.read(cx).value().trim().to_string(); + if text.is_empty() { + return Ok(None); + } + SessionTicket::decode_text(&text) + .map(Some) + .map_err(|error| format!("Invalid invite: {error}")) + } + + fn join_session(&mut self, window: &mut Window, cx: &mut Context) { + if self.joining_session.is_some() { + return; + } + let ticket = match self.parse_join_ticket(cx) { + Ok(Some(ticket)) => ticket, + Ok(None) => { + self.join_error = Some("Paste an invite before joining.".into()); + cx.notify(); + return; + }, + Err(error) => { + self.join_error = Some(error.into()); + cx.notify(); + return; + }, + }; + + match self + .workspace + .update(cx, |workspace, cx| workspace.join_collaboration_session(ticket, window, cx)) + { + Ok(Some(session)) => { + self.join_error = None; + self.joining_session = Some(session); + self.subscribe_join_session(session, cx); + cx.notify(); + }, + Ok(None) => { + self.join_error = Some("Joining collaboration session failed.".into()); + cx.notify(); + }, + Err(error) => { + self.join_error = Some(format!("The workspace is no longer available: {error}").into()); + cx.notify(); + }, + } + } + + fn subscribe_join_session(&mut self, session_id: SessionId, cx: &mut Context) { + self._join_subscription = crate::collab::session_for_id(session_id, cx).map(|session| cx.observe(&session, |_, _, cx| cx.notify())); + } + + fn scan_trusted_peers(&mut self, cx: &mut Context) { + let Some(panel_id) = self.panel_id else { return }; + let context = self + .workspace + .update(cx, |workspace, cx| workspace.collaboration_discovery_context(panel_id, cx)) + .ok() + .flatten(); + let Some((document_id, path)) = context else { + self.discovery_error = Some("Save this document before looking for trusted collaborators.".into()); + cx.notify(); + return; + }; + self.discovery_scanning = true; + self.discovery_error = None; + self.discovery_status = Some("Looking for trusted collaborators...".into()); + let receiver = crate::collab::scan_document_discovery(document_id, cx); + cx.spawn(async move |dialog, cx| { + let result = receiver.recv().await; + let _ = dialog.update(cx, |dialog, cx| { + dialog.discovery_scanning = false; + match result { + Ok(result) => { + dialog.discovery_status = if result.paused { + Some("Discovery is paused in Collaboration settings.".into()) + } else if result.active_transports.is_empty() { + Some("No discovery transport is enabled. Invite links and files still work.".into()) + } else { + Some( + format!( + "Checked {}.", + result + .active_transports + .iter() + .map(|transport| format!("{transport:?}")) + .collect::>() + .join(" and ") + ) + .into(), + ) + }; + let fingerprint = flowstate_collab::discovery::document_fingerprint(document_id); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let own = crate::app_settings::load_local_user_profile().identity_fingerprint; + dialog.discovery_peers = eligible_advertisements(result.advertisements, fingerprint, now, |identity| { + identity.to_string() != own + && crate::app_settings::standing_access_for_path(&identity.to_string(), &path) == crate::app_settings::StandingAccess::Allowed + }); + if !result.failures.is_empty() { + dialog.discovery_error = Some( + result + .failures + .into_iter() + .map(|(transport, error)| format!("{transport:?}: {error}")) + .collect::>() + .join("; ") + .into(), + ); + } + }, + Err(error) => dialog.discovery_error = Some(format!("Discovery stopped: {error}").into()), + } + cx.notify(); + }); + }) + .detach(); + } + + fn join_trusted_peer(&mut self, advertisement: DiscoveryAdvertisement, window: &mut Window, cx: &mut Context) { + if self.joining_session.is_some() { + return; + } + self.discovery_error = None; + let receiver = crate::collab::request_discovered_ticket(advertisement, cx); + let workspace = self.workspace.clone(); + let window = window.window_handle(); + cx.spawn(async move |dialog, cx| { + let result = receiver.recv().await; + let _ = window.update(cx, |_, window, cx| { + let _ = dialog.update(cx, |dialog, cx| { + match result { + Ok(Ok(ticket)) => match workspace.update(cx, |workspace, cx| workspace.join_collaboration_session(ticket, window, cx)) { + Ok(Some(session)) => { + dialog.joining_session = Some(session); + dialog.subscribe_join_session(session, cx); + }, + _ => dialog.discovery_error = Some("The trusted collaboration session could not be joined.".into()), + }, + Ok(Err(error)) => dialog.discovery_error = Some(format!("Trusted admission was refused: {error:#}").into()), + Err(error) => dialog.discovery_error = Some(format!("Trusted admission stopped: {error}").into()), + } + cx.notify(); + }); + }); + }) + .detach(); + } + + fn discovery_panel(&self, cx: &mut Context) -> AnyElement { + let mut panel = v_flex() + .gap_2() + .rounded_md() + .border_1() + .border_color(cx.theme().border) + .p_3() + .child( + h_flex() + .items_center() + .justify_between() + .child( + div() + .text_sm() + .font_weight(gpui::FontWeight::SEMIBOLD) + .child("Trusted collaborators"), + ) + .child( + Button::new("scan-trusted-collaborators") + .label(if self.discovery_scanning { "Looking..." } else { "Look now" }) + .small() + .outline() + .disabled(self.discovery_scanning) + .on_click(cx.listener(|dialog, _, _, cx| dialog.scan_trusted_peers(cx))), + ), + ) + .child(helper_text( + "Only safety-code-verified people with standing access to this document appear here.", + cx, + )); + if let Some(status) = self.discovery_status.clone() { + panel = panel.child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child(status), + ); + } + if self.discovery_peers.is_empty() && !self.discovery_scanning { + panel = panel.child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child("No trusted collaborators found."), + ); + } + for peer in self.discovery_peers.clone() { + let name = peer.profile.display_name.clone(); + panel = panel.child( + h_flex() + .items_center() + .justify_between() + .child(div().text_sm().child(name)) + .child( + Button::new(("join-trusted-collaborator", peer.device_id as u64)) + .label("Join") + .small() + .primary() + .on_click(cx.listener(move |dialog, _, window, cx| dialog.join_trusted_peer(peer.clone(), window, cx))), + ), + ); + } + panel + .when_some(self.discovery_error.clone(), |this, error| this.child(error_text(error, cx))) + .into_any_element() + } + + fn confirm_leave(&mut self, window: &mut Window, cx: &mut Context) { + let Some(panel_id) = self.panel_id else { + return; + }; + let _ = self + .workspace + .update(cx, |workspace, cx| workspace.confirm_leave_collaboration_on_panel(panel_id, window, cx)); + } + + fn on_key_down(&mut self, event: &KeyDownEvent, window: &mut Window, cx: &mut Context) { + match event.keystroke.key.as_str() { + "escape" => { + self.close(window, cx); + cx.stop_propagation(); + }, + "enter" if self.mode == CollabDialogMode::Join => { + self.join_session(window, cx); + cx.stop_propagation(); + }, + _ => {}, + } + } + + fn render_share_pane(&self, phase: Option<&SessionPhase>, cx: &mut Context) -> AnyElement { + if self.panel_id.is_none() { + return v_flex() + .gap_3() + .child(section_title("Share this document", cx)) + .child(helper_text("Open a rich-text document before starting a collaboration session.", cx)) + .into_any_element(); + } + + if phase.is_none_or(|phase| matches!(phase, SessionPhase::Detached(_))) { + return v_flex() + .gap_3() + .child(section_title("Share this document", cx)) + .child(helper_text( + "Start a session to create one invite ticket. Everyone who joins can edit, invite others, and leave independently.", + cx, + )) + .child(self.discovery_panel(cx)) + .child( + Button::new("start-collaboration-session") + .label("Start session") + .primary() + .on_click(cx.listener(|dialog, _, _, cx| dialog.start_session(cx))), + ) + .when_some(self.ticket_error.clone(), |this, error| this.child(error_text(error, cx))) + .into_any_element(); + } + + let Some(phase) = phase else { + return v_flex().into_any_element(); + }; + let roster = self + .panel_id + .map_or_else(Vec::new, |panel_id| crate::collab::roster_for_panel(panel_id, cx)); + v_flex() + .gap_3() + .child( + h_flex() + .items_center() + .justify_between() + .child(section_title("Invite people", cx)) + .child(status::status_pill(phase, cx)), + ) + .child(helper_text(connectivity_text(phase).as_str(), cx)) + .child(roster_list(roster, cx)) + .child(self.discovery_panel(cx)) + .child(helper_text( + "This invite names the document and works while the live session is available. Everyone who joins can edit and invite others.", + cx, + )) + .child(ticket_box(self.ticket_text.clone(), self.ticket_loading, cx)) + .child( + h_flex().gap_2().child( + Button::new("save-collaboration-invite") + .label("Save invite file") + .small() + .outline() + .disabled(self.ticket_loading || self.ticket_text.is_none()) + .on_click(cx.listener(|dialog, _, _, cx| dialog.save_invite_file(cx))), + ), + ) + .when_some(self.ticket_error.clone(), |this, error| this.child(error_text(error, cx))) + .when_some(self.copy_notice.clone(), |this, notice| this.child(success_text(notice, cx))) + .child( + h_flex().justify_end().child( + Button::new("leave-collaboration-session") + .label("Leave session") + .danger() + .on_click(cx.listener(|dialog, _, window, cx| dialog.confirm_leave(window, cx))), + ), + ) + .into_any_element() + } + + fn render_join_pane(&self, cx: &mut Context) -> AnyElement { + let parsed = self.parse_join_ticket(cx).ok().flatten(); + let progress = self.join_progress(cx); + v_flex() + .gap_3() + .child(section_title("Join a session", cx)) + .child(helper_text( + "Paste an invite ticket. Joining opens the shared document as a new tab and never merges into your existing files.", + cx, + )) + .child(Input::new(&self.join_input).w_full().cleanable(true)) + .when_some(parsed.as_ref().map(|ticket| ticket.title.clone()), |this, title| { + this.child(success_text(format!("Invite for: {title}").into(), cx)) + }) + .when_some(progress, |this, (is_error, text)| { + if is_error { + this.child(error_text(text, cx)) + } else { + this.child(progress_text(text, cx)) + } + }) + .when_some(self.join_error.clone(), |this, error| this.child(error_text(error, cx))) + .child( + h_flex().justify_end().child( + Button::new("join-collaboration-session") + .label(if self.joining_session.is_some() { "Joining..." } else { "Join" }) + .primary() + .disabled(self.joining_session.is_some()) + .on_click(cx.listener(|dialog, _, window, cx| dialog.join_session(window, cx))), + ), + ) + .into_any_element() + } + + fn join_progress(&self, cx: &App) -> Option<(bool, SharedString)> { + let phase = crate::collab::phase_for_session(self.joining_session?, cx)?; + match phase { + SessionPhase::Creating => Some((false, "Starting collaboration...".into())), + SessionPhase::Joining(stage) => Some((false, status::join_stage_label(&stage).into())), + SessionPhase::Attached(_) => Some((false, "Joined. Opening shared document...".into())), + SessionPhase::Detached(DetachReason::JoinFailed(error)) => Some((true, error.into())), + SessionPhase::Detached(_) => None, + } + } +} + +#[hotpath::measure_all] +impl Focusable for CollabShareDialog { + fn focus_handle(&self, cx: &App) -> FocusHandle { + self.join_input.focus_handle(cx) + } +} + +#[hotpath::measure_all] +impl Render for CollabShareDialog { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let phase = self + .panel_id + .and_then(|panel_id| crate::collab::phase_for_panel(panel_id, cx)); + + v_flex() + .max_h(px(560.0)) + .gap_4() + .on_key_down(cx.listener(Self::on_key_down)) + .child( + h_flex() + .gap_2() + .child( + tab_button("collab-share-tab", "Share", self.mode == CollabDialogMode::Share).on_click(cx.listener(|dialog, _, _, cx| { + dialog.set_mode(CollabDialogMode::Share, cx); + })), + ) + .child( + tab_button("collab-join-tab", "Join", self.mode == CollabDialogMode::Join).on_click(cx.listener(|dialog, _, window, cx| { + dialog.set_mode(CollabDialogMode::Join, cx); + dialog.focus(window, cx); + })), + ), + ) + .child( + div() + .flex_1() + .overflow_y_scrollbar() + .child(match self.mode { + CollabDialogMode::Share => self.render_share_pane(phase.as_ref(), cx), + CollabDialogMode::Join => self.render_join_pane(cx), + }), + ) + } +} diff --git a/crates/flowstate/src/collab/share_dialog_view.rs b/crates/flowstate/src/collab/share_dialog_view.rs new file mode 100644 index 00000000..93855b77 --- /dev/null +++ b/crates/flowstate/src/collab/share_dialog_view.rs @@ -0,0 +1,174 @@ +use gpui::{AnyElement, App, Context, IntoElement, ParentElement, SharedString, div, prelude::*, px, relative, rgb}; +use gpui_component::{ + ActiveTheme as _, Disableable, Sizable, + avatar::Avatar, + button::{Button, ButtonVariants as _}, + clipboard::Clipboard, + h_flex, + scroll::ScrollableElement, + v_flex, +}; + +use super::CollabShareDialog; +use crate::collab::{Connectivity, SessionPhase}; + +pub(super) fn tab_button(id: &'static str, label: &'static str, selected: bool) -> Button { + let button = Button::new(id).label(label).small(); + if selected { button.primary() } else { button.outline() } +} + +pub(super) fn section_title(title: &'static str, cx: &App) -> impl IntoElement { + div() + .text_lg() + .font_weight(gpui::FontWeight::SEMIBOLD) + .text_color(cx.theme().foreground) + .child(title) +} + +pub(super) fn helper_text(text: &str, cx: &App) -> impl IntoElement { + div() + .text_sm() + .line_height(relative(1.45)) + .text_color(cx.theme().muted_foreground) + .child(text.to_string()) +} + +pub(super) fn ticket_box(ticket: Option, loading: bool, cx: &mut Context) -> AnyElement { + h_flex() + .items_start() + .gap_2() + .rounded_md() + .border_1() + .border_color(cx.theme().border) + .bg(cx.theme().background) + .p_3() + .child( + div() + .flex_1() + .min_h(px(84.0)) + .max_h(px(132.0)) + .overflow_y_scrollbar() + .text_xs() + .line_height(relative(1.35)) + .text_color(cx.theme().foreground) + .child(if loading { + SharedString::from("Minting a fresh invite...") + } else { + ticket + .clone() + .unwrap_or_else(|| "Start sharing to mint an invite.".into()) + }), + ) + .child(copy_invite_button(ticket, loading, cx)) + .into_any_element() +} + +pub(super) fn error_text(text: SharedString, cx: &App) -> impl IntoElement { + div().text_sm().text_color(cx.theme().danger).child(text) +} + +pub(super) fn progress_text(text: SharedString, cx: &App) -> impl IntoElement { + div().text_sm().text_color(cx.theme().info).child(text) +} + +pub(super) fn success_text(text: SharedString, cx: &App) -> impl IntoElement { + div().text_sm().text_color(cx.theme().success).child(text) +} + +pub(super) fn roster_list(entries: Vec, cx: &App) -> AnyElement { + let mut list = v_flex() + .gap_2() + .rounded_md() + .border_1() + .border_color(cx.theme().border) + .bg(cx.theme().background) + .p_3() + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child("Participants"), + ); + + if entries.is_empty() { + return list + .child( + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child("Presence is starting..."), + ) + .into_any_element(); + } + + for entry in entries { + let name = entry.name; + let label = if entry.is_self { format!("{name} (you)") } else { name.clone() }; + let color = gpui::Hsla::from(rgb(entry.color_rgb)); + list = list.child( + h_flex() + .items_center() + .gap_2() + .child( + Avatar::new() + .name(name) + .small() + .bg(color.opacity(0.14)) + .text_color(color) + .border_color(color.opacity(0.32)), + ) + .child(div().size(px(8.0)).rounded_full().bg(color)) + .child( + div() + .text_sm() + .text_color(cx.theme().foreground) + .child(label), + ), + ); + } + list.into_any_element() +} + +pub(super) fn connectivity_text(phase: &SessionPhase) -> String { + match phase { + SessionPhase::Creating => "Starting collaboration...".to_string(), + SessionPhase::Joining(_) => "Joining collaboration...".to_string(), + SessionPhase::Attached(attachment) => match &attachment.connectivity { + Connectivity::Online if attachment.peers_present == 0 => "Only you - share the invite.".to_string(), + Connectivity::Online => format!("{} people in session.", attachment.peers_present + 1), + Connectivity::Offline { .. } => "Offline - reconnecting. Changes will sync when connected.".to_string(), + }, + SessionPhase::Detached(_) => "This document is local.".to_string(), + } +} + +fn copy_invite_button(ticket: Option, loading: bool, cx: &mut Context) -> AnyElement { + let Some(ticket) = ticket.filter(|_| !loading) else { + return Button::new("copy-collaboration-invite-disabled") + .label("Copy") + .small() + .disabled(true) + .into_any_element(); + }; + let dialog = cx.entity().downgrade(); + h_flex() + .items_center() + .gap_1() + .child( + Clipboard::new("copy-collaboration-invite") + .value(ticket) + .on_copied(move |_, _, cx| { + let _ = dialog.update(cx, |dialog, cx| { + dialog.copy_notice = Some("Invite copied to clipboard.".into()); + cx.notify(); + }); + }), + ) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child("Copy"), + ) + .into_any_element() +} diff --git a/crates/flowstate/src/collab/shutdown.rs b/crates/flowstate/src/collab/shutdown.rs new file mode 100644 index 00000000..8dd1923f --- /dev/null +++ b/crates/flowstate/src/collab/shutdown.rs @@ -0,0 +1,10 @@ +use gpui::BorrowAppContext; + +use super::CollabManager; + +pub fn shutdown(cx: &mut C) +where + C: BorrowAppContext, +{ + cx.update_default_global::(|manager, _| manager.shutdown_runtime()); +} diff --git a/crates/flowstate/src/collab/status.rs b/crates/flowstate/src/collab/status.rs new file mode 100644 index 00000000..7cee264e --- /dev/null +++ b/crates/flowstate/src/collab/status.rs @@ -0,0 +1,112 @@ +use gpui::{AnyElement, App, IntoElement, div, prelude::*, px, rgb}; +use gpui_component::{ + ActiveTheme as _, Sizable as _, + avatar::{Avatar, AvatarGroup}, + h_flex, +}; + +use super::{Attachment, Connectivity, JoinStage, SessionPhase}; + +pub fn participant_group(phase: &SessionPhase, entries: Vec, cx: &App) -> AnyElement { + let mut avatars = AvatarGroup::new().xsmall().limit(4).ellipsis(); + for entry in entries { + let color = gpui::Hsla::from(rgb(entry.color_rgb)); + avatars = avatars.child( + Avatar::new() + .name(entry.name) + .bg(color.opacity(0.14)) + .text_color(color) + .border_color(color.opacity(0.38)), + ); + } + let (_, status_color) = status_label_and_color(phase, cx); + h_flex() + .items_center() + .gap_1() + .child(avatars) + .child(div().size(px(6.0)).rounded_full().bg(status_color)) + .into_any_element() +} + +pub fn status_pill(phase: &SessionPhase, cx: &App) -> AnyElement { + let (label, color) = status_label_and_color(phase, cx); + h_flex() + .flex_none() + .items_center() + .gap_1() + .px_2() + .py_0p5() + .rounded_full() + .border_1() + .border_color(color.opacity(0.38)) + .bg(color.opacity(0.10)) + .child(div().size(px(6.0)).rounded_full().bg(color)) + .child(div().text_size(px(10.0)).text_color(color).child(label)) + .into_any_element() +} + +pub fn tab_badge(phase: &SessionPhase, cx: &App) -> Option { + let (count, color) = match phase { + SessionPhase::Attached(attachment) => { + let color = match &attachment.connectivity { + Connectivity::Online => cx.theme().success, + Connectivity::Offline { .. } => cx.theme().muted_foreground, + }; + (Some(attachment.peers_present + 1), color) + }, + SessionPhase::Creating | SessionPhase::Joining(_) => (None, cx.theme().info), + SessionPhase::Detached(_) => return None, + }; + + Some( + h_flex() + .items_center() + .gap_0p5() + .child(div().size(px(6.0)).rounded_full().bg(color)) + .when_some(count, |this, count| { + this.child( + div() + .text_size(px(9.0)) + .font_weight(gpui::FontWeight::SEMIBOLD) + .text_color(color) + .child(count.to_string()), + ) + }) + .into_any_element(), + ) +} + +fn status_label_and_color(phase: &SessionPhase, cx: &App) -> (String, gpui::Hsla) { + match phase { + SessionPhase::Creating => ("Starting share".to_string(), cx.theme().info), + SessionPhase::Joining(stage) => (join_stage_label(stage), cx.theme().info), + SessionPhase::Attached(attachment) => attached_label_and_color(attachment, cx), + SessionPhase::Detached(_) => ("Not shared".to_string(), cx.theme().muted_foreground), + } +} + +pub fn join_stage_label(stage: &JoinStage) -> String { + match stage { + JoinStage::Resolving => "Resolving invite".to_string(), + JoinStage::Subscribing => "Joining session".to_string(), + JoinStage::FetchingSnapshot { got, total } => match total { + Some(total) => format!("Fetching {got}/{total} bytes"), + None => "Fetching document".to_string(), + }, + JoinStage::Building => "Opening shared doc".to_string(), + } +} + +fn attached_label_and_color(attachment: &Attachment, cx: &App) -> (String, gpui::Hsla) { + match &attachment.connectivity { + Connectivity::Online => { + let participants = attachment.peers_present + 1; + if attachment.peers_present == 0 { + ("Only you".to_string(), cx.theme().success) + } else { + (format!("{participants} in session"), cx.theme().success) + } + }, + Connectivity::Offline { .. } => ("Offline - will sync".to_string(), cx.theme().warning), + } +} diff --git a/crates/flowstate/src/commands.rs b/crates/flowstate/src/commands.rs index a629516d..78d981f7 100644 --- a/crates/flowstate/src/commands.rs +++ b/crates/flowstate/src/commands.rs @@ -4,7 +4,7 @@ use gpui::actions; pub use keymap::{Keymap, KeymapEntry, register_default_keybindings, register_keymap}; -actions!(flowstate_workspace, [FindInDocumentAction]); +actions!(flowstate_workspace, [FindInDocumentAction, FidelityMarkAction]); pub const RICH_TEXT_CONTEXT: &str = "RichTextEditor"; @@ -76,6 +76,12 @@ pub enum CommandId { OpenDocument, OpenDemoDocument, CloseDocument, + ShareDocument, + JoinSession, + StartCollaboration, + CopyCollaborationTicket, + JoinCollaborationFromClipboard, + LeaveCollaboration, FindInDocument, ToggleRibbon, NextTab, @@ -201,6 +207,12 @@ pub const COMMAND_SPECS: &[CommandSpec] = &[ CommandSpec::new(CommandId::OpenDocument, "Open Document", APP, &[]), CommandSpec::new(CommandId::OpenDemoDocument, "Open Demo Document", APP, &[]), CommandSpec::new(CommandId::CloseDocument, "Close Document", APP, &[]), + CommandSpec::new(CommandId::ShareDocument, "Share / Collaborate...", APP, &[]), + CommandSpec::new(CommandId::JoinSession, "Join Collaboration Session...", APP, &[]), + CommandSpec::new(CommandId::StartCollaboration, "Start Collaboration", APP, &[]), + CommandSpec::new(CommandId::CopyCollaborationTicket, "Copy Collaboration Invite", APP, &[]), + CommandSpec::new(CommandId::JoinCollaborationFromClipboard, "Join Collaboration from Clipboard", APP, &[]), + CommandSpec::new(CommandId::LeaveCollaboration, "Leave Collaboration", APP, &[]), CommandSpec::new(CommandId::FindInDocument, "Find in Document", APP, &["cmd-f", "ctrl-f"]), CommandSpec::new(CommandId::ToggleRibbon, "Toggle Ribbon", APP, &[]), CommandSpec::new(CommandId::NextTab, "Next Tab", APP, &["ctrl-tab"]), @@ -267,13 +279,12 @@ pub fn default_keys_for(id: CommandId) -> &'static [&'static str] { #[hotpath::measure] pub fn active_keys_for(id: CommandId) -> Vec { - let keymap = crate::app_settings::load_keymap(); - keymap - .entries - .iter() - .filter(|entry| entry.command == id) - .map(|entry| entry.key.clone()) - .collect() + crate::app_settings::load_keys_for_command(id) +} + +#[hotpath::measure] +pub fn active_first_key_for(id: CommandId) -> Option { + crate::app_settings::load_first_key_for_command(id) } #[hotpath::measure] diff --git a/crates/flowstate/src/commands/keymap.rs b/crates/flowstate/src/commands/keymap.rs index 4de74ee0..4e69f6ea 100644 --- a/crates/flowstate/src/commands/keymap.rs +++ b/crates/flowstate/src/commands/keymap.rs @@ -151,6 +151,12 @@ fn action_for_command(command: CommandId) -> Option> { | CommandId::OpenDocument | CommandId::OpenDemoDocument | CommandId::CloseDocument + | CommandId::ShareDocument + | CommandId::JoinSession + | CommandId::StartCollaboration + | CommandId::CopyCollaborationTicket + | CommandId::JoinCollaborationFromClipboard + | CommandId::LeaveCollaboration | CommandId::ToggleRibbon | CommandId::NextTab | CommandId::PreviousTab diff --git a/crates/flowstate/src/docx_conversion/mod.rs b/crates/flowstate/src/docx_conversion/mod.rs index 99640b6d..39761b87 100644 --- a/crates/flowstate/src/docx_conversion/mod.rs +++ b/crates/flowstate/src/docx_conversion/mod.rs @@ -3,7 +3,8 @@ use std::{io, path::Path}; pub use flowstate_docx::{ CleanAction, CleanedDocx, DocxCleanReport, DocxCleanStats, DocxConversionReport, RecognitionRule, clean_docx_bytes, convert_cleaned_docx_to_document, convert_docx_bytes_to_document, convert_docx_to_db8, convert_docx_to_document, convert_docx_to_pdf, - convert_pdf_to_db8, embed_db8_file_in_pdf, extract_db8_bytes_from_pdf, write_docx, write_pdf, write_pdf_with_db8_bytes, + convert_pdf_to_db8, embed_db8_file_in_pdf, extract_db8_bytes_from_pdf, import_cleaned_docx_to_loro, import_docx_bytes_to_loro, + import_docx_to_loro, write_docx, write_pdf, write_pdf_with_db8_bytes, }; use crate::app_settings::load_document_theme; diff --git a/crates/flowstate/src/file_search.rs b/crates/flowstate/src/file_search.rs index 4905efe4..09ce1629 100644 --- a/crates/flowstate/src/file_search.rs +++ b/crates/flowstate/src/file_search.rs @@ -163,14 +163,15 @@ fn search_document_files( .filter_map(|file| supported_hit_from_fff_file(picker, root, file)) .collect::>(); + // §perf: dedup against borrowed hit paths instead of cloning each PathBuf into the set. let existing = hits .iter() - .map(|hit| hit.path.clone()) - .collect::>(); + .map(|hit| hit.path.as_path()) + .collect::>(); let mut supplemental_hits = search_supplemental_document_files(supplemental_files, query, limit.saturating_mul(SEARCH_OVERSAMPLE_FACTOR).max(limit)) .into_iter() - .filter(|hit| !existing.contains(&hit.path)) + .filter(|hit| !existing.contains(hit.path.as_path())) .collect::>(); hits.append(&mut supplemental_hits); hits.truncate(limit); @@ -301,10 +302,8 @@ fn is_supported_document_path(path: &Path) -> bool { path .extension() .and_then(|extension| extension.to_str()) - .is_some_and(|extension| { - let extension = extension.to_ascii_lowercase(); - SUPPORTED_DOCUMENT_EXTENSIONS.contains(&extension.as_str()) - }) + // §perf: compare case-insensitively without allocating a lowercased String. + .is_some_and(|extension| SUPPORTED_DOCUMENT_EXTENSIONS.iter().any(|e| extension.eq_ignore_ascii_case(e))) } #[hotpath::measure] diff --git a/crates/flowstate/src/flow/panel.rs b/crates/flowstate/src/flow/panel.rs index d086e776..8162a064 100644 --- a/crates/flowstate/src/flow/panel.rs +++ b/crates/flowstate/src/flow/panel.rs @@ -149,8 +149,13 @@ impl Panel for FlowPanel { if active { let editor = self.editor.clone(); let panel_id = self.id; - let _ = self.workspace.update(cx, |workspace, cx| { - workspace.set_active_flow(panel_id, editor, cx); + let workspace = self.workspace.clone(); + // Dock plumbing may invoke this while the Workspace lease is held + // (same double-lease shape as the fixed share-dialog panic) — defer. + cx.defer(move |cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.set_active_flow(panel_id, editor, cx); + }); }); } } @@ -158,8 +163,11 @@ impl Panel for FlowPanel { #[hotpath::measure] fn on_removed(&mut self, window: &mut Window, cx: &mut Context) { let panel_id = self.id; - let _ = self.workspace.update(cx, |workspace, cx| { - workspace.remove_document_panel(panel_id, window, cx); + let workspace = self.workspace.clone(); + window.defer(cx, move |window, cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.remove_document_panel(panel_id, window, cx); + }); }); } diff --git a/crates/flowstate/src/lib.rs b/crates/flowstate/src/lib.rs index 1b6957bd..c422958d 100644 --- a/crates/flowstate/src/lib.rs +++ b/crates/flowstate/src/lib.rs @@ -6,10 +6,12 @@ pub mod app; pub mod app_settings; +pub mod collab; pub mod commands; pub mod docx_conversion; pub mod file_search; pub mod flow; +pub mod logging; pub mod ribbon; pub mod rich_text_element; pub mod workspace; diff --git a/crates/flowstate/src/logging.rs b/crates/flowstate/src/logging.rs new file mode 100644 index 00000000..e682f8b5 --- /dev/null +++ b/crates/flowstate/src/logging.rs @@ -0,0 +1,336 @@ +use std::{ + env, fs, + path::{Path, PathBuf}, + time::{Duration, SystemTime}, +}; + +use anyhow::{Context as _, Result}; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::{EnvFilter, Layer as _, fmt, layer::SubscriberExt as _, util::SubscriberInitExt as _}; + +/// Default filter: only the most serious events (errors), for every target. +/// Raise Flowstate's own verbosity with `FLOWSTATE_LOG_LEVEL`, or take full +/// control (including dependencies) with `FLOWSTATE_LOG` / `RUST_LOG`. +const DEFAULT_LOG_FILTER: &str = "error"; + +/// Crates whose verbosity `FLOWSTATE_LOG_LEVEL` raises. Dependencies stay at the +/// default level so the output stays focused on Flowstate's own logs. +const FLOWSTATE_TARGETS: [&str; 3] = ["flowstate", "flowstate_collab", "gpui_flowtext"]; + +pub struct LoggingGuard { + _guard: WorkerGuard, + // Kept alive so the fidelity JSONL background writer flushes on shutdown. + _fidelity_guard: Option, + directory: PathBuf, +} + +impl LoggingGuard { + pub fn directory(&self) -> &Path { + &self.directory + } +} +pub fn init() -> Result { + let directory = log_directory(); + fs::create_dir_all(&directory).with_context(|| format!("creating log directory {} failed", directory.display()))?; + prune_log_files(&directory)?; + // One unique, sortable, human-readable stamp per run so successive runs never + // clobber each other's logs. (The old `rolling::daily` produced a single + // `flowstate.log.` that every run on the same day appended to, which made + // it impossible to tell one run's logs from another's.) + let run_stamp = chrono::Local::now().format("%Y%m%d-%H%M%S").to_string(); + let file_appender = tracing_appender::rolling::never(&directory, format!("flowstate-{run_stamp}.log")); + let (writer, guard) = tracing_appender::non_blocking(file_appender); + + // Optionally mirror logs to stdout (handy when running from a terminal). + let stdout_layer = log_to_stdout().then(|| { + fmt::layer() + .with_writer(std::io::stdout) + .with_ansi(true) + .with_file(true) + .with_line_number(true) + .with_target(true) + }); + + // When fidelity tracing is on, additionally stream ONLY the fidelity events + // (and violations/markers, matched by the `fidelity` target prefix) as + // machine-parseable JSON lines to a dedicated file, so an automated agent gets + // exactly one JSON object per event without scraping the human log format. + // The fidelity JSONL sink MUST be non-blocking. Fidelity `event()`s fire from + // the render/layout hot path (per visible paragraph, per decoration, per caret), + // so a blocking `Mutex` writer would stall the MAIN thread on file I/O + // once per event — on a large document that is thousands of synchronous writes + // per frame, which froze the window. A background worker (like the human log) + // keeps the UI thread free; the returned guard flushes it on shutdown. + let (fidelity_writer, fidelity_guard) = match trace_fidelity_enabled() + .then(|| directory.join(format!("flowstate-fidelity-{run_stamp}.jsonl"))) + .and_then(|path| fs::File::create(&path).ok()) + { + Some(file) => { + let (writer, guard) = tracing_appender::non_blocking(file); + (Some(writer), Some(guard)) + }, + None => (None, None), + }; + let fidelity_json_layer = fidelity_writer.map(|writer| { + fmt::layer() + .json() + .flatten_event(true) + .with_current_span(false) + .with_span_list(false) + .with_writer(writer) + .with_filter(EnvFilter::new("fidelity=debug")) + }); + + tracing_subscriber::registry() + .with(env_filter()) + .with( + fmt::layer() + .with_writer(writer) + .with_ansi(false) + .with_file(true) + .with_line_number(true) + .with_target(true) + .with_thread_ids(true) + .with_thread_names(true), + ) + .with(stdout_layer) + .with(fidelity_json_layer) + .try_init() + .context("initializing flowstate logging failed")?; + + // Tell the operator where THIS run's logs are, on stderr, so a terminal run + // shows the exact paths without grepping. + if trace_fidelity_enabled() { + eprintln!( + "flowstate: logs → {}/flowstate-{run_stamp}.log + flowstate-fidelity-{run_stamp}.jsonl (fidelity tracing ON)", + directory.display() + ); + } else { + eprintln!("flowstate: logs → {}/flowstate-{run_stamp}.log", directory.display()); + } + + Ok(LoggingGuard { + _guard: guard, + _fidelity_guard: fidelity_guard, + directory, + }) +} + +/// Resolves the log filter from the environment, in precedence order: +/// 1. `FLOWSTATE_LOG` — full [`EnvFilter`] directive (advanced; also controls +/// dependency crates), e.g. `flowstate_collab=trace,flowstate::collab=debug`. +/// 2. `RUST_LOG` — the standard [`EnvFilter`] directive. +/// 3. `FLOWSTATE_LOG_LEVEL` — a single level (`error|warn|info|debug|trace`) +/// applied to Flowstate's own crates, leaving dependencies at `error`. +/// 4. Default — `error` only. +fn env_filter() -> EnvFilter { + let mut filter = base_env_filter(); + if trace_fidelity_enabled() { + // A single `fidelity=debug` directive surfaces the whole diagnostics + // stream: EnvFilter matches targets by prefix, so this also covers the + // `fidelity.violation` target (emitted at error), regardless of the + // ambient log level. One env var thus yields both firehose and violations. + filter = filter.add_directive( + "fidelity=debug" + .parse() + .expect("static `fidelity=debug` directive is always valid"), + ); + } + filter +} + +fn base_env_filter() -> EnvFilter { + let Some(directive) = directive_from_env() else { + return EnvFilter::new(DEFAULT_LOG_FILTER); + }; + EnvFilter::try_new(&directive).unwrap_or_else(|error| { + eprintln!("invalid log filter {directive:?}: {error}; falling back to {DEFAULT_LOG_FILTER:?}"); + EnvFilter::new(DEFAULT_LOG_FILTER) + }) +} + +/// Mirrors `flowstate_fidelity`'s env parsing: any non-empty, non-`0` value of +/// `FLOWSTATE_TRACE_FIDELITY` turns diagnostics on. Kept local so logging setup +/// has no cross-crate ordering dependency. +fn trace_fidelity_enabled() -> bool { + env::var("FLOWSTATE_TRACE_FIDELITY").is_ok_and(|value| { + let trimmed = value.trim(); + !trimmed.is_empty() && trimmed != "0" + }) +} + +fn directive_from_env() -> Option { + for var in ["FLOWSTATE_LOG", "RUST_LOG"] { + if let Ok(filter) = env::var(var) + && !filter.trim().is_empty() + { + return Some(filter); + } + } + + let level = level_from_env()?; + let scoped = FLOWSTATE_TARGETS + .iter() + .map(|target| format!("{target}={level}")) + .collect::>() + .join(","); + Some(format!("{DEFAULT_LOG_FILTER},{scoped}")) +} + +fn level_from_env() -> Option<&'static str> { + let value = env::var("FLOWSTATE_LOG_LEVEL").ok()?; + match value.trim().to_ascii_lowercase().as_str() { + "" => None, + "error" => Some("error"), + "warn" | "warning" => Some("warn"), + "info" => Some("info"), + "debug" => Some("debug"), + "trace" => Some("trace"), + other => { + eprintln!("invalid FLOWSTATE_LOG_LEVEL {other:?}; expected error|warn|info|debug|trace"); + None + }, + } +} + +fn log_to_stdout() -> bool { + env::var("FLOWSTATE_LOG_STDOUT").is_ok_and(|value| is_truthy(&value)) +} + +fn is_truthy(value: &str) -> bool { + matches!(value.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on") +} + +fn log_directory() -> PathBuf { + // `FLOWSTATE_LOG_DIR` still wins when set. Otherwise default to a + // `flowstate-logs/` directory under the current working directory — for the + // usual `cargo run` from the repo root that puts logs right in the repo (and + // it's git-ignored) instead of a hidden platform data dir under `/tmp`-style + // paths, which is far easier to find during development. + env::var_os("FLOWSTATE_LOG_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + env::current_dir() + .unwrap_or_else(|_| PathBuf::from(".")) + .join("flowstate-logs") + }) +} + +/// Build a user-initiated diagnostics bundle without telemetry or network I/O. +/// Invite bearers and the user's home-directory prefix are removed before the +/// selected local log files are copied into one readable report. +pub fn export_redacted_diagnostics(destination: &Path) -> Result<()> { + let directory = log_directory(); + let mut logs = fs::read_dir(&directory) + .with_context(|| format!("reading log directory {} failed", directory.display()))? + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .filter(|path| matches!(path.extension().and_then(|extension| extension.to_str()), Some("log" | "jsonl"))) + .collect::>(); + logs.sort(); + let home = dirs::home_dir().and_then(|path| path.to_str().map(str::to_owned)); + let mut report = String::from("Flowstate diagnostics (local, redacted)\n\n"); + for path in logs.into_iter().rev().take(4).rev() { + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + use std::fmt::Write as _; + let _ = writeln!(report, "--- {name} ---"); + let contents = fs::read_to_string(&path).unwrap_or_else(|error| format!("[unreadable log: {error}]")); + for line in contents.lines() { + report.push_str(&redact_diagnostic_line(line, home.as_deref())); + report.push('\n'); + } + } + atomicwrites::AtomicFile::new(destination, atomicwrites::OverwriteBehavior::AllowOverwrite) + .write(|file| std::io::Write::write_all(file, report.as_bytes())) + .with_context(|| format!("writing diagnostics export {} failed", destination.display())) +} + +fn redact_diagnostic_line(line: &str, home: Option<&str>) -> String { + let mut redacted = line + .split_whitespace() + .map(|token| { + if token.contains(flowstate_collab::ticket::INVITE_URL_PREFIX) || token.contains(flowstate_collab::ticket::TICKET_KIND) { + "[REDACTED_INVITE]" + } else { + token + } + }) + .collect::>() + .join(" "); + if let Some(home) = home.filter(|home| !home.is_empty()) { + redacted = redacted.replace(home, "~"); + } + redacted +} +const MAX_LOG_FILES: usize = 14; +const MAX_LOG_AGE: Duration = Duration::from_hours(336); + +fn prune_log_files(directory: &Path) -> Result<()> { + let mut candidates = Vec::new(); + let now = SystemTime::now(); + for entry in fs::read_dir(directory).with_context(|| format!("reading log directory {} failed", directory.display()))? { + let entry = entry?; + let path = entry.path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + // Prune this run's per-run files (`flowstate-.log`, + // `flowstate-fidelity-.jsonl`) and any legacy `flowstate.log.`. + let extension = path.extension().and_then(|extension| extension.to_str()); + let is_run_log = name.starts_with("flowstate-") && matches!(extension, Some("log" | "jsonl")); + if !is_run_log && !name.starts_with("flowstate.log.") { + continue; + } + let metadata = entry.metadata()?; + let modified = metadata.modified().ok(); + let age = modified.and_then(|modified| now.duration_since(modified).ok()); + if age.is_some_and(|age| age > MAX_LOG_AGE) { + let _ = fs::remove_file(&path); + continue; + } + candidates.push((path, modified.unwrap_or(SystemTime::UNIX_EPOCH))); + } + + if candidates.len() > MAX_LOG_FILES { + candidates.sort_by_key(|(_, modified)| *modified); + for (path, _) in candidates.drain(..candidates.len() - MAX_LOG_FILES) { + let _ = fs::remove_file(path); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prunes_excess_log_files() { + let dir = tempfile::tempdir().unwrap(); + for index in 0..20 { + let path = dir.path().join(format!("flowstate.log.{index}")); + fs::write(&path, index.to_string()).unwrap(); + } + + prune_log_files(dir.path()).unwrap(); + + let retained = fs::read_dir(dir.path()) + .unwrap() + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name()) + .filter(|name| name.to_string_lossy().starts_with("flowstate.log.")) + .count(); + assert!(retained <= MAX_LOG_FILES); + } + + #[test] + fn diagnostic_redaction_removes_invites_and_home_paths() { + let line = "join flowstate://join#fscollab-secret from /home/alex/Documents/brief.db8"; + let redacted = redact_diagnostic_line(line, Some("/home/alex")); + assert!(!redacted.contains("fscollab-secret")); + assert!(redacted.contains("[REDACTED_INVITE]")); + assert!(redacted.contains("~/Documents/brief.db8")); + } +} diff --git a/crates/flowstate/src/main.rs b/crates/flowstate/src/main.rs index 2af3b150..6a50e0bc 100644 --- a/crates/flowstate/src/main.rs +++ b/crates/flowstate/src/main.rs @@ -7,7 +7,7 @@ use clap::{Parser, Subcommand}; use flowstate::{ docx_conversion::{convert_db8_to_docx, convert_db8_to_pdf, convert_docx_to_pdf, convert_pdf_to_db8}, - run_standalone, write_demo_document, + logging, run_standalone, write_demo_document, }; struct FlowstateAllocator; @@ -62,6 +62,11 @@ struct Cli { #[derive(Subcommand)] enum CliCommand { + /// Export recent local logs with invite bearers and home paths redacted. + ExportDiagnostics { + /// Destination text file. + output: PathBuf, + }, /// Convert a DB8 document to DOCX and exit. Db8ToDocx { /// Input `.db8` document. @@ -94,6 +99,18 @@ enum CliCommand { #[hotpath::main(allocator = FlowstateAllocator)] fn main() { + let _logging_guard = match logging::init() { + Ok(guard) => Some(guard), + Err(error) => { + eprintln!("flowstate logging initialization failed: {error:#}"); + None + }, + }; + + // Enable CRDT/document fidelity diagnostics when FLOWSTATE_TRACE_FIDELITY is + // set. Must run after logging init so violation dumps reach the subscriber. + flowstate_fidelity::init_from_env(); + let cli = Cli::parse(); if cli.write_demo_db8 { @@ -103,6 +120,9 @@ fn main() { if let Some(command) = cli.command { match command { + CliCommand::ExportDiagnostics { output } => { + logging::export_redacted_diagnostics(&output).expect("failed to export redacted diagnostics"); + }, CliCommand::Db8ToDocx { input, output } => { convert_db8_to_docx(input, output).expect("failed to export DOCX"); }, diff --git a/crates/flowstate/src/ribbon/editor_ribbon/command_dispatch.rs b/crates/flowstate/src/ribbon/editor_ribbon/command_dispatch.rs index a0fca768..10200aaf 100644 --- a/crates/flowstate/src/ribbon/editor_ribbon/command_dispatch.rs +++ b/crates/flowstate/src/ribbon/editor_ribbon/command_dispatch.rs @@ -65,7 +65,7 @@ fn perform_ribbon_command(editor: &mut RichTextEditor, command_id: RibbonCommand RibbonCommandId::Redo => { editor.redo(cx); }, - RibbonCommandId::ExportFormat | RibbonCommandId::ExportSend | RibbonCommandId::ToggleInvisibility => {}, + RibbonCommandId::Revisions | RibbonCommandId::ExportFormat | RibbonCommandId::ExportSend | RibbonCommandId::ToggleInvisibility => {}, } } diff --git a/crates/flowstate/src/ribbon/editor_ribbon/core.rs b/crates/flowstate/src/ribbon/editor_ribbon/core.rs index f2b14213..d98dee86 100644 --- a/crates/flowstate/src/ribbon/editor_ribbon/core.rs +++ b/crates/flowstate/src/ribbon/editor_ribbon/core.rs @@ -11,7 +11,7 @@ use gpui_component::menu::PopupMenuItem; use gpui_component::{ActiveTheme as _, Disableable as _, Icon, IconName, PixelsExt as _, Selectable as _, Sizable as _}; use serde::{Deserialize, Serialize}; -use crate::commands::{CommandId, active_keys_for}; +use crate::commands::{CommandId, active_first_key_for}; use crate::ribbon::style_catalog::{HIGHLIGHT_STYLE_SPECS, PARAGRAPH_STYLE_SPECS, SEMANTIC_STYLE_SPECS}; use crate::workspace::Workspace; use uuid::Uuid; diff --git a/crates/flowstate/src/ribbon/editor_ribbon/helpers.rs b/crates/flowstate/src/ribbon/editor_ribbon/helpers.rs index 5c825d68..f6f0f0eb 100644 --- a/crates/flowstate/src/ribbon/editor_ribbon/helpers.rs +++ b/crates/flowstate/src/ribbon/editor_ribbon/helpers.rs @@ -10,8 +10,8 @@ fn highlight_color(style: HighlightStyle, theme: &DocumentTheme) -> Hsla { #[hotpath::measure] fn shortcut_for(command_id: CommandId) -> Option { - active_keys_for(command_id).first().map(|key| { - Keystroke::parse(key) + active_first_key_for(command_id).map(|key| { + Keystroke::parse(&key) .map(|stroke| Kbd::format(&stroke)) .unwrap_or_else(|_| key.clone()) }) @@ -141,6 +141,7 @@ fn ribbon_command_key(command_id: RibbonCommandId) -> u64 { RibbonCommandId::ClearFormatting => 5_003, RibbonCommandId::Undo => 6_000, RibbonCommandId::Redo => 6_001, + RibbonCommandId::Revisions => 6_002, RibbonCommandId::ExportFormat => 7_000, RibbonCommandId::ExportSend => 7_001, RibbonCommandId::ToggleInvisibility => 8_000, diff --git a/crates/flowstate/src/ribbon/editor_ribbon/modern_impl.rs b/crates/flowstate/src/ribbon/editor_ribbon/modern_impl.rs index d99aa3a0..70311bff 100644 --- a/crates/flowstate/src/ribbon/editor_ribbon/modern_impl.rs +++ b/crates/flowstate/src/ribbon/editor_ribbon/modern_impl.rs @@ -87,6 +87,19 @@ impl ModernStylesRibbon { overflow_behavior: OverflowBehavior::KeepVisible, checked_highlight: None, }, + RibbonCommand { + id: RibbonCommandId::Revisions, + label: "Revisions", + group_id: "history", + shortcut: None, + command_id: None, + priority: 2, + accent: None, + selected: false, + disabled: panel_id.is_none(), + overflow_behavior: OverflowBehavior::KeepVisible, + checked_highlight: None, + }, ], }, ); diff --git a/crates/flowstate/src/ribbon/editor_ribbon/modern_layout.rs b/crates/flowstate/src/ribbon/editor_ribbon/modern_layout.rs index dfb09e67..32db4419 100644 --- a/crates/flowstate/src/ribbon/editor_ribbon/modern_layout.rs +++ b/crates/flowstate/src/ribbon/editor_ribbon/modern_layout.rs @@ -127,7 +127,9 @@ fn command_chip_width( cx: &mut Context, ) -> gpui::Pixels { match command.id { - RibbonCommandId::Undo | RibbonCommandId::Redo | RibbonCommandId::ToggleInvisibility => px(metrics.chip_height.as_f32()), + RibbonCommandId::Undo | RibbonCommandId::Redo | RibbonCommandId::Revisions | RibbonCommandId::ToggleInvisibility => { + px(metrics.chip_height.as_f32()) + }, RibbonCommandId::ExportFormat => { let text_width = measure_ribbon_text("Format", metrics.chip_text_size, window, cx).as_f32(); px(text_width + 10.0 + metrics.chip_padding_x.as_f32() * 2.0 + 10.0) @@ -247,6 +249,11 @@ fn modern_command_chip( let _ = workspace.update(cx, |workspace, cx| workspace.toggle_speech_document(panel_id, cx)); } }, + RibbonCommandId::Revisions => { + if let Some(workspace) = workspace.clone() { + let _ = workspace.update(cx, |workspace, cx| workspace.open_revision_dialog(window, cx)); + } + }, _ => { editor.update(cx, |editor, cx| { perform_ribbon_command(editor, command_id, window, cx); @@ -269,7 +276,7 @@ fn ribbon_command_color(command: &RibbonCommand, cx: &App) -> Hsla { | RibbonCommandId::SendToSpeechDocument | RibbonCommandId::SendToSpeechDocumentEnd => cx.theme().info, RibbonCommandId::ToggleSpeechDocument => cx.theme().success, - RibbonCommandId::Undo | RibbonCommandId::Redo => cx.theme().primary, + RibbonCommandId::Undo | RibbonCommandId::Redo | RibbonCommandId::Revisions => cx.theme().primary, RibbonCommandId::ExportFormat | RibbonCommandId::ExportSend => cx.theme().info, RibbonCommandId::ToggleInvisibility => cx.theme().primary, } diff --git a/crates/flowstate/src/ribbon/editor_ribbon/modern_types.rs b/crates/flowstate/src/ribbon/editor_ribbon/modern_types.rs index a229d6bd..0cf49e48 100644 --- a/crates/flowstate/src/ribbon/editor_ribbon/modern_types.rs +++ b/crates/flowstate/src/ribbon/editor_ribbon/modern_types.rs @@ -54,6 +54,7 @@ pub enum RibbonCommandId { Undo, Redo, + Revisions, ExportFormat, ExportSend, ToggleInvisibility, @@ -103,6 +104,7 @@ impl RibbonLabel { RibbonCommandId::ClearFormatting => Some("icons/eraser.svg"), RibbonCommandId::ExportFormat => Some("icons/export.svg"), RibbonCommandId::ExportSend => Some("icons/send-horizontal.svg"), + RibbonCommandId::Revisions => Some("icons/archive.svg"), _ => None, }; Self { diff --git a/crates/flowstate/src/rich_text_element/mod.rs b/crates/flowstate/src/rich_text_element/mod.rs index 16f42a9e..4bf4a4b9 100644 --- a/crates/flowstate/src/rich_text_element/mod.rs +++ b/crates/flowstate/src/rich_text_element/mod.rs @@ -64,6 +64,12 @@ pub fn flowstate_command_to_rich_text(command: CommandId) -> Option, + selection: crate::rich_text_element::EditorSelection, + author_user_id: u128, + author_display_name: String, + create_input: gpui::Entity, + reply_input: gpui::Entity, + edit_input: gpui::Entity, + focus: FocusHandle, + comments: Vec, + loading: bool, + busy: bool, + error: Option, + replying_to: Option, + editing_message: Option<(u128, u128)>, +} + +impl CommentDialog { + pub fn new( + io: DocIoHandle, + editor: Entity, + selection: crate::rich_text_element::EditorSelection, + author_user_id: u128, + author_display_name: String, + window: &mut Window, + cx: &mut Context, + ) -> Self { + let create_input = cx.new(|cx| InputState::new(window, cx).placeholder("Comment on the selected text")); + let reply_input = cx.new(|cx| InputState::new(window, cx).placeholder("Write a reply")); + let edit_input = cx.new(|cx| InputState::new(window, cx).placeholder("Edit comment")); + let mut dialog = Self { + io, + editor, + selection, + author_user_id, + author_display_name, + create_input, + reply_input, + edit_input, + focus: cx.focus_handle(), + comments: Vec::new(), + loading: false, + busy: false, + error: None, + replying_to: None, + editing_message: None, + }; + dialog.reload(cx); + dialog + } + + fn reload(&mut self, cx: &mut Context) { + self.loading = true; + let io = self.io.clone(); + cx.spawn(async move |dialog, cx| { + let result = io.comments().await; + let _ = dialog.update(cx, |dialog, cx| { + dialog.loading = false; + match result { + Ok(comments) => { + refresh_comment_annotations(&dialog.editor, &comments, cx); + dialog.comments = comments; + dialog.error = None; + }, + Err(error) => dialog.error = Some(format!("Loading comments failed: {error:#}").into()), + } + cx.notify(); + }); + }) + .detach(); + } + + fn create_comment(&mut self, window: &mut Window, cx: &mut Context) { + if self.busy { + return; + } + let body = self.create_input.read(cx).value().trim().to_string(); + if body.is_empty() { + self.error = Some("Write a comment first.".into()); + cx.notify(); + return; + } + self.busy = true; + let io = self.io.clone(); + let selection = self.selection.clone(); + let user_id = self.author_user_id; + let name = self.author_display_name.clone(); + let input = self.create_input.clone(); + let window = window.window_handle(); + cx.spawn(async move |dialog, cx| { + let result = io.create_comment(selection, body, user_id, name).await; + let _ = window.update(cx, |_, window, cx| { + let _ = dialog.update(cx, |dialog, cx| { + dialog.busy = false; + match result { + Ok(_) => { + input.update(cx, |input, cx| input.set_value("", window, cx)); + dialog.reload(cx); + }, + Err(error) => dialog.error = Some(format!("Adding comment failed: {error:#}").into()), + } + cx.notify(); + }); + }); + }) + .detach(); + } + + fn reply(&mut self, comment_id: u128, window: &mut Window, cx: &mut Context) { + if self.busy { + return; + } + let body = self.reply_input.read(cx).value().trim().to_string(); + if body.is_empty() { + self.error = Some("Write a reply first.".into()); + cx.notify(); + return; + } + self.busy = true; + let io = self.io.clone(); + let user_id = self.author_user_id; + let name = self.author_display_name.clone(); + let input = self.reply_input.clone(); + let window = window.window_handle(); + cx.spawn(async move |dialog, cx| { + let result = io.reply_to_comment(comment_id, body, user_id, name).await; + let _ = window.update(cx, |_, window, cx| { + let _ = dialog.update(cx, |dialog, cx| { + dialog.busy = false; + match result { + Ok(_) => { + input.update(cx, |input, cx| input.set_value("", window, cx)); + dialog.replying_to = None; + dialog.reload(cx); + }, + Err(error) => dialog.error = Some(format!("Adding reply failed: {error:#}").into()), + } + cx.notify(); + }); + }); + }) + .detach(); + } + + fn set_resolved(&mut self, comment_id: u128, resolved: bool, cx: &mut Context) { + if self.busy { + return; + } + self.busy = true; + let io = self.io.clone(); + cx.spawn(async move |dialog, cx| { + let result = io.set_comment_resolved(comment_id, resolved).await; + let _ = dialog.update(cx, |dialog, cx| { + dialog.busy = false; + match result { + Ok(()) => dialog.reload(cx), + Err(error) => dialog.error = Some(format!("Updating comment failed: {error:#}").into()), + } + cx.notify(); + }); + }) + .detach(); + } + + fn begin_edit(&mut self, comment_id: u128, message_id: u128, body: String, window: &mut Window, cx: &mut Context) { + self + .edit_input + .update(cx, |input, cx| input.set_value(body, window, cx)); + self.editing_message = Some((comment_id, message_id)); + self.edit_input.focus_handle(cx).focus(window); + cx.notify(); + } + + fn save_edit(&mut self, comment_id: u128, message_id: u128, cx: &mut Context) { + if self.busy { + return; + } + let body = self.edit_input.read(cx).value().trim().to_string(); + if body.is_empty() { + self.error = Some("A comment cannot be empty.".into()); + cx.notify(); + return; + } + self.busy = true; + let io = self.io.clone(); + let user_id = self.author_user_id; + cx.spawn(async move |dialog, cx| { + let result = io + .edit_comment_message(comment_id, message_id, body, user_id) + .await; + let _ = dialog.update(cx, |dialog, cx| { + dialog.busy = false; + match result { + Ok(()) => { + dialog.editing_message = None; + dialog.reload(cx); + }, + Err(error) => dialog.error = Some(format!("Editing comment failed: {error:#}").into()), + } + cx.notify(); + }); + }) + .detach(); + } + + fn delete(&mut self, comment_id: u128, cx: &mut Context) { + if self.busy { + return; + } + self.busy = true; + let io = self.io.clone(); + let user_id = self.author_user_id; + cx.spawn(async move |dialog, cx| { + let result = io.delete_comment(comment_id, user_id).await; + let _ = dialog.update(cx, |dialog, cx| { + dialog.busy = false; + match result { + Ok(()) => dialog.reload(cx), + Err(error) => dialog.error = Some(format!("Deleting comment failed: {error:#}").into()), + } + cx.notify(); + }); + }) + .detach(); + } +} + +pub(crate) fn refresh_comment_annotations( + editor: &Entity, + comments: &[RuntimeCommentThread], + cx: &mut App, +) { + let selections = comments + .iter() + .filter(|thread| !thread.resolved) + .filter_map(|thread| thread.anchor) + .map(|(start, end)| crate::rich_text_element::ExternalSelection { + selection: crate::rich_text_element::EditorSelection::range(start, end), + color_rgb: 0xd99a20, + }) + .collect(); + editor.update(cx, |editor, cx| editor.set_annotation_selections(selections, cx)); +} + +impl Focusable for CommentDialog { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus.clone() + } +} + +impl Render for CommentDialog { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let has_selection = self.selection.anchor != self.selection.head; + let mut content = v_flex().gap_3(); + if has_selection { + content = content.child( + v_flex() + .gap_2() + .child(div().font_semibold().child("New comment")) + .child(Input::new(&self.create_input).w_full()) + .child( + h_flex().justify_end().child( + Button::new("comment-create") + .label(if self.busy { "Adding..." } else { "Comment" }) + .small() + .primary() + .disabled(self.busy) + .on_click(cx.listener(|dialog, _, window, cx| dialog.create_comment(window, cx))), + ), + ), + ); + } else { + content = content.child( + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child("Select text before creating a comment. Existing threads remain available below."), + ); + } + if self.loading { + content = content.child(div().text_sm().child("Loading comments...")); + } + if self.comments.is_empty() && !self.loading { + content = content.child( + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .child("No comments yet."), + ); + } + for thread in self.comments.clone() { + let comment_id = thread.comment_id; + let resolved = thread.resolved; + let is_author = thread + .messages + .first() + .is_some_and(|message| message.author_user_id == self.author_user_id); + let mut card = v_flex() + .gap_2() + .rounded_md() + .border_1() + .border_color(cx.theme().border) + .p_3() + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child(if thread.anchor.is_some() { + format!("“{}”", thread.quoted_text) + } else { + format!("Original text was removed: “{}”", thread.quoted_text) + }), + ); + for message in thread.messages { + let message_id = message.message_id; + let message_body = message.body.clone(); + let message_is_author = message.author_user_id == self.author_user_id; + let editing = self.editing_message == Some((comment_id, message_id)); + let mut message_view = v_flex() + .gap_1() + .child( + h_flex() + .items_center() + .justify_between() + .child( + div() + .text_xs() + .font_semibold() + .child(message.author_display_name), + ) + .when(message_is_author && !editing, |row| { + row.child( + Button::new(("comment-edit", message_id as u64)) + .label("Edit") + .xsmall() + .outline() + .on_click( + cx.listener(move |dialog, _, window, cx| dialog.begin_edit(comment_id, message_id, message_body.clone(), window, cx)), + ), + ) + }), + ) + .when(!editing, |view| { + view.child( + h_flex() + .gap_1() + .child(div().text_sm().child(message.body)) + .when(message.updated_at_unix_secs > message.created_at_unix_secs, |row| { + row.child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child("(edited)"), + ) + }), + ) + }); + if editing { + message_view = message_view + .child(Input::new(&self.edit_input).w_full()) + .child( + h_flex() + .gap_2() + .justify_end() + .child( + Button::new(("comment-edit-cancel", message_id as u64)) + .label("Cancel") + .xsmall() + .outline() + .on_click(cx.listener(|dialog, _, _, cx| { + dialog.editing_message = None; + cx.notify(); + })), + ) + .child( + Button::new(("comment-edit-save", message_id as u64)) + .label("Save") + .xsmall() + .primary() + .disabled(self.busy) + .on_click(cx.listener(move |dialog, _, _, cx| dialog.save_edit(comment_id, message_id, cx))), + ), + ); + } + card = card.child(message_view); + } + card = card.child( + h_flex() + .gap_2() + .child( + Button::new(("comment-reply-toggle", comment_id as u64)) + .label("Reply") + .xsmall() + .outline() + .on_click(cx.listener(move |dialog, _, _, cx| { + dialog.replying_to = Some(comment_id); + cx.notify(); + })), + ) + .child( + Button::new(("comment-resolve", comment_id as u64)) + .label(if resolved { "Reopen" } else { "Resolve" }) + .xsmall() + .outline() + .disabled(self.busy) + .on_click(cx.listener(move |dialog, _, _, cx| dialog.set_resolved(comment_id, !resolved, cx))), + ) + .when(is_author, |row| { + row.child( + Button::new(("comment-delete", comment_id as u64)) + .label("Delete") + .xsmall() + .danger() + .disabled(self.busy) + .on_click(cx.listener(move |dialog, _, _, cx| dialog.delete(comment_id, cx))), + ) + }), + ); + if self.replying_to == Some(comment_id) { + card = card.child( + h_flex() + .gap_2() + .child(Input::new(&self.reply_input).w_full()) + .child( + Button::new(("comment-reply", comment_id as u64)) + .label("Send") + .small() + .primary() + .disabled(self.busy) + .on_click(cx.listener(move |dialog, _, window, cx| dialog.reply(comment_id, window, cx))), + ), + ); + } + content = content.child(card.opacity(if resolved { 0.68 } else { 1.0 })); + } + + v_flex() + .max_h(px(600.0)) + .gap_2() + .when_some(self.error.clone(), |this, error| { + this.child(div().text_sm().text_color(cx.theme().danger).child(error)) + }) + .child(div().flex_1().overflow_y_scrollbar().child(content)) + } +} diff --git a/crates/flowstate/src/workspace/document_panel.rs b/crates/flowstate/src/workspace/document_panel.rs index 8d41bddd..8b849ab4 100644 --- a/crates/flowstate/src/workspace/document_panel.rs +++ b/crates/flowstate/src/workspace/document_panel.rs @@ -315,8 +315,13 @@ impl Panel for DocumentPanel { if active { let editor = self.editor.clone(); let panel_id = self.id; - let _ = self.workspace.update(cx, |workspace, cx| { - workspace.set_active_document(panel_id, editor, cx); + let workspace = self.workspace.clone(); + // Dock plumbing may invoke this while the Workspace lease is held + // (same double-lease shape as the fixed share-dialog panic) — defer. + cx.defer(move |cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.set_active_document(panel_id, editor, cx); + }); }); } } @@ -324,8 +329,11 @@ impl Panel for DocumentPanel { #[hotpath::measure] fn on_removed(&mut self, window: &mut Window, cx: &mut Context) { let panel_id = self.id; - let _ = self.workspace.update(cx, |workspace, cx| { - workspace.remove_document_panel(panel_id, window, cx); + let workspace = self.workspace.clone(); + window.defer(cx, move |window, cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.remove_document_panel(panel_id, window, cx); + }); }); } diff --git a/crates/flowstate/src/workspace/file_management.rs b/crates/flowstate/src/workspace/file_management.rs index 98057473..746e574d 100644 --- a/crates/flowstate/src/workspace/file_management.rs +++ b/crates/flowstate/src/workspace/file_management.rs @@ -3,14 +3,14 @@ use std::{ path::{Path, PathBuf}, }; -use crate::rich_text_element::{Document, blank_document}; +use crate::rich_text_element::{DocumentProjection, blank_document}; pub const UNTITLED_DOCUMENT_NAME: &str = "Untitled.db8"; pub const UNTITLED_FLOW_NAME: &str = "Untitled.fl0"; /// Create the document used by File > New and the empty-workspace New button. #[hotpath::measure] -pub fn new_blank_document() -> Document { +pub fn new_blank_document() -> DocumentProjection { blank_document() } diff --git a/crates/flowstate/src/workspace/icons.rs b/crates/flowstate/src/workspace/icons.rs index 49fa963d..65374815 100644 --- a/crates/flowstate/src/workspace/icons.rs +++ b/crates/flowstate/src/workspace/icons.rs @@ -1,8 +1,10 @@ #[cfg(target_os = "macos")] use gpui::ParentElement; +#[cfg(not(target_os = "macos"))] +use gpui_component::IconName; use gpui_component::button::Button; use gpui_component::button::ButtonVariants as _; -use gpui_component::{Icon, IconName, Sizable as _}; +use gpui_component::{Icon, Sizable as _}; #[derive(Clone, Copy)] pub enum AppIcon { diff --git a/crates/flowstate/src/workspace/mod.rs b/crates/flowstate/src/workspace/mod.rs index e4a2001f..4552253c 100644 --- a/crates/flowstate/src/workspace/mod.rs +++ b/crates/flowstate/src/workspace/mod.rs @@ -1,9 +1,11 @@ +mod comment_dialog; mod document_panel; pub mod document_search; pub mod document_search_overlay; mod file_management; mod file_search_overlay; mod icons; +mod revision_dialog; mod workspace; pub use workspace::{Workspace, install_workspace_close_prompt, open_workspace_window}; diff --git a/crates/flowstate/src/workspace/revision_dialog.rs b/crates/flowstate/src/workspace/revision_dialog.rs new file mode 100644 index 00000000..909d58c5 --- /dev/null +++ b/crates/flowstate/src/workspace/revision_dialog.rs @@ -0,0 +1,165 @@ +use flowstate_collab::crdt_runtime::RuntimeRevisionInfo; +use gpui::{App, Context, FocusHandle, Focusable, IntoElement, ParentElement, Render, SharedString, WeakEntity, Window, div, prelude::*, px}; +use gpui_component::{ + ActiveTheme as _, StyledExt as _, WindowExt as _, + button::{Button, ButtonVariants as _}, + h_flex, + scroll::ScrollableElement, + v_flex, +}; +use uuid::Uuid; + +use super::Workspace; + +pub struct RevisionDialog { + workspace: WeakEntity, + panel_id: Uuid, + focus: FocusHandle, + loading: bool, + error: Option, + revisions: Vec, +} + +impl RevisionDialog { + pub fn new( + workspace: WeakEntity, + panel_id: Uuid, + receiver: Option>>>, + cx: &mut Context, + ) -> Self { + let loading = receiver.is_some(); + if let Some(receiver) = receiver { + cx.spawn(async move |dialog, cx| { + let result = receiver.recv().await; + let _ = dialog.update(cx, |dialog, cx| { + dialog.loading = false; + match result { + Ok(Ok(revisions)) => dialog.revisions = revisions, + Ok(Err(error)) => dialog.error = Some(error.to_string().into()), + Err(error) => dialog.error = Some(format!("Revision request closed: {error}").into()), + } + cx.notify(); + }); + }) + .detach(); + } + Self { + workspace, + panel_id, + focus: cx.focus_handle(), + loading, + error: (!loading).then(|| "This document has no revision runtime.".into()), + revisions: Vec::new(), + } + } + + pub fn focus(&self, window: &mut Window, _: &mut App) { + self.focus.focus(window); + } +} + +impl Focusable for RevisionDialog { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus.clone() + } +} + +impl Render for RevisionDialog { + fn render(&mut self, _: &mut Window, cx: &mut Context) -> impl IntoElement { + let content = if self.loading { + div() + .py_6() + .text_color(cx.theme().muted_foreground) + .child("Loading revisions...") + .into_any_element() + } else if let Some(error) = self.error.clone() { + div() + .py_6() + .text_color(cx.theme().danger) + .child(error) + .into_any_element() + } else if self.revisions.is_empty() { + div() + .py_6() + .text_color(cx.theme().muted_foreground) + .child("No saved revisions yet.") + .into_any_element() + } else { + v_flex() + .gap_1() + .children(self.revisions.iter().map(|revision| { + let workspace = self.workspace.clone(); + let panel_id = self.panel_id; + let revision_id = revision.revision_id; + let title: SharedString = revision.title.clone().into(); + let summary: SharedString = revision.summary.clone().into(); + let author = revision + .author_display_name + .clone() + .or_else(|| { + revision + .author_user_id + .map(|id| format!("User {}", &format!("{id:032x}")[..8])) + }) + .unwrap_or_else(|| "Unknown author".into()); + let timestamp = chrono::DateTime::from_timestamp(revision.created_at_unix_secs, 0) + .map(|timestamp| { + timestamp + .with_timezone(&chrono::Local) + .format("%b %-d, %Y %-I:%M %p") + .to_string() + }) + .unwrap_or_else(|| "Unknown time".into()); + Button::new(("open-revision", revision_id as u64)) + .ghost() + .w_full() + .child( + h_flex() + .w_full() + .justify_between() + .gap_3() + .child( + v_flex() + .min_w_0() + .items_start() + .child(div().font_semibold().text_ellipsis().child(title)) + .child( + div() + .text_sm() + .text_color(cx.theme().muted_foreground) + .text_ellipsis() + .child(summary), + ) + .child( + div() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child(format!("{author} · {timestamp}")), + ), + ) + .child( + div() + .flex_none() + .text_xs() + .text_color(cx.theme().muted_foreground) + .child(format!("{:08x}", revision_id as u64)), + ), + ) + .on_click(move |_, window, cx| { + let opened = workspace + .update(cx, |workspace, cx| workspace.open_document_revision(panel_id, revision_id, window, cx)) + .unwrap_or(false); + if opened { + window.close_dialog(cx); + } + }) + })) + .into_any_element() + }; + + v_flex() + .max_h(px(520.0)) + .min_h(px(160.0)) + .child(div().flex_1().overflow_y_scrollbar().child(content)) + } +} diff --git a/crates/flowstate/src/workspace/toolkit_panel.rs b/crates/flowstate/src/workspace/toolkit_panel.rs index 0a235b57..af64a4cd 100644 --- a/crates/flowstate/src/workspace/toolkit_panel.rs +++ b/crates/flowstate/src/workspace/toolkit_panel.rs @@ -1063,7 +1063,7 @@ fn toolkit_preview_document( fallback_text: &str, mut theme: DocumentTheme, expanded: bool, -) -> crate::rich_text_element::Document { +) -> crate::rich_text_element::DocumentProjection { theme.zoom_factor *= TOOLKIT_PREVIEW_ZOOM; theme.pageless_inset_x = px(10.0); theme.pageless_inset_top = px(8.0); diff --git a/crates/flowstate/src/workspace/workspace/collab.rs b/crates/flowstate/src/workspace/workspace/collab.rs new file mode 100644 index 00000000..8c1d7de8 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/collab.rs @@ -0,0 +1,409 @@ +#[hotpath::measure_all] +impl Workspace { + pub(crate) fn collaboration_discovery_context(&self, panel_id: Uuid, cx: &App) -> Option<(u128, PathBuf)> { + let panel = self + .document_panels + .iter() + .find(|panel| panel.read(cx).id() == panel_id)?; + let editor = panel.read(cx).editor(); + let editor = editor.read(cx); + Some((editor.document().ids.document_id, editor.document_path()?.clone())) + } + + pub fn open_comment_dialog(&mut self, window: &mut Window, cx: &mut Context) { + let Some(panel_id) = self.active_document_id else { return }; + let Some(editor) = self.active_editor.clone() else { return }; + let Some(io) = self.document_runtimes.get(&panel_id).cloned() else { + return; + }; + if self.comment_dialog.is_some() { + window.close_dialog(cx); + self.comment_dialog = None; + } + let selection = editor.read(cx).selection().clone(); + let profile = crate::app_settings::load_local_user_profile(); + let dialog = cx + .new(|cx| crate::workspace::comment_dialog::CommentDialog::new(io, editor, selection, profile.user_id, profile.display_name, window, cx)); + let rendered = dialog.clone(); + let workspace = cx.entity().downgrade(); + window.open_dialog(cx, move |component, _, _| { + let workspace = workspace.clone(); + component + .title("Comments") + .w(px(620.0)) + .max_w(px(620.0)) + .on_close(move |_, _, cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.comment_dialog = None; + cx.notify(); + }); + }) + .child(rendered.clone()) + }); + self.comment_dialog = Some(dialog); + cx.notify(); + } + + pub fn open_collaboration_dialog(&mut self, window: &mut Window, cx: &mut Context) { + self.open_collaboration_dialog_with_mode(crate::collab::share_dialog::CollabDialogMode::Share, window, cx); + } + + pub fn open_join_collaboration_dialog(&mut self, window: &mut Window, cx: &mut Context) { + self.open_collaboration_dialog_with_mode(crate::collab::share_dialog::CollabDialogMode::Join, window, cx); + } + + pub fn close_collaboration_dialog(&mut self, cx: &mut Context) { + self.collaboration_dialog = None; + cx.notify(); + } + + fn open_collaboration_dialog_with_mode( + &mut self, + mode: crate::collab::share_dialog::CollabDialogMode, + window: &mut Window, + cx: &mut Context, + ) { + if self.collaboration_dialog.is_some() { + window.close_dialog(cx); + self.collaboration_dialog = None; + } + let workspace = cx.entity().downgrade(); + let panel_id = self.active_editor.as_ref().and(self.active_document_id); + let dialog = cx.new(|cx| crate::collab::share_dialog::CollabShareDialog::new(workspace, panel_id, mode, window, cx)); + let dialog_for_render = dialog.clone(); + let workspace_for_close = cx.entity().downgrade(); + window.open_dialog(cx, move |component_dialog, _, _| { + let workspace_for_close = workspace_for_close.clone(); + component_dialog + .title("Share / Collaborate") + .w(px(620.0)) + .max_w(px(620.0)) + .on_close(move |_, _, cx| { + let _ = workspace_for_close.update(cx, |workspace, cx| workspace.close_collaboration_dialog(cx)); + }) + .child(dialog_for_render.clone()) + }); + dialog.update(cx, |dialog, cx| dialog.focus(window, cx)); + self.collaboration_dialog = Some(dialog); + cx.notify(); + } + + pub fn start_collaboration_on_active_document(&mut self, cx: &mut Context) -> Option { + self.start_collaboration_on_document(self.active_document_id?, cx) + } + + pub fn start_collaboration_on_document(&mut self, panel_id: Uuid, cx: &mut Context) -> Option { + let panel = self + .document_panels + .iter() + .find(|panel| panel.read(cx).id() == panel_id)?; + let editor = panel.read(cx).editor(); + let title = self + .document_panels + .iter() + .find(|panel| panel.read(cx).id() == panel_id) + .map(|panel| panel.read(cx).title_text().to_string()) + .unwrap_or_else(|| "Shared document".to_string()); + let runtime = self.document_runtimes.get(&panel_id)?.clone(); + + tracing::info!(%panel_id, title = %title, "workspace starting collaboration on document"); + // Loro-first: nothing is ever pending — local intents commit synchronously + // into the same gate-protected core the session will now publish from. + match crate::collab::start_session_for_panel(panel_id, editor, title, runtime, cx) { + Ok(session) => { + tracing::info!(%panel_id, %session, "workspace started collaboration on document"); + Some(session) + }, + Err(error) => { + tracing::error!(%panel_id, error = %format_args!("{error:#}"), "starting collaboration session failed"); + None + }, + } + } + + pub fn request_active_collaboration_ticket( + &mut self, + cx: &mut Context, + ) -> Option>> { + crate::collab::request_ticket_for_panel(self.active_document_id?, cx) + } + + pub fn copy_active_collaboration_ticket(&mut self, window: &mut Window, cx: &mut Context) -> bool { + let Some(panel_id) = self.active_document_id else { + return false; + }; + let mut ticket_rx = crate::collab::request_ticket_for_panel(panel_id, cx); + if ticket_rx.is_none() { + if self.start_collaboration_on_active_document(cx).is_none() { + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Share failed", + Some("The active document could not be shared."), + &[PromptButton::ok("Ok")], + cx, + )); + return true; + } + ticket_rx = crate::collab::request_ticket_for_panel(panel_id, cx); + } + + let Some(ticket_rx) = ticket_rx else { + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Share failed", + Some("The collaboration ticket is not available yet."), + &[PromptButton::ok("Ok")], + cx, + )); + return true; + }; + + let window_handle = window.window_handle(); + cx.spawn(async move |_, cx| { + let result = ticket_rx.recv().await; + let _ = window_handle.update(cx, |_, window, cx| match result { + Ok(Ok(ticket)) => { + tracing::info!(session = %ticket.session, bootstrap_count = ticket.bootstrap.len(), "copied collaboration invite ticket to clipboard"); + cx.write_to_clipboard(gpui::ClipboardItem::new_string(ticket.encode_invite_link())); + std::mem::drop(window.prompt( + PromptLevel::Info, + "Invite copied", + Some("A Flowstate invite link was copied to the clipboard."), + &[PromptButton::ok("Ok")], + cx, + )); + }, + Ok(Err(error)) => { + tracing::error!(error = %format_args!("{error:#}"), "creating collaboration invite failed"); + let detail = format!("Creating collaboration invite failed: {error:#}"); + std::mem::drop(window.prompt(PromptLevel::Critical, "Share failed", Some(&detail), &[PromptButton::ok("Ok")], cx)); + }, + Err(error) => { + tracing::error!(error = %error, "collaboration invite receiver closed"); + let detail = format!("Creating collaboration invite failed: {error}"); + std::mem::drop(window.prompt(PromptLevel::Critical, "Share failed", Some(&detail), &[PromptButton::ok("Ok")], cx)); + }, + }); + }) + .detach(); + true + } + + pub fn join_collaboration_from_clipboard(&mut self, window: &mut Window, cx: &mut Context) -> bool { + let Some(item) = cx.read_from_clipboard() else { + tracing::warn!("join collaboration from clipboard failed because clipboard is empty"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some("The clipboard does not contain a collaboration invite ticket."), + &[PromptButton::ok("Ok")], + cx, + )); + return true; + }; + let Some(text) = item.text() else { + tracing::warn!("join collaboration from clipboard failed because clipboard has no text"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some("The clipboard does not contain text."), + &[PromptButton::ok("Ok")], + cx, + )); + return true; + }; + let ticket = match flowstate_collab::ticket::SessionTicket::decode_text(&text) { + Ok(ticket) => ticket, + Err(error) => { + tracing::warn!(bytes = text.len(), error = %format_args!("{error:#}"), "clipboard collaboration invite decode failed"); + let detail = format!("The clipboard text is not a valid collaboration invite: {error:#}"); + std::mem::drop(window.prompt(PromptLevel::Critical, "Join failed", Some(&detail), &[PromptButton::ok("Ok")], cx)); + return true; + }, + }; + tracing::info!(session = %ticket.session, bootstrap_count = ticket.bootstrap.len(), "joining collaboration session from clipboard"); + self + .join_collaboration_session(ticket, window, cx) + .is_some() + } + + pub fn confirm_leave_collaboration_on_active_document(&mut self, window: &mut Window, cx: &mut Context) -> bool { + self + .active_document_id + .is_some_and(|panel_id| self.confirm_leave_collaboration_on_panel(panel_id, window, cx)) + } + + pub fn confirm_leave_collaboration_on_panel(&mut self, panel_id: Uuid, window: &mut Window, cx: &mut Context) -> bool { + let phase = crate::collab::phase_for_panel(panel_id, cx); + if !collaboration_phase_blocks_close(phase.as_ref()) { + return false; + } + let detail = format!("{} Your copy of the document stays open.", collaboration_leave_detail(phase.as_ref())); + let answer = window.prompt( + PromptLevel::Warning, + "Leave this session?", + Some(&detail), + &[PromptButton::ok("Leave"), PromptButton::cancel("Cancel")], + cx, + ); + cx.spawn(async move |workspace, cx| { + if !matches!(answer.await, Ok(0)) { + return; + } + let _ = workspace.update(cx, |workspace, cx| { + workspace.leave_collaboration_on_panel(panel_id, cx); + }); + }) + .detach(); + true + } + + pub fn leave_collaboration_on_panel(&mut self, panel_id: Uuid, cx: &mut Context) -> bool { + // Loro-first: leaving a session stops transport only. The document's write + // authority, gate, and I/O service are untouched — solo editing continues + // through the identical path (invariant 5). Nothing to re-attach. + let left = crate::collab::leave_session_for_panel(panel_id, cx); + tracing::info!(%panel_id, left, "workspace leave collaboration requested"); + left + } + + pub fn join_collaboration_session( + &mut self, + ticket: flowstate_collab::ticket::SessionTicket, + window: &mut Window, + cx: &mut Context, + ) -> Option { + tracing::info!(session = %ticket.session, bootstrap_count = ticket.bootstrap.len(), "workspace joining collaboration session"); + let request = match crate::collab::join_session(ticket, cx) { + Ok(request) => request, + Err(error) => { + tracing::error!(error = %format_args!("{error:#}"), "joining collaboration session failed"); + let detail = format!("Joining collaboration session failed: {error:#}"); + std::mem::drop(window.prompt(PromptLevel::Critical, "Join failed", Some(&detail), &[PromptButton::ok("Ok")], cx)); + return None; + }, + }; + let session = request.session; + let completed = request.completed; + let window_handle = window.window_handle(); + + cx.spawn(async move |workspace, cx| { + let result = completed.recv().await; + let _ = window_handle.update(cx, |_, window, cx| { + let _ = workspace.update(cx, |workspace, cx| { + match result { + Ok(Ok(joined)) => { + tracing::info!(session = %joined.session, title = %joined.title, "collaboration join completed; opening joined document"); + // Loro-first (spec §3 join gate): the session built the document + // services from the initial snapshot import; the workspace now + // takes the write authority + I/O handle and wires the panel + // exactly like a solo document (invariant 5). The session keeps + // transport only. + let Some((authority, io)) = crate::collab::take_joined_document_services_for_session(joined.session, cx) else { + tracing::error!(session = %joined.session, "joined collaboration document services are unavailable"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some("The joined collaboration document is unavailable."), + &[PromptButton::ok("Ok")], + cx, + )); + return; + }; + let attachment = DocumentRuntimeAttachment { authority, io }; + let panel = match workspace.add_joined_collaboration_panel(joined.document, joined.title, attachment, window, cx) { + Ok(panel) => panel, + Err(error) => { + tracing::error!(session = %joined.session, error = %format_args!("{error:#}"), "starting joined document runtime failed"); + let detail = format!("Joined document runtime could not be started: {error:#}"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + return; + }, + }; + let panel_id = panel.read(cx).id(); + let editor = panel.read(cx).editor(); + if let Err(error) = crate::collab::attach_joined_session(joined.session, panel_id, editor, cx) { + tracing::error!(session = %joined.session, %panel_id, error = %format_args!("{error:#}"), "collaboration joined document attachment failed"); + let detail = format!("Joined document opened, but collaboration attachment failed: {error:#}"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + } else if workspace.collaboration_dialog.is_some() { + // The panel was fully wired by `create_document_panel` (write + // authority, I/O hooks, runtime map) — no second attach pass. + workspace.close_collaboration_dialog(cx); + window.close_dialog(cx); + } + }, + Ok(Err(error)) => { + tracing::error!(%session, error = %format_args!("{error:#}"), "joining collaboration session failed"); + let detail = format!("Joining collaboration session failed: {error:#}"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + }, + Err(error) => { + tracing::error!(%session, error = %error, "collaboration join completion channel closed"); + let detail = format!("Joining collaboration session failed: {error}"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Join failed", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + }, + } + }); + }); + }) + .detach(); + + Some(session) + } + + fn add_joined_collaboration_panel( + &mut self, + document: DocumentProjection, + title: String, + attachment: DocumentRuntimeAttachment, + window: &mut Window, + cx: &mut Context, + ) -> anyhow::Result> { + let panel = self.create_document_panel(document, None, Some(title), DocumentRuntimeSource::Attachment(attachment), window, cx)?; + self.persist_temporary_workspace_session(cx); + cx.notify(); + Ok(panel) + } + + pub fn leave_collaboration_on_active_document(&mut self, cx: &mut Context) -> bool { + self + .active_document_id + .is_some_and(|panel_id| self.leave_collaboration_on_panel(panel_id, cx)) + } + + pub fn leave_all_collaboration_sessions(&mut self, cx: &mut Context) { + let panel_ids = self + .document_panels + .iter() + .map(|panel| panel.read(cx).id()) + .collect::>(); + for panel_id in panel_ids { + crate::collab::leave_session_for_panel(panel_id, cx); + } + } +} diff --git a/crates/flowstate/src/workspace/workspace/collab_prompts.rs b/crates/flowstate/src/workspace/workspace/collab_prompts.rs new file mode 100644 index 00000000..b1dd6b2f --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/collab_prompts.rs @@ -0,0 +1,323 @@ +#[derive(Clone)] +struct CollaborationClosePanel { + id: Uuid, + panel: PanelKind, + save_prompt: CollaborationSavePrompt, +} + +#[derive(Clone, Copy)] +enum CollaborationSavePrompt { + Clean, + Dirty, + Pathless, +} + +enum CollaborationCloseResolution { + Proceed { discard: Option }, + Cancelled, +} + +#[hotpath::measure_all] +impl Workspace { + fn close_collaboration_document_panel( + &mut self, + panel_id: Uuid, + panel_kind: PanelKind, + window: &mut Window, + cx: &mut Context, + ) -> bool { + let phase = crate::collab::phase_for_panel(panel_id, cx); + if !collaboration_phase_blocks_close(phase.as_ref()) { + return false; + } + + let detail = collaboration_leave_detail(phase.as_ref()); + let save_prompt = collaboration_save_prompt_for_panel(&panel_kind, cx); + let answer = window.prompt( + PromptLevel::Warning, + "Leave the collaboration session?", + Some(&detail), + &[PromptButton::ok("Leave"), PromptButton::cancel("Cancel")], + cx, + ); + let window_handle = window.window_handle(); + cx.spawn(async move |workspace, cx| { + if !matches!(answer.await, Ok(0)) { + return; + } + + let resolution = resolve_collaboration_save(panel_kind, save_prompt, window_handle, cx).await; + let CollaborationCloseResolution::Proceed { discard } = resolution else { + return; + }; + if let Some(panel) = discard { + panel.discard(cx); + } + + let _ = window_handle.update(cx, |_, window, cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.leave_collaboration_on_panel(panel_id, cx); + workspace.remove_document_panel(panel_id, window, cx); + }); + }); + }) + .detach(); + true + } + + fn request_close_window_with_collaboration(&mut self, window: &mut Window, cx: &mut Context) -> bool { + let collaboration_panels = self.collaboration_close_panels(cx); + if collaboration_panels.is_empty() { + return false; + } + + let collaboration_ids = collaboration_panels + .iter() + .map(|panel| panel.id) + .collect::>(); + let dirty_panels = self + .dirty_panels(cx) + .into_iter() + .filter(|panel| !collaboration_ids.contains(&panel.panel_id(cx))) + .collect::>(); + + let answer = if collaboration_panels.len() == 1 { + let phase = crate::collab::phase_for_panel(collaboration_panels[0].id, cx); + window.prompt( + PromptLevel::Warning, + "Leave the collaboration session and quit?", + Some(&collaboration_leave_detail(phase.as_ref())), + &[PromptButton::ok("Leave"), PromptButton::cancel("Cancel")], + cx, + ) + } else { + let detail = format!( + "You're in {} collaboration sessions. Leave all and quit?", + collaboration_panels.len() + ); + window.prompt( + PromptLevel::Warning, + "Leave collaboration sessions?", + Some(&detail), + &[PromptButton::ok("Leave"), PromptButton::cancel("Cancel")], + cx, + ) + }; + let window_handle = window.window_handle(); + + cx.spawn(async move |workspace, cx| { + if !matches!(answer.await, Ok(0)) { + return; + } + + let mut discards = Vec::new(); + for panel in collaboration_panels { + match resolve_collaboration_save(panel.panel, panel.save_prompt, window_handle, cx).await { + CollaborationCloseResolution::Proceed { discard } => { + if let Some(discard) = discard { + discards.push(discard); + } + }, + CollaborationCloseResolution::Cancelled => return, + } + } + + if !resolve_dirty_window_close(dirty_panels, window_handle, cx).await { + return; + } + + for panel in discards { + panel.discard(cx); + } + + let _ = window_handle.update(cx, |_, window, cx| { + let _ = workspace.update(cx, |workspace, cx| workspace.leave_all_collaboration_sessions(cx)); + window.remove_window(); + }); + }) + .detach(); + true + } + + fn collaboration_close_panels(&self, cx: &App) -> Vec { + self + .document_panels + .iter() + .filter_map(|panel| { + let panel_state = panel.read(cx); + let id = panel_state.id(); + let phase = crate::collab::phase_for_panel(id, cx); + collaboration_phase_blocks_close(phase.as_ref()).then(|| { + let panel_kind = PanelKind::Document { + panel: panel.clone(), + editor: panel_state.editor(), + }; + CollaborationClosePanel { + id, + save_prompt: collaboration_save_prompt_for_panel(&panel_kind, cx), + panel: panel_kind, + } + }) + }) + .collect() + } +} + +#[hotpath::measure_all] +impl PanelKind { + fn panel_id(&self, cx: &App) -> Uuid { + match self { + PanelKind::Document { panel, .. } => panel.read(cx).id(), + PanelKind::Flow { panel, .. } => panel.read(cx).id(), + } + } +} + +#[hotpath::measure] +async fn resolve_collaboration_save( + panel: PanelKind, + prompt: CollaborationSavePrompt, + window_handle: AnyWindowHandle, + cx: &mut gpui::AsyncApp, +) -> CollaborationCloseResolution { + match prompt { + CollaborationSavePrompt::Clean => CollaborationCloseResolution::Proceed { discard: None }, + CollaborationSavePrompt::Dirty | CollaborationSavePrompt::Pathless => { + let answer = match window_handle.update(cx, |_, window, cx| { + let (detail, save_label) = match prompt { + CollaborationSavePrompt::Pathless => ("This shared tab has no saved file.", "Save As..."), + CollaborationSavePrompt::Dirty => ("This document has unsaved changes.", "Save"), + CollaborationSavePrompt::Clean => unreachable!(), + }; + window.prompt( + PromptLevel::Warning, + "Save changes before closing?", + Some(detail), + &[PromptButton::ok(save_label), PromptButton::new("Don't Save"), PromptButton::cancel("Cancel")], + cx, + ) + }) { + Ok(answer) => answer, + Err(error) => { + show_save_failed(window_handle, cx, format!("failed to show save prompt: {error}")); + return CollaborationCloseResolution::Cancelled; + }, + }; + + match answer.await { + Ok(0) => match panel.save(window_handle, cx).await { + PanelSaveOutcome::Saved => CollaborationCloseResolution::Proceed { discard: None }, + PanelSaveOutcome::Cancelled => CollaborationCloseResolution::Cancelled, + PanelSaveOutcome::Failed(error) => { + show_save_failed(window_handle, cx, error); + CollaborationCloseResolution::Cancelled + }, + }, + Ok(1) => CollaborationCloseResolution::Proceed { discard: Some(panel) }, + _ => CollaborationCloseResolution::Cancelled, + } + }, + } +} + +#[hotpath::measure] +async fn resolve_dirty_window_close( + dirty_panels: Vec, + window_handle: AnyWindowHandle, + cx: &mut gpui::AsyncApp, +) -> bool { + if dirty_panels.is_empty() { + return true; + } + let message = if dirty_panels.len() == 1 { + "This document has unsaved changes." + } else { + "One or more documents have unsaved changes." + }; + let answer = match window_handle.update(cx, |_, window, cx| { + window.prompt( + PromptLevel::Warning, + "Save changes before closing?", + Some(message), + &[PromptButton::ok("Save"), PromptButton::new("Don't Save"), PromptButton::cancel("Cancel")], + cx, + ) + }) { + Ok(answer) => answer, + Err(error) => { + show_save_failed(window_handle, cx, format!("failed to show save prompt: {error}")); + return false; + }, + }; + + match answer.await { + Ok(0) => { + for panel in dirty_panels { + match panel.save(window_handle, cx).await { + PanelSaveOutcome::Saved => {}, + PanelSaveOutcome::Cancelled => return false, + PanelSaveOutcome::Failed(error) => { + show_save_failed(window_handle, cx, error); + return false; + }, + } + } + true + }, + Ok(1) => { + for panel in dirty_panels { + panel.discard(cx); + } + true + }, + _ => false, + } +} + +#[hotpath::measure] +fn collaboration_save_prompt_for_panel(panel: &PanelKind, cx: &App) -> CollaborationSavePrompt { + match panel { + PanelKind::Document { editor, .. } => { + let editor = editor.read(cx); + if editor.document_path().is_none() { + CollaborationSavePrompt::Pathless + } else if editor.has_unsaved_changes() { + CollaborationSavePrompt::Dirty + } else { + CollaborationSavePrompt::Clean + } + }, + PanelKind::Flow { editor, .. } => { + let editor = editor.read(cx); + if editor.document_path().is_none() { + CollaborationSavePrompt::Pathless + } else if editor.has_unsaved_changes() { + CollaborationSavePrompt::Dirty + } else { + CollaborationSavePrompt::Clean + } + }, + } +} + +#[hotpath::measure] +fn collaboration_phase_blocks_close(phase: Option<&crate::collab::SessionPhase>) -> bool { + phase.is_some_and(|phase| !matches!(phase, crate::collab::SessionPhase::Detached(_))) +} + +#[hotpath::measure] +fn collaboration_leave_detail(phase: Option<&crate::collab::SessionPhase>) -> String { + match phase { + Some(crate::collab::SessionPhase::Attached(attachment)) if attachment.peers_present == 0 => { + "This tab is live, but no other people are present right now.".to_string() + }, + Some(crate::collab::SessionPhase::Attached(attachment)) => format!( + "This tab is live with {} other {}.", + attachment.peers_present, + if attachment.peers_present == 1 { "person" } else { "people" } + ), + Some(crate::collab::SessionPhase::Creating) => "This tab is starting a collaboration session.".to_string(), + Some(crate::collab::SessionPhase::Joining(_)) => "This tab is joining a collaboration session.".to_string(), + _ => "This tab is connected to a collaboration session.".to_string(), + } +} diff --git a/crates/flowstate/src/workspace/workspace/collaboration_settings.rs b/crates/flowstate/src/workspace/workspace/collaboration_settings.rs new file mode 100644 index 00000000..3c6f7d40 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/collaboration_settings.rs @@ -0,0 +1,591 @@ +struct CollaborationProfileInputState { + input: Entity, + persisted_value: String, + _subscription: Subscription, +} + +struct DropboxConnectionInputState { + app_key: Entity, + root: Entity, +} + +struct TrustedCollaboratorInputState { + identity: Entity, + name: Entity, + compared: bool, + _subscriptions: Vec, +} + +struct CollaborationSquadInputState { + name: Entity, + member: Entity, +} + +fn collaboration_profile_item(_workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, window, cx| { + let profile = crate::app_settings::load_local_user_profile(); + let state = window.use_keyed_state("collaboration-profile-name", cx, { + let initial = profile.display_name.clone(); + move |window, cx| { + let input = cx.new(|cx| { + InputState::new(window, cx) + .default_value(initial.clone()) + .placeholder("Your display name") + }); + let subscription = cx.subscribe_in( + &input, + window, + move |state: &mut CollaborationProfileInputState, input, event: &InputEvent, _, cx| { + if !matches!(event, InputEvent::Change) { + return; + } + let display_name = input.read(cx).value().trim().to_string(); + if display_name.is_empty() || display_name == state.persisted_value { + return; + } + let current = crate::app_settings::load_local_user_profile(); + if let Err(error) = + crate::app_settings::save_local_collaboration_profile(display_name.clone(), current.color_rgb, current.avatar_path) + { + tracing::warn!(%error, "saving collaboration display name failed"); + } else { + state.persisted_value = display_name; + } + }, + ); + CollaborationProfileInputState { + input, + persisted_value: initial, + _subscription: subscription, + } + } + }); + + h_flex() + .w_full() + .items_center() + .justify_between() + .gap_3() + .child(div().w_40().text_sm().child("Display name")) + .child( + div() + .flex_1() + .min_w(px(220.0)) + .child(Input::new(&state.read(cx).input).w_full()), + ) + .into_any_element() + }) +} + +fn collaboration_discovery_pause_item(_workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, _, _| { + let settings = crate::app_settings::load_app_settings(); + h_flex() + .w_full() + .items_center() + .justify_between() + .gap_4() + .child( + div() + .flex_1() + .min_w_0() + .child(div().text_sm().child("Pause trusted-peer discovery")), + ) + .child( + Checkbox::new("collaboration-discovery-paused") + .small() + .checked(settings.collaboration_discovery_paused) + .on_click(move |checked, _, cx| { + let current = crate::app_settings::load_app_settings(); + if let Err(error) = + crate::app_settings::save_collaboration_discovery_options(*checked, current.bluetooth_collaboration_discovery_enabled) + { + tracing::warn!(%error, "saving discovery pause setting failed"); + return; + } + crate::collab::reconfigure_discovery(cx); + }), + ) + .into_any_element() + }) +} + +fn trusted_collaborators_item(workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, window, cx| { + let state = window.use_keyed_state("trusted-collaborator-input", cx, { + let workspace = workspace.clone(); + move |window, cx| { + let identity = cx.new(|cx| InputState::new(window, cx).placeholder("Portable identity key")); + let name = cx.new(|cx| InputState::new(window, cx).placeholder("Person's name")); + let subscriptions = [&identity, &name] + .into_iter() + .map(|input| { + let workspace = workspace.clone(); + cx.subscribe_in(input, window, move |_, _, event: &InputEvent, _, cx| { + if matches!(event, InputEvent::Change) { + let _ = workspace.update(cx, |_, cx| cx.notify()); + } + }) + }) + .collect(); + TrustedCollaboratorInputState { + identity, + name, + compared: false, + _subscriptions: subscriptions, + } + } + }); + let identity_input = state.read(cx).identity.clone(); + let name_input = state.read(cx).name.clone(); + let identity_text = identity_input.read(cx).value().trim().to_string(); + let remote_key = identity_text.parse::().ok(); + let safety_code = remote_key.as_ref().and_then(|remote| { + crate::app_settings::load_local_identity_secret().map(|local| flowstate_collab::identity::safety_code(&local.public(), remote)) + }); + let active_path = workspace + .upgrade() + .and_then(|workspace| workspace.read(cx).active_editor.clone()) + .and_then(|editor| editor.read(cx).document_path().cloned()); + let settings = crate::app_settings::load_app_settings(); + let mut content = v_flex() + .w_full() + .gap_2() + .child(div().text_sm().child("Trusted people")); + if settings.trusted_collaborators.is_empty() { + content = content.child(div().text_xs().child("No verified people yet.")); + } + for (index, collaborator) in settings.trusted_collaborators.into_iter().enumerate() { + let identity_key = collaborator.identity_key.clone(); + let workspace = workspace.clone(); + content = content.child( + h_flex() + .items_center() + .justify_between() + .gap_2() + .child( + v_flex() + .min_w_0() + .child(div().text_sm().child(collaborator.display_name)) + .child(div().text_xs().text_ellipsis().child(identity_key.clone())), + ) + .child( + Button::new(("trusted-collaborator-revoke", index)) + .label("Revoke") + .xsmall() + .danger() + .on_click(move |_, _, cx| { + if let Err(error) = crate::app_settings::remove_trusted_collaborator(&identity_key) { + tracing::warn!(%error, "revoking trusted collaborator failed"); + return; + } + crate::collab::reconfigure_discovery(cx); + let _ = workspace.update(cx, |_, cx| cx.notify()); + }), + ), + ); + } + let state_for_check = state.clone(); + let state_for_add = state.clone(); + let workspace_for_check = workspace.clone(); + let workspace_for_add = workspace.clone(); + content + .child(div().h(px(1.0)).w_full().bg(cx.theme().border)) + .child( + h_flex() + .gap_2() + .child(Input::new(&name_input).w_full()) + .child(Input::new(&identity_input).w_full()), + ) + .when_some(safety_code, |this, code| { + this + .child(div().text_sm().child(format!("Safety code: {code}"))) + .child( + h_flex() + .items_center() + .justify_between() + .child( + div() + .text_xs() + .child("Compare this code with the other person over a separate channel."), + ) + .child( + Checkbox::new("trusted-collaborator-compared") + .small() + .checked(state.read(cx).compared) + .on_click(move |checked, _, cx| { + state_for_check.update(cx, |state, cx| { + state.compared = *checked; + cx.notify(); + }); + let _ = workspace_for_check.update(cx, |_, cx| cx.notify()); + }), + ), + ) + }) + .child( + h_flex().justify_end().child( + Button::new("trusted-collaborator-add") + .label("Trust for this document") + .small() + .primary() + .disabled(remote_key.is_none() || active_path.is_none() || !state.read(cx).compared) + .on_click(move |_, window, cx| { + let identity_key = state_for_add + .read(cx) + .identity + .read(cx) + .value() + .trim() + .to_string(); + let display_name = state_for_add + .read(cx) + .name + .read(cx) + .value() + .trim() + .to_string(); + let Some(path) = active_path.clone() else { return }; + if display_name.is_empty() || identity_key.parse::().is_err() { + return; + } + let collaborator = crate::app_settings::TrustedCollaborator { + identity_key, + display_name, + avatar_path: None, + color_rgb: None, + verified: true, + scopes: vec![crate::app_settings::CollaborationScope::Document(path)], + }; + if let Err(error) = crate::app_settings::save_trusted_collaborator(collaborator) { + tracing::warn!(%error, "saving trusted collaborator failed"); + return; + } + state_for_add.update(cx, |state, cx| { + state.compared = false; + state + .identity + .update(cx, |input, cx| input.set_value("", window, cx)); + state + .name + .update(cx, |input, cx| input.set_value("", window, cx)); + cx.notify(); + }); + crate::collab::reconfigure_discovery(cx); + let _ = workspace_for_add.update(cx, |_, cx| cx.notify()); + }), + ), + ) + .into_any_element() + }) +} + +fn collaboration_bluetooth_item(_workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, _, _| { + let settings = crate::app_settings::load_app_settings(); + h_flex() + .w_full() + .items_center() + .justify_between() + .gap_4() + .child( + div() + .flex_1() + .min_w_0() + .child(div().text_sm().child("Nearby Bluetooth discovery")) + .child( + div() + .text_xs() + .child("Optional. Flowstate only advertises a short-lived signed locator."), + ), + ) + .child( + Checkbox::new("collaboration-bluetooth-enabled") + .small() + .checked(settings.bluetooth_collaboration_discovery_enabled) + .on_click(move |checked, _, cx| { + let current = crate::app_settings::load_app_settings(); + if let Err(error) = crate::app_settings::save_collaboration_discovery_options(current.collaboration_discovery_paused, *checked) { + tracing::warn!(%error, "saving Bluetooth discovery setting failed"); + return; + } + crate::collab::reconfigure_discovery(cx); + }), + ) + .into_any_element() + }) +} + +fn collaboration_squads_item(workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, window, cx| { + let state = window.use_keyed_state("collaboration-squad-input", cx, |window, cx| CollaborationSquadInputState { + name: cx.new(|cx| InputState::new(window, cx).placeholder("Squad name")), + member: cx.new(|cx| InputState::new(window, cx).placeholder("Verified member identity key")), + }); + let name = state.read(cx).name.clone(); + let member = state.read(cx).member.clone(); + let active_folder = workspace + .upgrade() + .and_then(|workspace| workspace.read(cx).active_editor.clone()) + .and_then(|editor| { + editor + .read(cx) + .document_path() + .and_then(|path| path.parent()) + .map(Path::to_path_buf) + }); + let squads = crate::app_settings::load_app_settings().collaboration_squads; + let mut content = v_flex() + .w_full() + .gap_2() + .child(div().text_sm().child("Squads")); + if squads.is_empty() { + content = content.child( + div() + .text_xs() + .child("No squads yet. Squads share a default folder scope."), + ); + } + for (index, squad) in squads.into_iter().enumerate() { + let id = squad.id.clone(); + let workspace = workspace.clone(); + content = content.child( + h_flex() + .items_center() + .justify_between() + .child( + v_flex().child(div().text_sm().child(squad.name)).child( + div() + .text_xs() + .child(format!("{} member(s)", squad.member_identity_keys.len())), + ), + ) + .child( + Button::new(("collaboration-squad-remove", index)) + .label("Remove") + .xsmall() + .danger() + .on_click(move |_, _, cx| { + if let Err(error) = crate::app_settings::remove_collaboration_squad(&id) { + tracing::warn!(%error, "removing collaboration squad failed"); + return; + } + crate::collab::reconfigure_discovery(cx); + let _ = workspace.update(cx, |_, cx| cx.notify()); + }), + ), + ); + } + let state_for_add = state.clone(); + let workspace_for_add = workspace.clone(); + content + .child( + h_flex() + .gap_2() + .child(Input::new(&name).w_full()) + .child(Input::new(&member).w_full()), + ) + .child( + h_flex().justify_end().child( + Button::new("collaboration-squad-add") + .label("Create folder squad") + .small() + .outline() + .disabled(active_folder.is_none()) + .on_click(move |_, window, cx| { + let name = state_for_add + .read(cx) + .name + .read(cx) + .value() + .trim() + .to_string(); + let member = state_for_add + .read(cx) + .member + .read(cx) + .value() + .trim() + .to_string(); + let settings = crate::app_settings::load_app_settings(); + let member_is_verified = settings + .trusted_collaborators + .iter() + .any(|contact| contact.verified && contact.identity_key == member); + let Some(folder) = active_folder.clone() else { return }; + if name.is_empty() || !member_is_verified { + tracing::warn!("a squad requires a name and a safety-code-verified member"); + return; + } + let squad = crate::app_settings::CollaborationSquad { + id: Uuid::new_v4().to_string(), + name, + member_identity_keys: vec![member], + default_scopes: vec![crate::app_settings::CollaborationScope::Folder(folder)], + }; + if let Err(error) = crate::app_settings::save_collaboration_squad(squad) { + tracing::warn!(%error, "saving collaboration squad failed"); + return; + } + state_for_add.update(cx, |state, cx| { + state + .name + .update(cx, |input, cx| input.set_value("", window, cx)); + state + .member + .update(cx, |input, cx| input.set_value("", window, cx)); + cx.notify(); + }); + crate::collab::reconfigure_discovery(cx); + let _ = workspace_for_add.update(cx, |_, cx| cx.notify()); + }), + ), + ) + .into_any_element() + }) +} + +fn dropbox_connection_item(_workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, window, cx| { + let settings = crate::app_settings::load_app_settings().dropbox_collaboration; + let state = window.use_keyed_state("collaboration-dropbox-connection", cx, { + let app_key = settings.app_key.clone(); + let root = settings.root.clone(); + move |window, cx| DropboxConnectionInputState { + app_key: cx.new(|cx| { + InputState::new(window, cx) + .default_value(app_key) + .placeholder("Dropbox app key") + }), + root: cx.new(|cx| { + InputState::new(window, cx) + .default_value(root) + .placeholder("Optional app-folder path") + }), + } + }); + let app_key = state.read(cx).app_key.clone(); + let root = state.read(cx).root.clone(); + let connected = settings.enabled && !settings.access_token.is_empty(); + + v_flex() + .w_full() + .gap_2() + .child( + h_flex() + .gap_3() + .child(div().w_40().text_sm().child("App key")) + .child(Input::new(&app_key).w_full()), + ) + .child( + h_flex() + .gap_3() + .child(div().w_40().text_sm().child("Discovery root")) + .child(Input::new(&root).w_full()), + ) + .child( + h_flex() + .items_center() + .justify_between() + .child( + div() + .text_xs() + .child(if connected { "Connected" } else { "Not connected" }), + ) + .child(if connected { + Button::new("collaboration-dropbox-disconnect") + .label("Disconnect") + .small() + .outline() + .on_click(move |_, _, cx| { + if let Err(error) = crate::app_settings::disconnect_dropbox_collaboration() { + tracing::warn!(%error, "disconnecting Dropbox failed"); + return; + } + crate::collab::reconfigure_discovery(cx); + }) + } else { + Button::new("collaboration-dropbox-connect") + .label("Connect Dropbox") + .small() + .primary() + .on_click(move |_, _, cx| { + let app_key = app_key.read(cx).value().trim().to_string(); + let root = root.read(cx).value().trim().to_string(); + if let Err(error) = crate::app_settings::save_dropbox_connection_draft(app_key, root) + .and_then(|_| crate::collab::dropbox_oauth::begin(cx).map_err(std::io::Error::other)) + { + tracing::warn!(%error, "starting Dropbox connection failed"); + } + }) + }), + ) + .into_any_element() + }) +} + +fn dropbox_document_binding_item(workspace: WeakEntity) -> SettingItem { + SettingItem::render(move |_, _, cx| { + let path = workspace + .upgrade() + .and_then(|workspace| workspace.read(cx).active_editor.clone()) + .and_then(|editor| editor.read(cx).document_path().cloned()); + let binding = path + .as_deref() + .and_then(crate::app_settings::load_dropbox_document_binding); + let connected = crate::app_settings::load_dropbox_collaboration().is_some(); + let label = binding.as_ref().map_or_else( + || "This document is local only".to_string(), + |binding| format!("Linked to {}", binding.remote_path), + ); + let path_for_action = path.clone(); + let is_bound = binding.is_some(); + let workspace_for_action = workspace.clone(); + h_flex() + .w_full() + .items_center() + .justify_between() + .gap_3() + .child( + v_flex() + .min_w_0() + .child(div().text_sm().child("Active document")) + .child(div().text_xs().text_ellipsis().child(label)), + ) + .child( + Button::new("collaboration-dropbox-document-binding") + .label(if is_bound { "Unlink" } else { "Link to Dropbox" }) + .small() + .outline() + .disabled(path.is_none() || (!connected && !is_bound)) + .on_click(move |_, _, cx| { + let Some(path) = path_for_action.clone() else { return }; + let result = if is_bound { + crate::app_settings::remove_dropbox_document_binding(&path).map(|_| ()) + } else { + let settings = crate::app_settings::load_app_settings().dropbox_collaboration; + let Some(filename) = path.file_name().and_then(|name| name.to_str()) else { + tracing::warn!(path = %path.display(), "Dropbox link requires a Unicode filename"); + return; + }; + let root = settings.root.trim_matches('/'); + let remote_path = if root.is_empty() { + format!("/{filename}") + } else { + format!("/{root}/{filename}") + }; + crate::app_settings::save_dropbox_document_binding(crate::app_settings::DropboxDocumentBinding { + local_path: path, + remote_path, + revision: None, + }) + }; + if let Err(error) = result { + tracing::warn!(%error, "updating Dropbox document link failed"); + } + let _ = workspace_for_action.update(cx, |_, cx| cx.notify()); + }), + ) + .into_any_element() + }) +} diff --git a/crates/flowstate/src/workspace/workspace/documents.rs b/crates/flowstate/src/workspace/workspace/documents.rs index 7255ce76..c6b9b404 100644 --- a/crates/flowstate/src/workspace/workspace/documents.rs +++ b/crates/flowstate/src/workspace/workspace/documents.rs @@ -1,6 +1,117 @@ +enum DocumentRuntimeSource { + FromProjection, + Runtime(Box), + Attachment(DocumentRuntimeAttachment), +} + +/// Loro-first document wiring (spec §3): the write authority handed to the +/// editor (the ONE local write path, identical for solo and collaborative +/// documents) plus the I/O service handle for saves/exports/imports. +pub(crate) struct DocumentRuntimeAttachment { + pub authority: std::sync::Arc, + pub io: flowstate_collab::doc_io::DocIoHandle, +} + +/// Wrap a document core in the Loro-first services: the write gate, the +/// editor's write authority, and the background I/O service thread. +pub(crate) fn attach_local_write(core: flowstate_collab::crdt_runtime::CrdtRuntime) -> std::io::Result { + let (handle, gate) = flowstate_collab::local_write::LocalDocHandle::new(core, flowstate_collab::local_write::LocalWriteConfig::default()); + let io = flowstate_collab::doc_io::DocIoHandle::spawn(gate)?; + Ok(DocumentRuntimeAttachment { + authority: std::sync::Arc::new(handle), + io, + }) +} + +/// Install the write authority + native save/export/recovery I/O hooks onto an +/// editor (spec §5). Shared by the normal one-shot open (`create_document_panel`) +/// and the §act-three C phase-G attach (`attach_runtime_to_pending_panel`), so +/// both wire the editor identically. `set_write_authority` replaces the editor's +/// projection with the authority's canonical one, so a read-only phase-V editor +/// becomes editable and authoritative here. +fn install_editor_write_authority( + editor: &Entity, + attachment: &DocumentRuntimeAttachment, + document: DocumentProjection, + cx: &mut Context, +) { + let smart_word_selection = load_smart_word_selection(); + let save_io = attachment.io.clone(); + let export_io = attachment.io.clone(); + let recovery_io = attachment.io.clone(); + let authority = std::sync::Arc::clone(&attachment.authority); + editor.update(cx, |editor, cx| { + editor.set_smart_word_selection(smart_word_selection, cx); + // Loro-first (spec §5): the editor's write authority IS the document. + // Local intents commit synchronously; hooks below are pure I/O — no + // pending-edit drains, no projection replacement, no undo hook (undo + // executes through the authority). + editor.set_write_authority(authority, document, cx); + editor.set_native_save_hook(Some(Rc::new(move |path, _assets| { + let io = save_io.clone(); + Box::pin(async move { + let title = document_package_title_for_path(&path); + // The checkpoint's LocalUpdate events are intentionally dropped here: + // this UI-agnostic save hook future has no GPUI context to reach the + // collaboration session. The workspace re-syncs collaborators after + // the save completes via collab::refresh_after_external_checkpoint. + io.checkpoint_package(title.clone(), Some(path.clone())) + .await + .map_err(runtime_io_error)?; + if crate::app_settings::load_dropbox_document_binding(&path).is_some() { + let package = io + .package_bytes(title.clone()) + .await + .map_err(runtime_io_error)?; + crate::collab::dropbox_checkpoint::sync_bound_checkpoint(&path, title, &io, package).await?; + } + Ok(()) + }) + }))); + editor.set_native_export_hook(Some(Rc::new(move |path, format, _assets| { + let io = export_io.clone(); + Box::pin(async move { + match format { + crate::rich_text_element::DocumentExportFormat::Native | crate::rich_text_element::DocumentExportFormat::NativeWithExtension(_) => { + let bytes = io + .package_bytes(document_package_title_for_path(&path)) + .await + .map_err(runtime_io_error)?; + write_bytes_to_path(&path, &bytes)?; + }, + crate::rich_text_element::DocumentExportFormat::Docx => { + let document = io.projection_snapshot().await.map_err(runtime_io_error)?; + crate::docx_conversion::write_docx(&path, &document)?; + }, + crate::rich_text_element::DocumentExportFormat::Pdf => { + let document = io.projection_snapshot().await.map_err(runtime_io_error)?; + let bytes = io + .package_bytes("PDF Source".to_string()) + .await + .map_err(runtime_io_error)?; + crate::docx_conversion::write_pdf_with_db8_bytes(&path, &document, &bytes)?; + }, + }; + Ok(()) + }) + }))); + editor.set_native_recovery_hook(Some(Rc::new(move |path| { + let io = recovery_io.clone(); + Box::pin(async move { + let bytes = io + .package_bytes("Recovery snapshot".to_string()) + .await + .map_err(runtime_io_error)?; + write_bytes_to_path(&path, &bytes) + }) + }))); + }); +} + #[hotpath::measure_all] impl Workspace { pub fn new(initial_path: Option, window: &mut Window, cx: &mut Context) -> Self { + crate::collab::init(cx); let zoom_slider = cx.new(|_| { SliderState::new() .min(25.0) @@ -49,6 +160,9 @@ impl Workspace { let mut this = Self { document_panels: Vec::new(), + document_runtimes: FxHashMap::default(), // §perf: FxHash for trusted Uuid keys + pending_authority_panels: FxHashSet::default(), + document_runtime_flush_pending: FxHashSet::default(), // §perf: FxHash for trusted Uuid keys flow_panels: Vec::new(), active_document_id: None, active_editor: None, @@ -66,8 +180,8 @@ impl Workspace { tab_bar_scroll_handle: ScrollHandle::new(), pinned_document_ids: Vec::new(), speech_document_id: None, - speech_word_count_cache: HashMap::new(), - speech_word_count_pending: HashSet::new(), + speech_word_count_cache: FxHashMap::default(), // §perf: FxHash for trusted Uuid keys + speech_word_count_pending: FxHashSet::default(), // §perf: FxHash for trusted Uuid keys body_resizable_state: cx.new(|_| ResizableState::default()), content_resizable_state: cx.new(|_| ResizableState::default()), ribbon_resizable_state: cx.new(|_| ResizableState::default()), @@ -86,8 +200,14 @@ impl Workspace { document_style_section: DocumentStyleSection::Text, settings_section: WorkspaceSettingsSection::General, autosave_enabled: load_autosave(), - autosave_document_generations: HashMap::new(), - autosave_flow_in_flight: HashSet::new(), + autosave_document_generations: FxHashMap::default(), // §perf: FxHash for trusted Uuid keys + autosave_pending_generation: FxHashMap::default(), // §act-five P9-throttle debounce + autosave_flow_in_flight: FxHashSet::default(), // §perf: FxHash for trusted Uuid keys + collaboration_dialog: None, + revision_dialog: None, + comment_dialog: None, + collab_notice_subscriptions: FxHashMap::default(), // §perf: FxHash for trusted SessionId keys + collab_incompatible_version_notices: HashSet::new(), file_search_overlay: None, tub_root: None, tub_index: None, @@ -143,20 +263,75 @@ impl Workspace { fn create_document_panel( &mut self, - mut document: Document, + mut document: DocumentProjection, path: Option, title: Option, + runtime: DocumentRuntimeSource, window: &mut Window, cx: &mut Context, - ) -> Entity { + ) -> anyhow::Result> { // DB8 stores style assignments, not style appearance. The render theme is // local user preference loaded from app settings. document.theme = load_document_theme(); - let editor = cx.new(|cx| RichTextEditor::new_with_path(document, path.clone(), cx)); - let smart_word_selection = load_smart_word_selection(); - editor.update(cx, |editor, cx| { - editor.set_smart_word_selection(smart_word_selection, cx); - }); + let runtime_title = title + .as_deref() + .or_else(|| { + path + .as_deref() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + }) + .unwrap_or("Flowstate Document"); + // §15/§31: freshly spawned runtimes get the durable author identity bound + // below. A `Handle` is an already-live collaboration runtime that registered + // its identity at join time (see `CollabSession::finish_join_snapshot`), so + // re-binding it here would be a redundant second call on the same runtime. + let mut register_author_identity = true; + // Loro-first (spec §3): every document — solo or collaborative — gets the + // SAME wiring: a gate-protected core, the editor's write authority + // (LocalDocHandle), and the background I/O service (DocIoHandle). + let attachment = match runtime { + DocumentRuntimeSource::FromProjection => { + let imported = flowstate_document::import_document_projection(document.clone(), runtime_title) + .map_err(|error| anyhow::anyhow!("creating canonical Loro document failed: {error}"))?; + let core = flowstate_collab::crdt_runtime::CrdtRuntime::from_imported_document(imported) + .map_err(|error| anyhow::anyhow!("creating canonical Loro runtime failed: {error:#}"))?; + attach_local_write(core).map_err(|error| anyhow::anyhow!("starting Loro-first document services failed: {error:#}"))? + }, + DocumentRuntimeSource::Runtime(runtime) => { + attach_local_write(*runtime).map_err(|error| anyhow::anyhow!("starting Loro-first document services failed: {error:#}"))? + }, + DocumentRuntimeSource::Attachment(attachment) => { + register_author_identity = false; + attachment + }, + }; + let local_theme = document.theme.clone(); + document = attachment + .authority + .projection() + .map_err(|error| anyhow::anyhow!("reading canonical startup projection failed: {error}"))?; + document.theme = local_theme; + + if register_author_identity { + // Fire-and-forget: a failure to register author identity must never block + // or break opening the document. The settings read/persist happens on a + // background thread to keep the foreground non-blocking. + let identity_io = attachment.io.clone(); + cx.spawn(async move |_, cx| { + let (user_id, display_name) = cx + .background_executor() + .spawn(async { load_local_user_identity() }) + .await; + if let Err(error) = identity_io.set_author_identity(user_id, display_name).await { + tracing::warn!(error = %format_args!("{error:#}"), "binding durable author identity to document runtime failed"); + } + }) + .detach(); + } + + let editor = cx.new(|cx| RichTextEditor::new_with_path(document.clone(), path.clone(), cx)); + install_editor_write_authority(&editor, &attachment, document, cx); let workspace = cx.entity().downgrade(); let title = title .or_else(|| { @@ -173,9 +348,13 @@ impl Workspace { } let panel = cx.new(|cx| DocumentPanel::new_with_title(title, path, editor.clone(), workspace, window, cx)); let id = panel.read(cx).id(); + self.document_runtimes.insert(id, attachment.io.clone()); self.editor_subscriptions.push(( id, cx.observe(&editor, move |workspace, editor, cx| { + // Loro-first: no runtime flush — intents commit synchronously in the + // editor's write authority. The observation drives view-model updates + // and autosave only. let viewport_paragraph = workspace.active_editor_viewport_paragraph(cx); workspace.update_outline_viewport_paragraph(viewport_paragraph, cx); workspace.maybe_autosave_document(id, editor.clone(), cx); @@ -185,7 +364,107 @@ impl Workspace { self.active_editor = Some(editor); self.active_flow = None; self.document_panels.push(panel.clone()); - panel + Ok(panel) + } + + pub fn request_document_revisions( + &self, + panel_id: Uuid, + cx: &mut Context, + ) -> Option>>> { + let runtime = self.document_runtimes.get(&panel_id)?.clone(); + let (tx, rx) = async_channel::bounded(1); + cx.spawn(async move |_, _| { + let _ = tx.send(runtime.revisions().await).await; + }) + .detach(); + Some(rx) + } + + pub fn open_revision_dialog(&mut self, window: &mut Window, cx: &mut Context) { + let Some(panel_id) = self.active_document_id else { + return; + }; + if self.revision_dialog.is_some() { + window.close_dialog(cx); + self.revision_dialog = None; + } + let workspace = cx.entity().downgrade(); + let revisions = self.request_document_revisions(panel_id, cx); + let dialog = cx.new(|cx| crate::workspace::revision_dialog::RevisionDialog::new(workspace, panel_id, revisions, cx)); + let dialog_for_render = dialog.clone(); + let workspace_for_close = cx.entity().downgrade(); + window.open_dialog(cx, move |component_dialog, _, _| { + let workspace_for_close = workspace_for_close.clone(); + component_dialog + .title("Document Revisions") + .w(px(520.0)) + .max_w(px(520.0)) + .on_close(move |_, _, cx| { + let _ = workspace_for_close.update(cx, |workspace, cx| { + workspace.revision_dialog = None; + cx.notify(); + }); + }) + .child(dialog_for_render.clone()) + }); + dialog.update(cx, |dialog, cx| dialog.focus(window, cx)); + self.revision_dialog = Some(dialog); + cx.notify(); + } + + pub fn open_document_revision(&mut self, panel_id: Uuid, revision_id: u128, window: &mut Window, cx: &mut Context) -> bool { + let Some(runtime) = self.document_runtimes.get(&panel_id).cloned() else { + return false; + }; + let window_handle = window.window_handle(); + cx.spawn(async move |workspace, cx| { + let result = runtime.fork_revision(revision_id).await; + let _ = window_handle.update(cx, |_, window, cx| { + let _ = workspace.update(cx, |workspace, cx| match result { + Ok(events) => { + let fork = events.into_iter().find_map(|event| match event { + flowstate_collab::crdt_runtime::RuntimeEvent::RevisionForked { document, package, .. } => Some((*document, *package)), + _ => None, + }); + let Some((document, package)) = fork else { + tracing::warn!(revision_id, "revision fork command returned no fork payload"); + return; + }; + let fork_runtime = match flowstate_collab::crdt_runtime::CrdtRuntime::from_package(package, None) { + Ok(runtime) => runtime, + Err(error) => { + tracing::error!(revision_id, error = %format_args!("{error:#}"), "creating revision fork runtime failed"); + return; + }, + }; + let panel = match workspace.create_document_panel( + document, + None, + Some(format!("Revision {revision_id:x}")), + DocumentRuntimeSource::Runtime(Box::new(fork_runtime)), + window, + cx, + ) { + Ok(panel) => panel, + Err(error) => { + tracing::error!(revision_id, error = %format_args!("{error:#}"), "starting revision fork runtime failed"); + return; + }, + }; + panel + .read(cx) + .editor() + .update(cx, |editor, cx| editor.mark_as_unsaved_branch(cx)); + }, + Err(error) => { + tracing::error!(revision_id, error = %format_args!("{error:#}"), "opening document revision failed"); + }, + }); + }); + }) + .detach(); + true } pub fn set_active_document(&mut self, panel_id: Uuid, editor: Entity, cx: &mut Context) { @@ -214,6 +493,24 @@ impl Workspace { } pub fn remove_document_panel(&mut self, panel_id: Uuid, _: &mut Window, cx: &mut Context) { + crate::collab::leave_session_for_panel(panel_id, cx); + // §P3 (act two): flush the projection/search caches into the package on + // close (revision-less, off-thread) so the NEXT preview/open of this file + // hits the cache fast path — every edit nulls the cache, so a closed + // edited document otherwise previews via a multi-second full Loro + // materialization until an explicit save. + if let Some(io) = self.document_runtimes.get(&panel_id) { + let io = io.clone(); + cx.background_executor() + .spawn(async move { + if let Err(error) = io.flush_package_caches().await { + tracing::debug!(%error, "close-time package cache flush failed"); + } + }) + .detach(); + } + self.document_runtimes.remove(&panel_id); + self.document_runtime_flush_pending.remove(&panel_id); let closing_active_document = self.active_document_id == Some(panel_id); if let Some(panel) = self .document_panels @@ -309,16 +606,65 @@ impl Workspace { let path_for_recent = path.clone(); cx.spawn(async move |workspace, cx| { let path_for_error = path.clone(); + // §act-three C phase V: a cheap cached projection paints a read-only panel + // immediately, while phase G (the full Loro import + runtime) loads off + // thread. Editing enables when the authority attaches (phase G below). + let phase_v_path = path.clone(); + let cached = cx + .background_executor() + .spawn(async move { read_open_projection_fast(&phase_v_path) }) + .await; + let pending_panel_id = if let Some(projection) = cached { + let scroll_target = target_paragraph_ix; + window_handle + .update(cx, |_, window, cx| { + workspace + .update(cx, |workspace, cx| { + let id = workspace.create_pending_document_panel(projection, Some(path_for_recent.clone()), None, window, cx); + if let Some(paragraph_ix) = scroll_target { + workspace.scroll_active_editor_to_paragraph(paragraph_ix, window, cx); + } + id + }) + .ok() + }) + .ok() + .flatten() + } else { + None + }; + let loaded = cx .background_executor() .spawn(async move { load_workspace_document(path) }) .await; match loaded { - Ok(LoadedWorkspaceDocument::Document { document, path, title }) => { + Ok(LoadedWorkspaceDocument::Document { + document, + runtime, + path, + title, + }) => { let _ = window_handle.update(cx, |_, window, cx| { let _ = workspace.update(cx, |workspace, cx| { + workspace + .recent_document_previews + .insert(path_for_recent.clone(), recent_document_preview_document(&document)); workspace.record_recent_document(path_for_recent.clone(), cx); - workspace.add_document_panel_with_title(*document, path, title, window, cx); + // Phase G: attach the runtime to the phase-V panel if it is still + // open; otherwise (no cache existed, or the pending panel was + // closed while loading) open a fresh full panel. `attach` gives + // the runtime back when it cannot use it. + let fresh_runtime = match pending_panel_id { + Some(id) => workspace + .attach_runtime_to_pending_panel(id, *runtime, path.clone(), cx) + .err() + .map(|boxed| *boxed), + None => Some(*runtime), + }; + if let Some(runtime) = fresh_runtime { + workspace.add_document_panel_with_title(*document, path, title, runtime, window, cx); + } if let Some(paragraph_ix) = target_paragraph_ix { workspace.scroll_active_editor_to_paragraph(paragraph_ix, window, cx); cx.on_next_frame(window, move |workspace, window, cx| { @@ -464,15 +810,21 @@ impl Workspace { let id = match loaded { LoadedWorkspaceDocument::Document { document, + runtime, path, title, } => { - let panel = self.create_document_panel(*document, path, title, window, cx); + let panel = match self.create_document_panel(*document, path, title, DocumentRuntimeSource::Runtime(runtime), window, cx) { + Ok(panel) => panel, + Err(error) => { + tracing::error!(error = %format_args!("{error:#}"), "restoring document runtime failed"); + continue; + }, + }; let panel_id = panel.read(cx).id(); panel.update(cx, |panel, _| { if !entry.collapsed_outline_items.is_empty() { - panel.collapsed_outline_items = - Some(entry.collapsed_outline_items.iter().copied().collect()); + panel.collapsed_outline_items = Some(entry.collapsed_outline_items.iter().copied().collect()); } panel.outline_scrolled_paragraph = entry.outline_scrolled_paragraph; }); @@ -548,9 +900,9 @@ impl Workspace { .spawn({ let path = path.clone(); async move { - let mut loaded = load_document_for_open(&path).ok()?; - loaded.document.theme = load_document_theme(); - Some(recent_document_preview_document(&loaded.document)) + let mut document = load_document_preview(&path).ok()?; + document.theme = load_document_theme(); + Some(recent_document_preview_document(&document)) } }) .await; @@ -609,23 +961,52 @@ impl Workspace { .detach(); } - fn add_document_panel(&mut self, document: Document, path: Option, window: &mut Window, cx: &mut Context) { - self.create_document_panel(document, path, None, window, cx); - self.persist_temporary_workspace_session(cx); - cx.notify(); + fn add_document_panel(&mut self, document: DocumentProjection, path: Option, window: &mut Window, cx: &mut Context) { + match self.create_document_panel(document, path, None, DocumentRuntimeSource::FromProjection, window, cx) { + Ok(_) => { + self.persist_temporary_workspace_session(cx); + cx.notify(); + }, + Err(error) => { + let detail = format!("The canonical Loro runtime could not be started: {error:#}"); + tracing::error!(error = %format_args!("{error:#}"), "creating document panel failed"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Document could not be opened", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + }, + } } fn add_document_panel_with_title( &mut self, - document: Document, + document: DocumentProjection, path: Option, title: Option, + runtime: flowstate_collab::crdt_runtime::CrdtRuntime, window: &mut Window, cx: &mut Context, ) { - self.create_document_panel(document, path, title, window, cx); - self.persist_temporary_workspace_session(cx); - cx.notify(); + match self.create_document_panel(document, path, title, DocumentRuntimeSource::Runtime(Box::new(runtime)), window, cx) { + Ok(_) => { + self.persist_temporary_workspace_session(cx); + cx.notify(); + }, + Err(error) => { + let detail = format!("The canonical Loro runtime could not be started: {error:#}"); + tracing::error!(error = %format_args!("{error:#}"), "creating document panel failed"); + std::mem::drop(window.prompt( + PromptLevel::Critical, + "Document could not be opened", + Some(&detail), + &[PromptButton::ok("Ok")], + cx, + )); + }, + } } fn create_flow_panel( @@ -667,6 +1048,141 @@ impl Workspace { cx.notify(); } + /// §act-three C (phase V): open a panel painting a cheap cached projection + /// with NO write authority — a read-only display surface (scroll/select/copy + /// work; editing is inert). The authority runtime attaches later via + /// [`Self::attach_runtime_to_pending_panel`] (phase G). Returns the panel id + /// so the caller can complete the attach when the runtime finishes loading. + /// + /// Session-persist and autosave skip pending panels (a read-only phase-V + /// panel has no runtime to save); both fire on attach. Because the panel is + /// read-only until attach, NO local edit can be made — so none can be lost in + /// the window (the fidelity-safe form of the spec's phase-V open). + fn create_pending_document_panel( + &mut self, + mut document: DocumentProjection, + path: Option, + title: Option, + window: &mut Window, + cx: &mut Context, + ) -> Uuid { + document.theme = load_document_theme(); + let editor = cx.new(|cx| RichTextEditor::new_with_path(document, path.clone(), cx)); + let smart_word_selection = load_smart_word_selection(); + editor.update(cx, |editor, cx| { + editor.set_smart_word_selection(smart_word_selection, cx); + // No `set_write_authority` — the editor stays a read-only display surface + // (editor/local_write_path.rs: "No authority ⇒ read-only display surface"). + }); + let workspace = cx.entity().downgrade(); + let title = title + .or_else(|| { + path + .as_ref() + .and_then(|path| path.file_name()) + .map(|name| name.to_string_lossy().to_string()) + }) + .or_else(|| Some(self.next_untitled_title(cx))); + if let Some(title) = title.clone() { + editor.update(cx, |editor, cx| { + editor.set_document_display_name(title.into(), cx); + }); + } + let panel = cx.new(|cx| DocumentPanel::new_with_title(title, path, editor.clone(), workspace, window, cx)); + let id = panel.read(cx).id(); + self.pending_authority_panels.insert(id); + self.active_document_id = Some(id); + self.active_editor = Some(editor); + self.active_flow = None; + self.document_panels.push(panel); + cx.notify(); + id + } + + /// §act-three C (phase G): attach the freshly-loaded runtime to a panel + /// opened read-only by [`Self::create_pending_document_panel`]. Installs the + /// write authority + I/O hooks (making the editor editable and swapping in + /// the runtime's canonical projection), registers the runtime, wires the + /// autosave observation, and binds the durable author identity — exactly the + /// wiring the one-shot open does. Returns `Ok(())` when attached (the runtime + /// is consumed). Returns `Err(runtime)` — handing the runtime back — when the + /// pending panel was closed before the runtime arrived, so the caller opens a + /// fresh full panel instead. + fn attach_runtime_to_pending_panel( + &mut self, + panel_id: Uuid, + runtime: flowstate_collab::crdt_runtime::CrdtRuntime, + canonical_path: Option, + cx: &mut Context, + ) -> Result<(), Box> { + // The panel closed while loading — hand the runtime back for a fresh open. + let Some(panel) = self + .document_panels + .iter() + .find(|panel| panel.read(cx).id() == panel_id) + .filter(|_| self.pending_authority_panels.contains(&panel_id)) + .cloned() + else { + self.pending_authority_panels.remove(&panel_id); + return Err(Box::new(runtime)); + }; + // From here the panel exists; any failure leaves it read-only (degraded but + // safe) rather than opening a duplicate — attach failures are rare hard + // faults (I/O thread spawn / projection read). + self.pending_authority_panels.remove(&panel_id); + let attachment = match attach_local_write(runtime) { + Ok(attachment) => attachment, + Err(error) => { + tracing::error!(error = %format_args!("{error:#}"), "attaching Loro-first services to pending panel failed; leaving panel read-only"); + return Ok(()); + }, + }; + // Adopt the authority's canonical startup projection (the runtime advanced + // the frontier during construction; the editor must start from it). + let mut document = match attachment.authority.projection() { + Ok(document) => document, + Err(error) => { + tracing::error!(error = %format_args!("{error:#}"), "reading canonical startup projection for pending panel failed; leaving panel read-only"); + return Ok(()); + }, + }; + document.theme = load_document_theme(); + let editor = panel.read(cx).editor(); + install_editor_write_authority(&editor, &attachment, document, cx); + // §data-loss fix (2026-07-08): the pending panel seeded the editor with the + // SOURCE path for phase-V display/recents. Now that the authority is + // attached and the editor becomes writable, reconcile its write path to the + // authoritative open path — `None` for imported docx/pdf, so autosave never + // targets (and overwrites) the source file. Mirrors the one-shot open path. + editor.update(cx, |editor, cx| editor.set_runtime_document_path(canonical_path, cx)); + self + .document_runtimes + .insert(panel_id, attachment.io.clone()); + self.editor_subscriptions.push(( + panel_id, + cx.observe(&editor, move |workspace, editor, cx| { + let viewport_paragraph = workspace.active_editor_viewport_paragraph(cx); + workspace.update_outline_viewport_paragraph(viewport_paragraph, cx); + workspace.maybe_autosave_document(panel_id, editor.clone(), cx); + }), + )); + // Bind the durable author identity (fire-and-forget; never blocks open). + let identity_io = attachment.io.clone(); + cx.spawn(async move |_, cx| { + let (user_id, display_name) = cx + .background_executor() + .spawn(async { load_local_user_identity() }) + .await; + if let Err(error) = identity_io.set_author_identity(user_id, display_name).await { + tracing::warn!(error = %format_args!("{error:#}"), "binding durable author identity to attached document runtime failed"); + } + }) + .detach(); + self.persist_temporary_workspace_session(cx); + cx.notify(); + Ok(()) + } + pub fn close_document_panel(&mut self, panel_id: Uuid, window: &mut Window, cx: &mut Context) { let document_panel = self .document_panels @@ -692,6 +1208,9 @@ impl Workspace { else { return; }; + if self.close_collaboration_document_panel(panel_id, panel_kind.clone(), window, cx) { + return; + } if !panel_kind.is_dirty(cx) { self.remove_document_panel(panel_id, window, cx); return; @@ -732,8 +1251,12 @@ impl Workspace { } fn request_close_window(&mut self, window: &mut Window, cx: &mut Context) { + if self.request_close_window_with_collaboration(window, cx) { + return; + } let dirty_panels = self.dirty_panels(cx); if dirty_panels.is_empty() { + self.leave_all_collaboration_sessions(cx); window.remove_window(); return; } @@ -752,7 +1275,7 @@ impl Workspace { ); let window_handle = window.window_handle(); - cx.spawn(async move |_, cx| { + cx.spawn(async move |workspace, cx| { let should_close = match answer.await { Ok(0) => { let mut ok = true; @@ -782,7 +1305,10 @@ impl Workspace { }; if should_close { - let _ = window_handle.update(cx, |_, window, _| window.remove_window()); + let _ = window_handle.update(cx, |_, window, cx| { + let _ = workspace.update(cx, |workspace, cx| workspace.leave_all_collaboration_sessions(cx)); + window.remove_window(); + }); } }) .detach(); @@ -794,14 +1320,28 @@ impl Workspace { self.prompt_save_active_as(editor, window, cx); return; } + let panel_id = self.active_document_id; let save_task = editor.update(cx, |editor, cx| editor.save(cx)); let window_handle = window.window_handle(); - cx.spawn(async move |_, cx| { - if let Err(error) = save_task.await { - let detail = error.to_string(); - let _ = window_handle.update(cx, |_, window, cx| { - window.prompt(PromptLevel::Critical, "Save failed", Some(&detail), &[PromptButton::ok("Ok")], cx) - }); + cx.spawn(async move |workspace, cx| { + match save_task.await { + // A successful save checkpoints the runtime, recording a named + // revision into Loro. For a collaborating document that op must be + // advertised so peers converge promptly; for a solo document the + // refresh is a no-op because no session is registered for the panel. + Ok(()) => { + if let Some(panel_id) = panel_id { + let _ = workspace.update(cx, |_, cx| { + crate::collab::refresh_after_external_checkpoint(panel_id, cx); + }); + } + }, + Err(error) => { + let detail = error.to_string(); + let _ = window_handle.update(cx, |_, window, cx| { + window.prompt(PromptLevel::Critical, "Save failed", Some(&detail), &[PromptButton::ok("Ok")], cx) + }); + }, } }) .detach(); @@ -874,21 +1414,56 @@ impl Workspace { if !has_path || !has_unsaved_changes { return; } - if self.autosave_document_generations.get(&panel_id) == Some(&generation) { + // Already saved (or already scheduled) for this exact generation? Skip. + if self.autosave_document_generations.get(&panel_id) == Some(&generation) + || self.autosave_pending_generation.get(&panel_id) == Some(&generation) + { return; } + // §act-five P9-throttle: DEBOUNCE. Every keystroke bumps the generation and + // used to fire a full checkpoint (document_from_loro + search reindex + + // snapshot) — pegging the off-gate package thread during typing/collab. + // Record the latest pending generation and schedule a TRAILING save; a newer + // edit overwrites `autosave_pending_generation`, so only the last edit of a + // burst survives the debounce and ONE checkpoint runs per quiet period. No + // edit is lost: the trailing timer always fires for the final generation, and + // close-time `flush_package_caches` covers the tail. + const AUTOSAVE_DEBOUNCE: std::time::Duration = std::time::Duration::from_millis(900); self - .autosave_document_generations + .autosave_pending_generation .insert(panel_id, generation); - let save_task = editor.update(cx, |editor, cx| editor.save(cx)); cx.spawn(async move |workspace, cx| { - if let Err(error) = save_task.await { - eprintln!("autosave failed: {error}"); - let _ = workspace.update(cx, |workspace, _| { - if workspace.autosave_document_generations.get(&panel_id) == Some(&generation) { - workspace.autosave_document_generations.remove(&panel_id); - } - }); + cx.background_executor().timer(AUTOSAVE_DEBOUNCE).await; + // Superseded by a newer edit within the debounce window? Let that one save. + let save_task = workspace.update(cx, |workspace, cx| { + if workspace.autosave_pending_generation.get(&panel_id) != Some(&generation) { + return None; + } + workspace.autosave_pending_generation.remove(&panel_id); + workspace + .autosave_document_generations + .insert(panel_id, generation); + Some(editor.update(cx, |editor, cx| editor.save(cx))) + }); + let Ok(Some(save_task)) = save_task else { + return; + }; + match save_task.await { + // Keep collaborating peers in sync with the autosave checkpoint; a + // no-op for solo documents (no session registered for the panel). + Ok(()) => { + let _ = workspace.update(cx, |_, cx| { + crate::collab::refresh_after_external_checkpoint(panel_id, cx); + }); + }, + Err(error) => { + eprintln!("autosave failed: {error}"); + let _ = workspace.update(cx, |workspace, _| { + if workspace.autosave_document_generations.get(&panel_id) == Some(&generation) { + workspace.autosave_document_generations.remove(&panel_id); + } + }); + }, } }) .detach(); @@ -951,6 +1526,9 @@ impl Workspace { panel.update(cx, |panel, cx| panel.set_path(path, cx)); } workspace.persist_temporary_workspace_session(cx); + // Advertise the save checkpoint to collaborators (no-op when the + // panel has no active collaboration session). + crate::collab::refresh_after_external_checkpoint(panel_id, cx); cx.notify(); }); }, @@ -1152,9 +1730,31 @@ fn show_save_failed(window_handle: AnyWindowHandle, cx: &mut gpui::AsyncApp, det }); } +fn document_package_title_for_path(path: &Path) -> String { + path + .file_name() + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_else(|| "Flowstate Document".to_string()) +} + +fn write_bytes_to_path(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent)?; + } + atomicwrites::AtomicFile::new(path, atomicwrites::AllowOverwrite) + .write(|file| std::io::Write::write_all(file, bytes)) + .map_err(Into::into) +} + +/// Stable short tag for a runtime event variant, used only by fidelity firehose +/// lines so a document-runtime commit stream reads which transition was applied. enum LoadedWorkspaceDocument { Document { - document: Box, + document: Box, + runtime: Box, path: Option, title: Option, }, @@ -1175,6 +1775,7 @@ fn load_workspace_document(path: PathBuf) -> Result PathBuf { - std::env::temp_dir().join("flowstate-open-tabs-session.json") + // FLOWSTATE_CONFIG_DIR is the headless-test sandbox root; the tab-session + // file must follow it or tests restore/clobber the real user's open tabs. + std::env::var_os("FLOWSTATE_CONFIG_DIR") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir) + .join("flowstate-open-tabs-session.json") } #[hotpath::measure] @@ -1241,7 +1847,7 @@ fn persist_temporary_workspace_session_to_disk(session: TemporaryWorkspaceSessio match serde_json::to_vec(&session) { Ok(bytes) => { - if let Err(error) = fs::write(&path, bytes) { + if let Err(error) = write_bytes_to_path(&path, &bytes) { eprintln!("failed to write temporary workspace session {}: {error}", path.display()); } }, @@ -1252,19 +1858,19 @@ fn persist_temporary_workspace_session_to_disk(session: TemporaryWorkspaceSessio } #[hotpath::measure] -fn recent_document_preview_document(document: &Document) -> Document { +fn recent_document_preview_document(document: &DocumentProjection) -> DocumentProjection { const MAX_PARAGRAPHS: usize = 12; const MAX_CHARS: usize = 2_200; let mut remaining_chars = MAX_CHARS; let mut paragraphs = Vec::new(); - for paragraph in document.paragraphs.iter().take(MAX_PARAGRAPHS) { + for (paragraph_ix, paragraph) in document.paragraphs.iter().take(MAX_PARAGRAPHS).enumerate() { if remaining_chars == 0 { break; } - let mut run_start = paragraph.byte_range.start; + let mut run_start = flowstate_document::paragraph_byte_range(document, paragraph_ix).start; let mut runs = Vec::new(); for run in ¶graph.runs { if remaining_chars == 0 { diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/actions.rs b/crates/flowstate/src/workspace/workspace/headless_tests/actions.rs new file mode 100644 index 00000000..a4e629d0 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/actions.rs @@ -0,0 +1,74 @@ +//! Command dispatch through the window-keybinding surface — the same switch +//! real keystrokes reach via the interceptor installed in `Workspace::new`. + +use gpui::TestAppContext; + +use crate::commands::CommandId; + +use super::support; + +#[gpui::test] +fn new_document_command_creates_panel(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + let handled = h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::NewDocument, window, cx)); + cx.run_until_parked(); + assert!(handled); + assert_eq!(h.read(cx, |ws| ws.document_panels.len()), 1); +} + +#[gpui::test] +fn share_command_opens_dialog_over_document(cx: &mut TestAppContext) { + // The exact shape of the field crash: a keybinding-dispatched ShareDocument + // command entering open_collaboration_dialog inside the workspace update. + let h = support::open_workspace(cx); + h.new_document(cx); + let handled = h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::ShareDocument, window, cx)); + cx.run_until_parked(); + assert!(handled); + assert!(h.read(cx, |ws| ws.collaboration_dialog.is_some())); +} + +#[gpui::test] +fn join_command_opens_dialog_without_document(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + let handled = h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::JoinSession, window, cx)); + cx.run_until_parked(); + assert!(handled); + assert!(h.read(cx, |ws| ws.collaboration_dialog.is_some())); +} + +#[gpui::test] +fn zoom_commands_require_an_editor(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + assert!(!h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::ZoomIn, window, cx))); + assert!(!h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::ZoomOut, window, cx))); + h.new_document(cx); + assert!(h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::ZoomIn, window, cx))); + assert!(h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::ZoomOut, window, cx))); +} + +#[gpui::test] +fn start_collaboration_command_without_document_is_unhandled(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + let handled = h.update(cx, |ws, window, cx| { + ws.handle_window_keybinding(CommandId::StartCollaboration, window, cx) + }); + assert!(!handled, "no active document — command must fall through"); +} + +#[gpui::test] +fn close_document_command_with_no_document_is_safe(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::CloseDocument, window, cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.document_panels.is_empty())); +} + +#[gpui::test] +fn find_in_document_command_survives_with_and_without_document(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::FindInDocument, window, cx)); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.handle_window_keybinding(CommandId::FindInDocument, window, cx)); + cx.run_until_parked(); +} diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/collab_glue.rs b/crates/flowstate/src/workspace/workspace/headless_tests/collab_glue.rs new file mode 100644 index 00000000..17b48544 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/collab_glue.rs @@ -0,0 +1,52 @@ +//! Collaboration glue above the session layer: clipboard join failures land +//! as prompts (not panics), leave is a no-op without a session, and the full +//! host-session start/leave round trip works headlessly. + +use gpui::{ClipboardItem, TestAppContext}; + +use super::support; + +#[gpui::test] +fn join_from_empty_clipboard_prompts(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + let handled = h.update(cx, |ws, window, cx| ws.join_collaboration_from_clipboard(window, cx)); + assert!(handled); + cx.run_until_parked(); + assert!(cx.has_pending_prompt(), "empty clipboard must surface a Join failed prompt"); + cx.simulate_prompt_answer("Ok"); + cx.run_until_parked(); +} + +#[gpui::test] +fn join_from_garbage_clipboard_prompts(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + cx.write_to_clipboard(ClipboardItem::new_string("definitely not a flowstate invite".into())); + let handled = h.update(cx, |ws, window, cx| ws.join_collaboration_from_clipboard(window, cx)); + assert!(handled); + cx.run_until_parked(); + assert!(cx.has_pending_prompt(), "invalid ticket must surface a Join failed prompt"); + cx.simulate_prompt_answer("Ok"); + cx.run_until_parked(); +} + +#[gpui::test] +fn leave_collaboration_without_session_is_a_noop(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + assert!(!h.update(cx, |ws, window, cx| ws.confirm_leave_collaboration_on_active_document(window, cx))); + assert!(!h.update(cx, |ws, _, cx| ws.leave_collaboration_on_active_document(cx))); +} + +#[gpui::test] +fn start_and_leave_collaboration_session_round_trip(cx: &mut TestAppContext) { + // End-to-end through the real CollabManager: starts the local network + // runtime (local socket bind only; discovery is paused by the sandbox). + let h = support::open_workspace(cx); + h.new_document(cx); + let session = h.update(cx, |ws, _, cx| ws.start_collaboration_on_active_document(cx)); + assert!(session.is_some(), "hosting a session on a fresh document must succeed"); + cx.run_until_parked(); + let left = h.update(cx, |ws, _, cx| ws.leave_collaboration_on_active_document(cx)); + assert!(left, "leaving the session we just started must succeed"); + cx.run_until_parked(); +} diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/dialogs.rs b/crates/flowstate/src/workspace/workspace/headless_tests/dialogs.rs new file mode 100644 index 00000000..bb7139ec --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/dialogs.rs @@ -0,0 +1,81 @@ +//! Dialog lifecycle under the real workspace wiring. The share-dialog test is +//! the regression net for the field panic where the dialog constructor called +//! `workspace.update` while the Workspace lease was held (GPUI double lease). + +use gpui::TestAppContext; + +use super::support; + +#[gpui::test] +fn share_dialog_opens_over_active_document(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.open_collaboration_dialog(window, cx)); + // Flush the deferred trusted-peer scan — the old double-lease panic site. + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.collaboration_dialog.is_some())); +} + +#[gpui::test] +fn share_dialog_opens_with_no_document(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.update(cx, |ws, window, cx| ws.open_collaboration_dialog(window, cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.collaboration_dialog.is_some())); +} + +#[gpui::test] +fn join_dialog_opens(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.open_join_collaboration_dialog(window, cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.collaboration_dialog.is_some())); +} + +#[gpui::test] +fn share_dialog_reopen_replaces_previous(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.open_collaboration_dialog(window, cx)); + cx.run_until_parked(); + let first = h + .read(cx, |ws| ws.collaboration_dialog.clone()) + .expect("first dialog open"); + h.update(cx, |ws, window, cx| ws.open_join_collaboration_dialog(window, cx)); + cx.run_until_parked(); + let second = h + .read(cx, |ws| ws.collaboration_dialog.clone()) + .expect("second dialog open"); + assert_ne!(first.entity_id(), second.entity_id(), "reopen must build a fresh dialog"); +} + +#[gpui::test] +fn close_collaboration_dialog_clears_state(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.open_collaboration_dialog(window, cx)); + cx.run_until_parked(); + h.update(cx, |ws, _, cx| ws.close_collaboration_dialog(cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.collaboration_dialog.is_none())); +} + +#[gpui::test] +fn comment_dialog_opens_once_runtime_attaches(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + // The comment dialog needs the panel's I/O runtime, which attaches from an + // OS thread outside the test dispatcher. + h.wait_until(cx, "document runtime attach", |ws| { + ws.active_document_id + .is_some_and(|id| ws.document_runtimes.contains_key(&id)) + }); + h.update(cx, |ws, window, cx| ws.open_comment_dialog(window, cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.comment_dialog.is_some())); + // Reopen path (closes the previous dialog first). + h.update(cx, |ws, window, cx| ws.open_comment_dialog(window, cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.comment_dialog.is_some())); +} diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/documents.rs b/crates/flowstate/src/workspace/workspace/headless_tests/documents.rs new file mode 100644 index 00000000..1421f643 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/documents.rs @@ -0,0 +1,87 @@ +//! Document panel lifecycle: creation, activation, close, and the sandboxed +//! open-tabs session file. + +use gpui::TestAppContext; + +use super::support; + +#[gpui::test] +fn new_document_activates_panel(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + assert!(h.read(cx, |ws| ws.document_panels.is_empty() && ws.active_document_id.is_none())); + h.new_document(cx); + h.read(cx, |ws| { + assert_eq!(ws.document_panels.len(), 1); + assert!(ws.active_document_id.is_some()); + assert!(ws.active_editor.is_some()); + }); +} + +#[gpui::test] +fn second_document_takes_focus_and_activation_switches_back(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + let first = h + .read(cx, |ws| ws.active_document_id) + .expect("first panel active"); + h.new_document(cx); + let second = h + .read(cx, |ws| ws.active_document_id) + .expect("second panel active"); + assert_ne!(first, second, "new document must become active"); + + h.update(cx, |ws, _, cx| { + let (id, editor) = { + let panel = ws + .document_panels + .iter() + .find(|panel| panel.read(cx).id() == first) + .expect("first panel still open"); + (panel.read(cx).id(), panel.read(cx).editor()) + }; + ws.set_active_document(id, editor, cx); + }); + cx.run_until_parked(); + assert_eq!(h.read(cx, |ws| ws.active_document_id), Some(first)); +} + +#[gpui::test] +fn close_active_document_removes_panel(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.close_active_document(window, cx)); + cx.run_until_parked(); + // A blank untitled document may count as dirty; if the save prompt fired, + // decline saving and let the close proceed. + if cx.has_pending_prompt() { + cx.simulate_prompt_answer("Don't Save"); + cx.run_until_parked(); + } + h.wait_until(cx, "panel closed", |ws| ws.document_panels.is_empty()); + assert_eq!(h.read(cx, |ws| ws.active_document_id), None); + assert!(h.read(cx, |ws| ws.active_editor.is_none())); +} + +#[gpui::test] +fn temporary_session_file_lands_in_sandbox(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + // Persist through the workspace path (debounced spawn), then verify the + // direct write, so both prove the FLOWSTATE_CONFIG_DIR reroute: a leak to + // the real std::env::temp_dir() would clobber the user's real tab session. + h.update(cx, |ws, _, cx| ws.persist_temporary_workspace_session(cx)); + cx.run_until_parked(); + + super::super::persist_temporary_workspace_session_to_disk(super::super::TemporaryWorkspaceSession { + entries: vec![], + active_index: None, + ribbon_collapsed: false, + outline_collapsed: false, + toolkit_collapsed: false, + pinned_entry_indices: vec![], + speech_entry_index: None, + }); + let sandbox_session = support::sandbox_config_dir().join("flowstate-open-tabs-session.json"); + let path = super::super::temporary_workspace_session_path(); + assert_eq!(path, sandbox_session, "session file must resolve into the sandbox"); +} diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/flows_and_keystrokes.rs b/crates/flowstate/src/workspace/workspace/headless_tests/flows_and_keystrokes.rs new file mode 100644 index 00000000..27406a98 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/flows_and_keystrokes.rs @@ -0,0 +1,52 @@ +//! Flow panel lifecycle plus true keystroke-level dispatch through the +//! window interceptor installed in `Workspace::new` (one level above the +//! `handle_window_keybinding` unit the other suites exercise). + +use gpui::TestAppContext; + +use super::support; + +#[gpui::test] +fn new_flow_creates_and_activates_flow_panel(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.update(cx, |ws, window, cx| ws.new_flow(window, cx)); + cx.run_until_parked(); + h.read(cx, |ws| { + assert_eq!(ws.flow_panels.len(), 1); + assert!(ws.active_flow.is_some()); + assert!(ws.active_document_id.is_some(), "flow panel takes a panel id"); + }); +} + +#[gpui::test] +fn flow_and_document_panels_coexist(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.update(cx, |ws, window, cx| ws.new_flow(window, cx)); + cx.run_until_parked(); + h.read(cx, |ws| { + assert_eq!(ws.document_panels.len(), 1); + assert_eq!(ws.flow_panels.len(), 1); + assert!(ws.active_flow.is_some(), "flow was opened last and must be active"); + }); +} + +#[gpui::test] +fn ctrl_tab_keystroke_cycles_tabs_through_the_interceptor(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.new_document(cx); + let before = h.read(cx, |ws| ws.active_document_id); + cx.simulate_keystrokes(h.window, "ctrl-tab"); + cx.run_until_parked(); + let after = h.read(cx, |ws| ws.active_document_id); + assert_ne!(before, after, "ctrl-tab must reach the workspace keybinding interceptor"); +} + +#[gpui::test] +fn find_keystroke_is_absorbed_without_panic(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + cx.simulate_keystrokes(h.window, "ctrl-f"); + cx.run_until_parked(); +} diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/mod.rs b/crates/flowstate/src/workspace/workspace/headless_tests/mod.rs new file mode 100644 index 00000000..e9c99517 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/mod.rs @@ -0,0 +1,23 @@ +//! Headless GPUI tests for the workspace/view layer. +//! +//! These run the REAL app wiring — `open_workspace_window`, the real +//! `Workspace`, real dialogs — under `gpui::TestAppContext` (no GPU, no +//! fonts, no real config dir). They exist because this layer had zero +//! coverage: the collab soaks enter below the workspace, so a double-lease +//! panic in dialog wiring shipped to the field (share-dialog constructor +//! calling `workspace.update` inside the workspace's own update). +//! +//! Ground rules for adding tests here: +//! - Go through `support::open_workspace` so config/data dirs stay sandboxed. +//! - Never install the app's custom prompt renderer: prompts must reach the +//! test platform queue for `simulate_prompt_answer`. +//! - Discovery is force-paused by the sandbox settings; don't unpause it. + +mod support; + +mod actions; +mod collab_glue; +mod dialogs; +mod documents; +mod flows_and_keystrokes; +mod tabs_and_overlays; diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/support.rs b/crates/flowstate/src/workspace/workspace/headless_tests/support.rs new file mode 100644 index 00000000..458fb4f9 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/support.rs @@ -0,0 +1,99 @@ +use std::path::PathBuf; +use std::sync::OnceLock; + +use gpui::{AnyWindowHandle, Entity, TestAppContext, Window}; + +use crate::workspace::{Workspace, open_workspace_window}; + +/// Process-wide sandbox for every on-disk artifact the app touches: settings +/// (incl. the first-run profile mint), the tub data dir, and the open-tabs +/// session file. Set once, before the first `load_app_settings`, while all +/// other test threads are still blocked on the `OnceLock`. +fn sandbox_root() -> &'static PathBuf { + static SANDBOX: OnceLock = OnceLock::new(); + SANDBOX.get_or_init(|| { + let root = std::env::temp_dir().join(format!("flowstate-headless-{}", std::process::id())); + let config = root.join("config"); + let data = root.join("data"); + std::fs::create_dir_all(config.join("flowstate")).expect("create sandbox config dir"); + std::fs::create_dir_all(&data).expect("create sandbox data dir"); + // Discovery stays paused so no test ever reaches a real transport + // (BLE/Dropbox). BLE is opt-in-default-off anyway; the pause covers all. + std::fs::write(config.join("flowstate/settings.toml"), "collaboration_discovery_paused = true\n").expect("write sandbox settings"); + // SAFETY: single writer inside OnceLock init; every test enters through + // this function before any env read of these keys, and concurrent + // first-callers are parked on the OnceLock until it returns. + unsafe { std::env::set_var("FLOWSTATE_CONFIG_DIR", &config) }; + // SAFETY: same single-writer OnceLock-init guarantee as above. + unsafe { std::env::set_var("FLOWSTATE_DATA_DIR", &data) }; + root + }) +} + +pub fn sandbox_config_dir() -> PathBuf { + sandbox_root().join("config") +} + +pub struct WorkspaceHarness { + pub window: AnyWindowHandle, + pub workspace: Entity, +} + +impl WorkspaceHarness { + /// Run `f` against the workspace with the window available — the same shape + /// as a real event handler (workspace lease held, window borrowed). + pub fn update(&self, cx: &mut TestAppContext, f: impl FnOnce(&mut Workspace, &mut Window, &mut gpui::Context) -> R) -> R { + let workspace = self.workspace.clone(); + self + .window + .update(cx, |_, window, cx| workspace.update(cx, |ws, cx| f(ws, window, cx))) + .expect("workspace window is open") + } + + pub fn read(&self, cx: &mut TestAppContext, f: impl FnOnce(&Workspace) -> R) -> R { + let workspace = self.workspace.clone(); + self + .window + .update(cx, |_, _, cx| f(workspace.read(cx))) + .expect("workspace window is open") + } + + /// Create a blank in-memory document panel and wait for the quiet state. + pub fn new_document(&self, cx: &mut TestAppContext) { + self.update(cx, |ws, window, cx| ws.new_document(window, cx)); + cx.run_until_parked(); + } + + /// Wait (real time — document runtimes live on OS threads outside the test + /// dispatcher) until `ready` observes the state it wants, or panic. + pub fn wait_until(&self, cx: &mut TestAppContext, what: &str, mut ready: impl FnMut(&Workspace) -> bool) { + for _ in 0..500 { + cx.run_until_parked(); + if self.read(cx, |ws| ready(ws)) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + panic!("timed out waiting for: {what}"); + } +} + +/// Boot the real app wiring headlessly: component/theme globals, the real +/// keymap, and the production `open_workspace_window` path (close-prompt +/// install, session restore, initial frame). Deliberately does NOT install +/// the custom prompt renderer — prompts must stay on the test platform's +/// queue so tests can drive them with `simulate_prompt_answer`. +pub fn open_workspace(cx: &mut TestAppContext) -> WorkspaceHarness { + sandbox_root(); + cx.update(|cx| { + gpui_component::init(cx); + crate::app::register_rich_text_editor_keybindings(cx); + }); + let workspace = cx.update(|cx| open_workspace_window(None, cx)); + cx.run_until_parked(); + let workspace = workspace + .upgrade() + .expect("workspace entity alive after window open"); + let window = *cx.windows().first().expect("workspace window exists"); + WorkspaceHarness { window, workspace } +} diff --git a/crates/flowstate/src/workspace/workspace/headless_tests/tabs_and_overlays.rs b/crates/flowstate/src/workspace/workspace/headless_tests/tabs_and_overlays.rs new file mode 100644 index 00000000..8f2e3a75 --- /dev/null +++ b/crates/flowstate/src/workspace/workspace/headless_tests/tabs_and_overlays.rs @@ -0,0 +1,121 @@ +//! Tab navigation, pinning, ribbon, speech-document routing, and the +//! full-screen overlays (settings, styles, file search) — each test forces a +//! real headless render of the surface it opens. + +use gpui::TestAppContext; + +use crate::commands::CommandId; + +use super::support; + +fn command(h: &support::WorkspaceHarness, cx: &mut TestAppContext, id: CommandId) -> bool { + let handled = h.update(cx, |ws, window, cx| ws.handle_window_keybinding(id, window, cx)); + cx.run_until_parked(); + handled +} + +#[gpui::test] +fn next_and_previous_tab_cycle_through_documents(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + for _ in 0..3 { + h.new_document(cx); + } + let order: Vec<_> = h.read(cx, |ws| ws.document_panels.iter().map(|p| p.entity_id()).collect()); + assert_eq!(order.len(), 3); + + let start = h + .read(cx, |ws| ws.active_document_id) + .expect("active document"); + assert!(command(&h, cx, CommandId::NextTab)); + let next = h + .read(cx, |ws| ws.active_document_id) + .expect("active document"); + assert_ne!(start, next, "NextTab must move activation"); + assert!(command(&h, cx, CommandId::PreviousTab)); + assert_eq!(h.read(cx, |ws| ws.active_document_id), Some(start), "PreviousTab must move back"); +} + +#[gpui::test] +fn switch_to_tab_shortcuts_activate_by_index(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + h.new_document(cx); + assert!(command(&h, cx, CommandId::SwitchToTab1)); + let on_first = h.read(cx, |ws| ws.active_document_id); + assert!(command(&h, cx, CommandId::SwitchToTab2)); + let on_second = h.read(cx, |ws| ws.active_document_id); + assert!(on_first.is_some() && on_second.is_some()); + assert_ne!(on_first, on_second); + // Out-of-range index must be harmless. + assert!(command(&h, cx, CommandId::SwitchToTab9)); +} + +#[gpui::test] +fn toggle_pin_tab_round_trips(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + let id = h + .read(cx, |ws| ws.active_document_id) + .expect("active document"); + assert!(command(&h, cx, CommandId::TogglePinTab)); + assert!(h.read(cx, |ws| ws.pinned_document_ids.contains(&id))); + assert!(command(&h, cx, CommandId::TogglePinTab)); + assert!(!h.read(cx, |ws| ws.pinned_document_ids.contains(&id))); +} + +#[gpui::test] +fn toggle_ribbon_flips_and_renders(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + let before = h.read(cx, |ws| ws.ribbon_collapsed); + assert!(command(&h, cx, CommandId::ToggleRibbon)); + assert_ne!(h.read(cx, |ws| ws.ribbon_collapsed), before); + assert!(command(&h, cx, CommandId::ToggleRibbon)); + assert_eq!(h.read(cx, |ws| ws.ribbon_collapsed), before); +} + +#[gpui::test] +fn speech_document_toggle_marks_and_clears(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + let id = h + .read(cx, |ws| ws.active_document_id) + .expect("active document"); + assert!(command(&h, cx, CommandId::ToggleSpeechDocument)); + assert_eq!(h.read(cx, |ws| ws.speech_document_id), Some(id)); + assert!(command(&h, cx, CommandId::ToggleSpeechDocument)); + assert_eq!(h.read(cx, |ws| ws.speech_document_id), None); +} + +#[gpui::test] +fn settings_and_styles_overlays_render_headlessly(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.new_document(cx); + for overlay in [ + super::super::WorkspaceSettingsOverlay::Settings, + super::super::WorkspaceSettingsOverlay::Styles, + ] { + h.update(cx, |ws, _, cx| { + ws.settings_overlay = Some(overlay); + cx.notify(); + }); + // run_until_parked draws the frame — a panic in the overlay render + // (the layer with zero coverage until now) fails the test here. + cx.run_until_parked(); + h.update(cx, |ws, _, cx| { + ws.settings_overlay = None; + cx.notify(); + }); + cx.run_until_parked(); + } +} + +#[gpui::test] +fn file_search_overlay_opens_and_closes(cx: &mut TestAppContext) { + let h = support::open_workspace(cx); + h.update(cx, |ws, window, cx| ws.open_file_search_overlay(window, cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.file_search_overlay.is_some())); + h.update(cx, |ws, _, cx| ws.close_file_search_overlay(cx)); + cx.run_until_parked(); + assert!(h.read(cx, |ws| ws.file_search_overlay.is_none())); +} diff --git a/crates/flowstate/src/workspace/workspace/keybindings.rs b/crates/flowstate/src/workspace/workspace/keybindings.rs index 83fd8507..a68bd7d6 100644 --- a/crates/flowstate/src/workspace/workspace/keybindings.rs +++ b/crates/flowstate/src/workspace/workspace/keybindings.rs @@ -52,6 +52,18 @@ impl Workspace { self.close_active_document(window, cx); true }, + CommandId::ShareDocument => { + self.open_collaboration_dialog(window, cx); + true + }, + CommandId::JoinSession => { + self.open_join_collaboration_dialog(window, cx); + true + }, + CommandId::StartCollaboration => self.start_collaboration_on_active_document(cx).is_some(), + CommandId::CopyCollaborationTicket => self.copy_active_collaboration_ticket(window, cx), + CommandId::JoinCollaborationFromClipboard => self.join_collaboration_from_clipboard(window, cx), + CommandId::LeaveCollaboration => self.confirm_leave_collaboration_on_active_document(window, cx), CommandId::FindInDocument => self.open_active_document_search_bar(window, cx), CommandId::ZoomIn => { if let Some(editor) = self.active_editor.clone() { @@ -184,6 +196,10 @@ impl Workspace { self.open_active_document_search_bar(window, cx); } + fn on_fidelity_mark(&mut self, _: &crate::commands::FidelityMarkAction, _window: &mut Window, _cx: &mut Context) { + flowstate_fidelity::marker("keybinding"); + } + fn open_active_document_search_bar(&mut self, window: &mut Window, cx: &mut Context) -> bool { let Some(active_document_id) = self.active_document_id else { return false; @@ -207,7 +223,10 @@ impl Workspace { } fn non_document_keybinding_surface_is_open(&self) -> bool { - self.settings_overlay.is_some() || self.file_search_overlay.is_some() + self.settings_overlay.is_some() + || self.file_search_overlay.is_some() + || self.collaboration_dialog.is_some() + || self.revision_dialog.is_some() } fn focused_workspace_input_is_focused(&self, window: &Window, cx: &App) -> bool { @@ -225,6 +244,14 @@ impl Workspace { .file_search_overlay .as_ref() .is_some_and(|overlay| overlay.read(cx).focus_handle(cx).is_focused(window)) + || self + .collaboration_dialog + .as_ref() + .is_some_and(|dialog| dialog.read(cx).focus_handle(cx).is_focused(window)) + || self + .revision_dialog + .as_ref() + .is_some_and(|dialog| dialog.read(cx).focus_handle(cx).is_focused(window)) || self .document_panels .iter() diff --git a/crates/flowstate/src/workspace/workspace/load.rs b/crates/flowstate/src/workspace/workspace/load.rs index 6884e225..bcbc7bbe 100644 --- a/crates/flowstate/src/workspace/workspace/load.rs +++ b/crates/flowstate/src/workspace/workspace/load.rs @@ -1,29 +1,86 @@ struct LoadedDocumentForOpen { - document: Document, + document: DocumentProjection, + runtime: flowstate_collab::crdt_runtime::CrdtRuntime, path: Option, title: Option, } +/// §P3 preview→open bridge: the opening screen's docx preview runs the ENTIRE +/// clean/interpret/structured pipeline to materialize a projection, and the +/// subsequent open re-ran the identical pipeline from scratch (~2.4s on the +/// reference doc) before the Loro import. Both produce the same +/// `DocumentProjection` (`convert_docx_to_document` and +/// `import_cleaned_docx_to_loro` share `build_structured_document`), so the +/// preview's result feeds the open directly when the file is unchanged +/// (mtime + length fingerprint). Small LRU: the open screen previews several +/// recents; entries are dropped once used or displaced. +static DOCX_PREVIEW_BRIDGE: std::sync::Mutex> = + std::sync::Mutex::new(Vec::new()); +const DOCX_PREVIEW_BRIDGE_CAP: usize = 4; + +fn docx_fingerprint(path: &Path) -> Option<(std::time::SystemTime, u64)> { + let metadata = std::fs::metadata(path).ok()?; + Some((metadata.modified().ok()?, metadata.len())) +} + +fn store_docx_preview_bridge(path: &Path, document: &DocumentProjection) { + let Some((modified, len)) = docx_fingerprint(path) else { + return; + }; + let Ok(mut bridge) = DOCX_PREVIEW_BRIDGE.lock() else { + return; + }; + bridge.retain(|(existing, ..)| existing != path); + bridge.push((path.to_path_buf(), modified, len, document.clone())); + if bridge.len() > DOCX_PREVIEW_BRIDGE_CAP { + bridge.remove(0); + } +} + +fn take_docx_preview_bridge(path: &Path) -> Option { + let (modified, len) = docx_fingerprint(path)?; + let mut bridge = DOCX_PREVIEW_BRIDGE.lock().ok()?; + let ix = bridge + .iter() + .position(|(existing, existing_modified, existing_len, _)| existing == path && *existing_modified == modified && *existing_len == len)?; + Some(bridge.remove(ix).3) +} + #[hotpath::measure] fn load_document_for_open(path: &PathBuf) -> std::io::Result { if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) { if extension.eq_ignore_ascii_case("docx") { - let (document, _) = convert_docx_to_document(path)?; + let title = path + .with_extension("db8") + .file_name() + .map(|name| name.to_string_lossy().to_string()); + let imported = match take_docx_preview_bridge(path) { + // The preview already ran the whole docx pipeline on this exact file + // content; only the Loro import remains. + Some(document) => flowstate_document::import_document_projection(document, title.as_deref().unwrap_or("Imported Document"))?, + None => import_docx_to_loro(path, title.as_deref().unwrap_or("Imported Document"))?.0, + }; + let runtime = flowstate_collab::crdt_runtime::CrdtRuntime::from_imported_document(imported).map_err(runtime_io_error)?; + // Runtime startup records replica metadata and therefore advances the + // canonical frontier. The editor must start from that exact frontier or + // its first command is correctly rejected as stale. + let document = runtime.projection_snapshot().map_err(runtime_io_error)?; return Ok(LoadedDocumentForOpen { document, + runtime, path: None, - title: path - .with_extension("db8") - .file_name() - .map(|name| name.to_string_lossy().to_string()), + title, }); } if extension.eq_ignore_ascii_case("pdf") { let db8_bytes = crate::docx_conversion::extract_db8_bytes_from_pdf(path)? .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "PDF does not contain embedded Flowstate DB8 data"))?; - let document = flowstate_document::read_db8_bytes(&db8_bytes)?; + let package = flowstate_document::DocumentPackage::from_bytes(&db8_bytes)?; + let runtime = flowstate_collab::crdt_runtime::CrdtRuntime::from_package(package, None).map_err(runtime_io_error)?; + let document = runtime.projection_snapshot().map_err(runtime_io_error)?; return Ok(LoadedDocumentForOpen { document, + runtime, path: None, title: path .with_extension("db8") @@ -33,9 +90,102 @@ fn load_document_for_open(path: &PathBuf) -> std::io::Result { + let document = runtime.projection_snapshot().map_err(runtime_io_error)?; + (document, runtime) + }, + Err(error) if runtime_root_io_kind(&error) == Some(std::io::ErrorKind::NotFound) => { + let imported = flowstate_document::import_document_projection(new_blank_document(), "Flowstate Document")?; + let runtime = flowstate_collab::crdt_runtime::CrdtRuntime::from_imported_document(imported).map_err(runtime_io_error)?; + let document = runtime.projection_snapshot().map_err(runtime_io_error)?; + (document, runtime) + }, + Err(error) => return Err(runtime_io_error(error)), + }; + Ok(LoadedDocumentForOpen { document, + runtime, path: Some(path.clone()), title: None, }) } + +/// §act-three C (phase V): the cheapest renderable projection for `path`, +/// WITHOUT constructing the authority runtime — the frontier-current `.db8` +/// projection cache (~0.3s, no snapshot/segment decode or Loro import) or a +/// still-warm docx preview-bridge projection. Returns `None` when no fast +/// projection exists (the caller then just waits for the full phase-G load). +/// Read-only: this projection paints immediately; editing enables when the +/// authority attaches (phase G). Never fabricates or mutates state. +pub(crate) fn read_open_projection_fast(path: &Path) -> Option { + if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) { + if extension.eq_ignore_ascii_case("docx") { + // The open screen's preview already materialized this exact file; reuse + // it for an instant paint (the Loro import still runs in phase G). Peek + // WITHOUT consuming — `load_document_for_open` takes the bridge entry. + return peek_docx_preview_bridge(path); + } + if extension.eq_ignore_ascii_case("pdf") { + // PDFs carry an embedded .db8 but need extraction; no cheap cache path. + return None; + } + } + // .db8 (and unknown extensions): prefer the frontier-current FULL projection + // cache (whole document, best for scroll during the phase-V window); fall + // back to the §act-four M2 preview header (leading blocks — still an instant + // first-screen paint for an edited-but-unflushed doc the full cache misses). + // Phase G attaches the complete authority underneath either way. + flowstate_document::DocumentPackage::read_cached_projection(path) + .ok() + .flatten() + .or_else(|| flowstate_document::DocumentPackage::read_preview_header(path).ok().flatten()) +} + +fn peek_docx_preview_bridge(path: &Path) -> Option { + let (modified, len) = docx_fingerprint(path)?; + let bridge = DOCX_PREVIEW_BRIDGE.lock().ok()?; + bridge + .iter() + .find(|(existing, existing_modified, existing_len, _)| existing == path && *existing_modified == modified && *existing_len == len) + .map(|(.., document)| document.clone()) +} + +fn runtime_io_error(error: anyhow::Error) -> std::io::Error { + if let Some(io_error) = error.root_cause().downcast_ref::() { + return std::io::Error::new(io_error.kind(), error.to_string()); + } + std::io::Error::new(std::io::ErrorKind::InvalidData, error.to_string()) +} + +fn runtime_root_io_kind(error: &anyhow::Error) -> Option { + error.root_cause().downcast_ref::().map(std::io::Error::kind) +} + +#[hotpath::measure] +fn load_document_preview(path: &Path) -> std::io::Result { + if let Some(extension) = path.extension().and_then(|extension| extension.to_str()) { + if extension.eq_ignore_ascii_case("docx") { + let (document, _) = convert_docx_to_document(path)?; + store_docx_preview_bridge(path, &document); + return Ok(document); + } + if extension.eq_ignore_ascii_case("pdf") { + let db8_bytes = crate::docx_conversion::extract_db8_bytes_from_pdf(path)? + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "PDF does not contain embedded Flowstate DB8 data"))?; + return flowstate_document::read_db8_bytes(&db8_bytes); + } + } + // §act-four M2 preview fast path: the tiny preview header (leading blocks + // only) renders the preview in O(viewport), independent of document size AND + // of full-cache freshness — it rides journal deltas, so an edited-but- + // unflushed document still previews without touching the snapshot. Falls back + // to the §P3 full projection cache, then the full read. + if let Some(document) = flowstate_document::DocumentPackage::read_preview_header(path)? { + return Ok(document); + } + if let Some(document) = flowstate_document::DocumentPackage::read_cached_projection(path)? { + return Ok(document); + } + flowstate_document::read_db8(path) +} diff --git a/crates/flowstate/src/workspace/workspace/mod.rs b/crates/flowstate/src/workspace/workspace/mod.rs index 230b9d30..5c77017e 100644 --- a/crates/flowstate/src/workspace/workspace/mod.rs +++ b/crates/flowstate/src/workspace/workspace/mod.rs @@ -1,5 +1,5 @@ use std::{ - cell::Cell, + cell::{Cell, RefCell}, collections::{HashMap, HashSet}, fs, path::{Path, PathBuf}, @@ -31,22 +31,23 @@ use gpui_component::tab::{Tab, TabBar}; use gpui_component::tree::{TreeItem, TreeState, tree}; use gpui_component::{ ActiveTheme as _, Colorize as _, Disableable, Icon, IconName, PixelsExt, Root, Selectable, Sizable, Theme, ThemeRegistry, TitleBar, - VirtualListScrollHandle, h_flex, v_flex, + VirtualListScrollHandle, WindowExt as _, h_flex, v_flex, }; +use rustc_hash::{FxHashMap, FxHashSet}; use uuid::Uuid; use crate::app_settings::{ - load_autosave, load_document_theme, load_recent_documents, load_send_custom_directory, load_send_to_document_directory, - load_smart_word_selection, load_tub_root, save_autosave, save_document_theme, save_recent_documents, save_send_custom_directory, - save_send_to_document_directory, save_smart_word_selection, save_theme_name, + load_autosave, load_document_theme, load_local_user_identity, load_recent_documents, load_send_custom_directory, + load_send_to_document_directory, load_smart_word_selection, load_tub_root, save_autosave, save_document_theme, save_recent_documents, + save_send_custom_directory, save_send_to_document_directory, save_smart_word_selection, save_theme_name, }; use crate::commands::CommandId; -use crate::docx_conversion::convert_docx_to_document; +use crate::docx_conversion::{convert_docx_to_document, import_docx_to_loro}; use crate::flow::{FlowEditor, FlowPanel}; use crate::rich_text_element::{ - ArmedInlineTool, CustomParagraphBorder, Document, DocumentTheme, InputParagraph, InputRun, ParagraphStyle, RichTextDocumentElement, + ArmedInlineTool, CustomParagraphBorder, DocumentProjection, DocumentTheme, InputParagraph, InputRun, ParagraphStyle, RichTextDocumentElement, RichTextEditor, Save, SectionKind, ThemeUnderline, ZoomIn, ZoomOut, document_from_input, document_text_slice, flowstate_document_theme, - load_or_create_document, paragraph_byte_range, paragraph_index_for_id, + paragraph_byte_range, paragraph_index_for_id, }; use crate::workspace::document_panel::DocumentPanel; use crate::workspace::file_management::{ @@ -62,8 +63,18 @@ const SIDE_PANEL_COLLAPSED_WIDTH: Pixels = px(30.0); #[path = "../toolkit_panel.rs"] mod toolkit_panel; +#[cfg(test)] +mod headless_tests; + pub struct Workspace { document_panels: Vec>, + // §perf: Uuid keys are locally generated and trusted; use FxHash to avoid SipHash overhead. + document_runtimes: FxHashMap, + document_runtime_flush_pending: FxHashSet, + /// §act-three C (background open): panels painted read-only from a phase-V + /// cached projection whose authority runtime has not yet attached (phase G). + /// Editing is inert until attach; session-persist + autosave skip them. + pending_authority_panels: FxHashSet, flow_panels: Vec>, active_document_id: Option, active_editor: Option>, @@ -73,7 +84,7 @@ pub struct Workspace { toolkit_collapsed: bool, active_toolkit_tool: Option, recent_documents: Vec, - recent_document_previews: HashMap, + recent_document_previews: HashMap, recent_document_preview_generation: u64, temporary_workspace_session_pending: Option, temporary_workspace_session_persist_scheduled: bool, @@ -81,8 +92,9 @@ pub struct Workspace { tab_bar_scroll_handle: ScrollHandle, pinned_document_ids: Vec, speech_document_id: Option, - speech_word_count_cache: HashMap, - speech_word_count_pending: HashSet, + // §perf: Uuid keys are locally generated and trusted; use FxHash to avoid SipHash overhead. + speech_word_count_cache: FxHashMap, + speech_word_count_pending: FxHashSet, body_resizable_state: Entity, content_resizable_state: Entity, ribbon_resizable_state: Entity, @@ -101,8 +113,19 @@ pub struct Workspace { document_style_section: DocumentStyleSection, settings_section: WorkspaceSettingsSection, autosave_enabled: bool, - autosave_document_generations: HashMap, - autosave_flow_in_flight: HashSet, + // §perf: Uuid keys are locally generated and trusted; use FxHash to avoid SipHash overhead. + autosave_document_generations: FxHashMap, + /// §act-five P9-throttle: the latest edit generation a debounced autosave is + /// SCHEDULED for. A newer edit overwrites it, so the trailing timer coalesces a + /// burst into ONE checkpoint instead of a full checkpoint per keystroke. + autosave_pending_generation: FxHashMap, + autosave_flow_in_flight: FxHashSet, + collaboration_dialog: Option>, + revision_dialog: Option>, + comment_dialog: Option>, + // §perf: SessionId keys are locally generated and trusted; use FxHash to avoid SipHash overhead. + collab_notice_subscriptions: FxHashMap, + collab_incompatible_version_notices: HashSet, file_search_overlay: Option>, tub_root: Option, tub_index: Option>, @@ -171,6 +194,7 @@ enum WorkspaceSettingsOverlay { #[derive(Clone, Copy, PartialEq, Eq)] enum WorkspaceSettingsSection { General, + Collaboration, Keymap, } @@ -217,6 +241,7 @@ impl WorkspaceSettingsSection { fn title(self) -> &'static str { match self { Self::General => "General", + Self::Collaboration => "Collaboration", Self::Keymap => "Keymap", } } @@ -224,7 +249,8 @@ impl WorkspaceSettingsSection { fn index(self) -> usize { match self { Self::General => 0, - Self::Keymap => 1, + Self::Collaboration => 1, + Self::Keymap => 2, } } } @@ -261,6 +287,8 @@ impl DocumentStyleSection { } include!("documents.rs"); +include!("collab_prompts.rs"); +include!("collab.rs"); include!("workspace_state.rs"); include!("load.rs"); include!("traits.rs"); @@ -276,6 +304,7 @@ include!("window.rs"); include!("outline.rs"); include!("top_bar.rs"); include!("style_settings.rs"); +include!("collaboration_settings.rs"); include!("keymap_settings.rs"); include!("theme.rs"); include!("tests.rs"); diff --git a/crates/flowstate/src/workspace/workspace/outline.rs b/crates/flowstate/src/workspace/workspace/outline.rs index d4e6602f..6f3bea83 100644 --- a/crates/flowstate/src/workspace/workspace/outline.rs +++ b/crates/flowstate/src/workspace/workspace/outline.rs @@ -42,7 +42,7 @@ impl OutlineCache { self.visible_revision = revision; } - fn update_signature(&mut self, document: &Document, edit_generation: u64) -> bool { + fn update_signature(&mut self, document: &DocumentProjection, edit_generation: u64) -> bool { let paragraph_count = outline_signature_entries_into(document, &mut self.signature_scratch); let unchanged = self.signature.paragraph_count == paragraph_count && self.signature.entries == self.signature_scratch; self.signature.paragraph_count = paragraph_count; @@ -119,7 +119,7 @@ fn outline_node_to_tree_item(node: &OutlineNode, collapsed_items: &HashSet Vec { +fn outline_nodes(document: &DocumentProjection) -> Vec { let signature = outline_signature(document); outline_nodes_from_entries(&signature.entries) } @@ -143,14 +143,14 @@ fn outline_nodes_from_entries(entries: &[OutlineEntry]) -> Vec { } #[hotpath::measure] -fn outline_signature(document: &Document) -> OutlineSignature { +fn outline_signature(document: &DocumentProjection) -> OutlineSignature { let mut entries = Vec::new(); let paragraph_count = outline_signature_entries_into(document, &mut entries); OutlineSignature { paragraph_count, entries } } #[hotpath::measure] -fn outline_signature_entries_into(document: &Document, entries: &mut Vec) -> usize { +fn outline_signature_entries_into(document: &DocumentProjection, entries: &mut Vec) -> usize { let mut entry_ix = 0usize; for (paragraph_ix, paragraph) in document.paragraphs.iter().enumerate() { let Some(level) = outline_level(paragraph.style) else { @@ -264,14 +264,14 @@ fn outline_paragraph_ix(id: &str) -> Option { #[cfg(test)] #[hotpath::measure] -fn outline_paragraph_label(document: &Document, paragraph_ix: usize) -> String { +fn outline_paragraph_label(document: &DocumentProjection, paragraph_ix: usize) -> String { let mut label = String::with_capacity(80); outline_paragraph_label_into(document, paragraph_ix, &mut label); label } #[hotpath::measure] -fn outline_paragraph_label_into(document: &Document, paragraph_ix: usize, label: &mut String) { +fn outline_paragraph_label_into(document: &DocumentProjection, paragraph_ix: usize, label: &mut String) { let paragraph_range = paragraph_byte_range(document, paragraph_ix); const MAX_BYTES: usize = 80; const TRUNCATED_BYTES: usize = MAX_BYTES - 3; diff --git a/crates/flowstate/src/workspace/workspace/render_documents.rs b/crates/flowstate/src/workspace/workspace/render_documents.rs index 187be03c..fd0b30bb 100644 --- a/crates/flowstate/src/workspace/workspace/render_documents.rs +++ b/crates/flowstate/src/workspace/workspace/render_documents.rs @@ -75,6 +75,7 @@ impl Workspace { .children(tabs.into_iter().map(|tab| { let panel_id = tab.id; let workspace = workspace.clone(); + let collab_phase = crate::collab::phase_for_panel(panel_id, cx); let tab_prefix = h_flex() .ml(px(5.0)) .mr(px(-3.0)) @@ -104,6 +105,9 @@ impl Workspace { .border_color(cx.theme().warning.opacity(0.72)) .child(pin_label), ) + }) + .when_some(collab_phase.as_ref().and_then(|phase| crate::collab::status::tab_badge(phase, cx)), |this, badge| { + this.child(badge) }); let close_button = icon_button(("close-tab", panel_id.as_u128() as u64), AppIcon::Close) .tooltip("Close document") diff --git a/crates/flowstate/src/workspace/workspace/render_outline.rs b/crates/flowstate/src/workspace/workspace/render_outline.rs index fedf6c0f..fba3ed85 100644 --- a/crates/flowstate/src/workspace/workspace/render_outline.rs +++ b/crates/flowstate/src/workspace/workspace/render_outline.rs @@ -110,24 +110,29 @@ impl Workspace { .into_any_element() } - fn search_match_outline_paragraphs(&self, cx: &App) -> HashSet { + fn search_match_outline_paragraphs(&self, cx: &App) -> FxHashSet { let Some(active_document_id) = self.active_document_id else { - return HashSet::new(); + return FxHashSet::default(); }; let Some(cache) = self.outline_cache.as_ref() else { - return HashSet::new(); + return FxHashSet::default(); }; let Some(panel) = self .document_panels .iter() .find(|panel| panel.read(cx).id() == active_document_id) else { - return HashSet::new(); + return FxHashSet::default(); }; - let mut outline_paragraphs = HashSet::new(); let panel = panel.read(cx); - for paragraph_ix in panel.search_match_paragraphs() { + // §perf: FxHashSet + skip building the set entirely when no search match is active. + let mut matches = panel.search_match_paragraphs().peekable(); + if matches.peek().is_none() { + return FxHashSet::default(); + } + let mut outline_paragraphs = FxHashSet::default(); + for paragraph_ix in matches { if let Some(outline_paragraph) = active_visible_outline_paragraph_from_visible(&cache.visible_paragraphs, paragraph_ix) { outline_paragraphs.insert(outline_paragraph); } diff --git a/crates/flowstate/src/workspace/workspace/render_settings.rs b/crates/flowstate/src/workspace/workspace/render_settings.rs index 21827191..55874030 100644 --- a/crates/flowstate/src/workspace/workspace/render_settings.rs +++ b/crates/flowstate/src/workspace/workspace/render_settings.rs @@ -198,6 +198,7 @@ impl Workspace { }, WorkspaceSettingsOverlay::Settings => match self.settings_section { WorkspaceSettingsSection::General => "app-popup-settings-general", + WorkspaceSettingsSection::Collaboration => "app-popup-settings-collaboration", WorkspaceSettingsSection::Keymap => "app-popup-settings-keymap", }, }; @@ -597,6 +598,26 @@ impl Workspace { .item(send_to_document_directory_item(workspace.clone())) .item(send_custom_directory_item(workspace.clone())), ), + SettingPage::new("Collaboration") + .group(reset_workspace_settings_section_group( + workspace.clone(), + WorkspaceSettingsSection::Collaboration, + )) + .group( + SettingGroup::new() + .title("Identity and discovery") + .item(collaboration_profile_item(workspace.clone())) + .item(trusted_collaborators_item(workspace.clone())) + .item(collaboration_squads_item(workspace.clone())) + .item(collaboration_discovery_pause_item(workspace.clone())) + .item(collaboration_bluetooth_item(workspace.clone())), + ) + .group( + SettingGroup::new() + .title("Dropbox") + .item(dropbox_connection_item(workspace.clone())) + .item(dropbox_document_binding_item(workspace.clone())), + ), SettingPage::new("Keymap") .group(reset_workspace_settings_section_group( workspace.clone(), @@ -745,6 +766,13 @@ impl Workspace { .detach(); cx.notify(); }, + WorkspaceSettingsSection::Collaboration => { + if let Err(error) = crate::app_settings::save_collaboration_discovery_options(false, false) { + tracing::warn!(%error, "resetting collaboration discovery settings failed"); + } + crate::collab::reconfigure_discovery(cx); + cx.notify(); + }, } } } diff --git a/crates/flowstate/src/workspace/workspace/render_status.rs b/crates/flowstate/src/workspace/workspace/render_status.rs index 7fede050..bc19e9ec 100644 --- a/crates/flowstate/src/workspace/workspace/render_status.rs +++ b/crates/flowstate/src/workspace/workspace/render_status.rs @@ -232,6 +232,8 @@ impl Workspace { } fn render_status_bar(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + self.sync_collab_notice_subscriptions(window, cx); + let speech_word_count = if let (Some(document_id), Some(editor)) = (self.active_document_id, self.active_editor.as_ref()) { let editor = editor.read(cx); let generation = editor.edit_generation(); @@ -245,15 +247,49 @@ impl Workspace { None } else { self.speech_word_count_pending.insert(document_id); + // §A10.6: the O(doc) walk used to run right here on the FOREGROUND + // executor, once per keystroke. Now: clone the projection (cheap — + // persistent, structurally-shared trees), take this document's + // per-paragraph memo, and count on the background executor. Only + // paragraphs whose (id, version) memo went stale are recounted. let document = editor.document().clone(); + let paragraph_cache = SPEECH_WORD_COUNT_PARAGRAPH_CACHES + .lock() + .expect("speech word-count paragraph cache lock poisoned") + .remove(&document_id) + .unwrap_or_default(); cx.spawn(async move |this, cx| { - let count = speech_word_count(&document); + let (count, paragraph_cache) = cx + .background_executor() + .spawn(async move { + let mut paragraph_cache = paragraph_cache; + let (count, _recounted) = speech_word_count_incremental(&document, &mut paragraph_cache); + (count, paragraph_cache) + }) + .await; let _ = this.update(cx, |this, cx| { let is_open = this.document_panels.iter().any(|panel| panel.read(cx).id() == document_id); if is_open { this.speech_word_count_cache.insert(document_id, (generation, count)); } this.speech_word_count_pending.remove(&document_id); + // Return the memo for the next recount, and prune memos of + // documents that closed while a count was in flight (the + // workspace-side total cache is pruned in `close_document`; this + // module-scope map is pruned here). + let open_ids = this + .document_panels + .iter() + .map(|panel| panel.read(cx).id()) + .collect::>(); + let mut caches = SPEECH_WORD_COUNT_PARAGRAPH_CACHES + .lock() + .expect("speech word-count paragraph cache lock poisoned"); + if is_open { + caches.insert(document_id, paragraph_cache); + } + caches.retain(|id, _| open_ids.contains(id)); + drop(caches); cx.notify(); }); }) @@ -264,6 +300,12 @@ impl Workspace { None }; let zoom = self.active_editor.as_ref().map(|editor| editor.read(cx).zoom_percent()); + let collab_phase = self + .active_document_id + .and_then(|panel_id| crate::collab::phase_for_panel(panel_id, cx)); + let collab_roster = self + .active_document_id + .map_or_else(Vec::new, |panel_id| crate::collab::roster_for_panel(panel_id, cx)); if let Some(percent) = zoom { self.sync_zoom_slider(percent, window, cx); } @@ -277,6 +319,19 @@ impl Workspace { .border_color(cx.theme().border) .bg(cx.theme().background) .child(div().flex_1()) + .when_some(collab_phase, |this, phase| { + if matches!(phase, crate::collab::SessionPhase::Detached(_)) { + this + } else { + this.child( + Button::new("collaboration-status-pill") + .text() + .compact() + .child(crate::collab::status::participant_group(&phase, collab_roster, cx)) + .on_click(cx.listener(|workspace, _, window, cx| workspace.open_collaboration_dialog(window, cx))), + ) + } + }) .when_some(zoom, |this, percent| this.child(self.render_zoom_slider(percent, cx))) .when_some(speech_word_count, |this, count| { this.child( @@ -289,36 +344,363 @@ impl Workspace { ) }) } -} -fn speech_word_count(document: &Document) -> usize { - document - .paragraphs - .iter() - .map(|paragraph| { - let paragraph_is_tag = paragraph.style == flowstate_document::PARAGRAPH_TAG; - let mut run_start = paragraph.byte_range.start; - paragraph - .runs + fn sync_collab_notice_subscriptions(&mut self, window: &mut Window, cx: &mut Context) { + // §perf: with no existing subscriptions and no panel in a collab session, the + // full scan/retain is a no-op — skip building the set entirely (the common case). + if self.collab_notice_subscriptions.is_empty() + && !self + .document_panels .iter() - .map(|run| { - let run_end = run_start + run.len; - let count = if paragraph_is_tag || run.styles.semantic == flowstate_document::SEMANTIC_CITE || run.styles.highlight.is_some() { - count_words(&document_text_slice(document, run_start..run_end)) - } else { - 0 - }; - run_start = run_end; - count - }) - .sum::() - }) - .sum() + .any(|panel| crate::collab::session_for_panel(panel.read(cx).id(), cx).is_some()) + { + return; + } + + // §perf: FxHashSet (SessionId keys are trusted/local) with capacity for the panels. + let mut active_sessions: FxHashSet = + FxHashSet::with_capacity_and_hasher(self.document_panels.len(), rustc_hash::FxBuildHasher); + let window_handle = window.window_handle(); + + for panel in &self.document_panels { + let panel_id = panel.read(cx).id(); + let Some(session) = crate::collab::session_for_panel(panel_id, cx) else { + continue; + }; + let session_id = session.read(cx).session_id(); + active_sessions.insert(session_id); + if self.collab_notice_subscriptions.contains_key(&session_id) { + continue; + } + + let notice_window_handle = window_handle; + let subscription = cx.subscribe(&session, move |workspace, _, notice: &crate::collab::SessionNotice, cx| { + let notice = notice.clone(); + if let crate::collab::SessionNotice::IncompatibleVersion(peer) = ¬ice + && !workspace.collab_incompatible_version_notices.insert(peer.clone()) + { + return; + } + if let Err(error) = notice_window_handle.update(cx, |_, window, cx| { + crate::collab::notify::show_session_notice(¬ice, window, cx); + }) { + tracing::warn!("showing collaboration notification failed: {error}"); + } + }); + self.collab_notice_subscriptions.insert(session_id, subscription); + } + + self + .collab_notice_subscriptions + .retain(|session_id, _| active_sessions.contains(session_id)); + } +} + +// §A10.6 — status-bar speech word count. +// +// Per-document, per-paragraph word-count memo. It lives at module scope keyed +// by the owning document panel's id (this file is spliced into the `Workspace` +// module via `include!`, and the recount round-trips the map through the +// background executor); the workspace-side `speech_word_count_cache` keeps only +// the finished `(edit_generation, total)` per document. +// +// A memo entry is reused only while BOTH stamps match: +// - `version`: bumped by every editor content/formatting edit to the paragraph; +// - `shape`: FxHash of `(style, runs)` — guards the full projection rebuilds +// (remote/structural imports) that reset `version` to 0, whenever they change +// a paragraph's run structure, lengths, or styling. +struct SpeechWordCountEntry { + version: u64, + shape: u64, + words: usize, +} + +type SpeechWordCountParagraphCache = FxHashMap; + +// §perf: FxHash for trusted, locally-generated Uuid keys. +static SPEECH_WORD_COUNT_PARAGRAPH_CACHES: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| std::sync::Mutex::new(FxHashMap::default())); + +fn paragraph_shape_stamp(paragraph: &flowstate_document::Paragraph) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = rustc_hash::FxHasher::default(); + paragraph.style.hash(&mut hasher); + paragraph.runs.hash(&mut hasher); + hasher.finish() +} + +/// Incremental speech word count: sums per-paragraph counts, reusing the memo +/// for every paragraph whose `(version, shape)` stamps are unchanged and +/// recounting only the rest. `cache` is rebuilt to exactly the live paragraph +/// ids, so entries of deleted paragraphs are evicted. Returns +/// `(total, recounted_paragraphs)`; the second element backs the +/// incrementality tripwire in tests. +fn speech_word_count_incremental(document: &DocumentProjection, cache: &mut SpeechWordCountParagraphCache) -> (usize, usize) { + let mut fresh: SpeechWordCountParagraphCache = + FxHashMap::with_capacity_and_hasher(document.paragraphs.len(), rustc_hash::FxBuildHasher); + let mut total = 0usize; + let mut recounted = 0usize; + for (paragraph_ix, paragraph) in document.paragraphs.iter().enumerate() { + let paragraph_id = document.ids.paragraph_ids.get(paragraph_ix).copied(); + let shape = paragraph_shape_stamp(paragraph); + let entry = match paragraph_id.and_then(|id| cache.remove(&id)) { + Some(entry) if entry.version == paragraph.version && entry.shape == shape => entry, + _ => { + recounted += 1; + SpeechWordCountEntry { + version: paragraph.version, + shape, + words: paragraph_speech_word_count(document, paragraph_ix, paragraph), + } + }, + }; + total += entry.words; + if let Some(paragraph_id) = paragraph_id { + fresh.insert(paragraph_id, entry); + } + } + *cache = fresh; + (total, recounted) +} + +/// One paragraph's word count, with EXACTLY the original full walk's +/// semantics: a run's text counts iff the paragraph is a tag paragraph or the +/// run is cite-semantic or highlighted, and each counted run is word-split +/// independently (`split_whitespace` boundaries, keeping only words containing +/// at least one alphanumeric char) — so a word straddling a run boundary still +/// contributes once per counted run it touches. +fn paragraph_speech_word_count(document: &DocumentProjection, paragraph_ix: usize, paragraph: &flowstate_document::Paragraph) -> usize { + let paragraph_is_tag = paragraph.style == flowstate_document::PARAGRAPH_TAG; + let run_counts = |run: &flowstate_document::TextRun| { + paragraph_is_tag || run.styles.semantic == flowstate_document::SEMANTIC_CITE || run.styles.highlight.is_some() + }; + // Skip the byte-range derivation for paragraphs with no counted runs. + if !paragraph.runs.iter().any(run_counts) { + return 0; + } + let mut run_start = flowstate_document::paragraph_byte_range(document, paragraph_ix).start; + let mut words = 0usize; + for run in ¶graph.runs { + let run_end = run_start + run.len; + if run_counts(run) { + words += count_words_in_document_range(document, run_start..run_end); + } + run_start = run_end; + } + words +} + +/// Word count over a byte range of the document rope, streamed chunk-by-chunk +/// with no per-run String materialization (`document_text_slice` here was +/// 853k calls / 306.8 MB in the hotpath profile). Semantically identical to +/// `slice.split_whitespace().filter(|word| word.chars().any(char::is_alphanumeric)).count()` +/// over the materialized slice: `split_whitespace` breaks on +/// `char::is_whitespace`, and a token counts iff any of its chars is +/// alphanumeric. +fn count_words_in_document_range(document: &DocumentProjection, range: std::ops::Range) -> usize { + let len = document.text.byte_len(); + let start = range.start.min(len); + let end = range.end.min(len); + if start >= end { + return 0; + } + let mut words = 0usize; + let mut in_word = false; + let mut word_has_alphanumeric = false; + for chunk in document.text.byte_slice(start..end).chunks() { + for character in chunk.chars() { + if character.is_whitespace() { + if in_word && word_has_alphanumeric { + words += 1; + } + in_word = false; + word_has_alphanumeric = false; + } else { + in_word = true; + word_has_alphanumeric = word_has_alphanumeric || character.is_alphanumeric(); + } + } + } + if in_word && word_has_alphanumeric { + words += 1; + } + words } -fn count_words(text: &str) -> usize { - text - .split_whitespace() - .filter(|word| word.chars().any(char::is_alphanumeric)) - .count() +#[cfg(test)] +mod speech_word_count_tests { + use super::*; + use crate::rich_text_element::{HighlightStyle, RunStyles}; + + fn plain(text: &str) -> InputRun { + InputRun { + text: text.to_string(), + styles: RunStyles::default(), + } + } + + fn cite(text: &str) -> InputRun { + InputRun { + text: text.to_string(), + styles: RunStyles { + semantic: flowstate_document::SEMANTIC_CITE, + ..RunStyles::default() + }, + } + } + + fn highlighted(text: &str) -> InputRun { + InputRun { + text: text.to_string(), + styles: RunStyles { + highlight: Some(HighlightStyle::Custom(0)), + ..RunStyles::default() + }, + } + } + + fn fixture_inputs() -> Vec { + vec![ + // Tag paragraph: EVERY run counts, styled or not (6 words). + InputParagraph { + style: flowstate_document::PARAGRAPH_TAG, + runs: vec![plain("Tag heading words here"), cite(" plus cite")], + }, + // Normal paragraph: only the cite run counts; "spl|it" straddles the + // cite→plain boundary and must split per run (3 words, not 2 or 5). + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![cite("cited evidence spl"), plain("it uncounted words")], + }, + // A straddle between two counted runs stays two run-local tokens + // ("foo" + "bar baz" = 3 words, not "foobar baz" = 2). + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![highlighted("foo"), cite("bar baz")], + }, + // Punctuation-only tokens are filtered; U+2028 is whitespace (2 words). + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![highlighted("-- !! a1\u{2028}b2"), plain("nothing here counts")], + }, + // No counted runs at all (0 words). + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("entirely uncounted text")], + }, + ] + } + + fn build_document(inputs: Vec, ids: &[u128]) -> DocumentProjection { + assert_eq!(inputs.len(), ids.len()); + let mut document = document_from_input(flowstate_document_theme(), inputs); + document.ids.paragraph_ids = Arc::new(ids.iter().copied().map(flowstate_document::ParagraphId).collect()); + document + } + + /// The pre-A10.6 full walk, verbatim — the counting-semantics oracle. + fn speech_word_count_reference(document: &DocumentProjection) -> usize { + document + .paragraphs + .iter() + .enumerate() + .map(|(paragraph_ix, paragraph)| { + let paragraph_is_tag = paragraph.style == flowstate_document::PARAGRAPH_TAG; + let mut run_start = flowstate_document::paragraph_byte_range(document, paragraph_ix).start; + paragraph + .runs + .iter() + .map(|run| { + let run_end = run_start + run.len; + let count = if paragraph_is_tag || run.styles.semantic == flowstate_document::SEMANTIC_CITE || run.styles.highlight.is_some() { + document_text_slice(document, run_start..run_end) + .split_whitespace() + .filter(|word| word.chars().any(char::is_alphanumeric)) + .count() + } else { + 0 + }; + run_start = run_end; + count + }) + .sum::() + }) + .sum() + } + + #[test] + fn incremental_speech_word_count_matches_fresh_full_count_across_edits() { + let ids: Vec = (1..=5).collect(); + let base = build_document(fixture_inputs(), &ids); + let mut cache = SpeechWordCountParagraphCache::default(); + + // Cold count == the oracle, every paragraph recounted. + let (initial, recounted) = speech_word_count_incremental(&base, &mut cache); + assert_eq!(initial, speech_word_count_reference(&base)); + assert_eq!(initial, 14, "fixture semantics moved: tag=6, cite-straddle=3, two-run straddle=3, filtered=2, uncounted=0"); + assert_eq!(recounted, base.paragraphs.len()); + + // Unchanged document: everything served from the memo. + let (unchanged, recounted) = speech_word_count_incremental(&base, &mut cache); + assert_eq!(unchanged, initial); + assert_eq!(recounted, 0); + + // (a1) Same-length text swap + the editor's content-edit version bump: + // version alone must invalidate the memo (shape is unchanged). + let mut inputs = fixture_inputs(); + inputs[1].runs[0] = cite("citedevidences spl"); // same 18 bytes, 2 words instead of 3 + let mut edited = build_document(inputs, &ids); + let mut bumped = edited.paragraphs[1].clone(); + bumped.version = bumped.version.wrapping_add(1); + edited.paragraphs.set(1, bumped); + let (count, recounted) = speech_word_count_incremental(&edited, &mut cache); + assert_eq!(count, speech_word_count_reference(&edited)); + assert_eq!(count, 13); + assert_eq!(recounted, 1, "only the edited paragraph recounts"); + + // (a2) Length-changing edit to the same paragraph. + let mut inputs = fixture_inputs(); + inputs[1].runs[0] = cite("cited evidence rewritten with more words spl"); + let mut edited = build_document(inputs, &ids); + let mut bumped = edited.paragraphs[1].clone(); + bumped.version = bumped.version.wrapping_add(1); + edited.paragraphs.set(1, bumped); + let (count, recounted) = speech_word_count_incremental(&edited, &mut cache); + assert_eq!(count, speech_word_count_reference(&edited)); + assert_eq!(count, 18); + assert_eq!(recounted, 1); + + // (b) Delete a paragraph: nothing recounts, its memo entry is evicted. + let mut inputs = fixture_inputs(); + inputs.remove(1); + let mut remaining_ids = ids.clone(); + remaining_ids.remove(1); + let deleted = build_document(inputs, &remaining_ids); + let (count, recounted) = speech_word_count_incremental(&deleted, &mut cache); + assert_eq!(count, speech_word_count_reference(&deleted)); + assert_eq!(count, 11); + assert_eq!(recounted, 0, "surviving paragraphs are all memo hits"); + assert!(!cache.contains_key(&flowstate_document::ParagraphId(2)), "deleted paragraph's memo must be evicted"); + assert_eq!(cache.len(), deleted.paragraphs.len()); + + // (c) Insert a paragraph: only the new one recounts. + let mut inputs = fixture_inputs(); + inputs.remove(1); + inputs.insert( + 2, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![cite("brand new inserted card"), plain(" tail")], + }, + ); + let mut inserted_ids = remaining_ids.clone(); + inserted_ids.insert(2, 99); + let inserted = build_document(inputs, &inserted_ids); + let (count, recounted) = speech_word_count_incremental(&inserted, &mut cache); + assert_eq!(count, speech_word_count_reference(&inserted)); + assert_eq!(count, 15); + assert_eq!(recounted, 1, "only the inserted paragraph recounts"); + assert!(cache.contains_key(&flowstate_document::ParagraphId(99))); + assert_eq!(cache.len(), inserted.paragraphs.len()); + } } diff --git a/crates/flowstate/src/workspace/workspace/render_top_bar.rs b/crates/flowstate/src/workspace/workspace/render_top_bar.rs index 0990071f..a39caaff 100644 --- a/crates/flowstate/src/workspace/workspace/render_top_bar.rs +++ b/crates/flowstate/src/workspace/workspace/render_top_bar.rs @@ -2,6 +2,10 @@ impl Workspace { fn render_top_bar(&mut self, _window: &Window, cx: &mut Context) -> impl IntoElement { let workspace = cx.entity().downgrade(); + let active_collaborating = self + .active_document_id + .and_then(|panel_id| crate::collab::phase_for_panel(panel_id, cx)) + .is_some_and(|phase| !matches!(phase, crate::collab::SessionPhase::Detached(_))); TitleBar::new() .on_close_window(move |_, window, cx| { let _ = workspace.update(cx, |workspace, cx| workspace.request_close_window(window, cx)); @@ -16,12 +20,15 @@ impl Workspace { .child(file_top_bar_button(self.active_document_id.is_some(), cx)) .child(insert_top_bar_button(cx, self.active_editor.is_some())) .child(document_top_bar_button(cx)) + .child(collaboration_top_bar_button(cx, self.active_document_id.is_some(), active_collaborating)) .child(view_top_bar_button( cx, !self.outline_collapsed, !self.ribbon_collapsed, !self.toolkit_collapsed, )) + .child(div().flex_1()) + .child(share_top_bar_button(cx, self.active_editor.is_some(), active_collaborating)) .child(settings_top_bar_button(cx)), ) } diff --git a/crates/flowstate/src/workspace/workspace/top_bar.rs b/crates/flowstate/src/workspace/workspace/top_bar.rs index d9397c6e..f2ba8d8e 100644 --- a/crates/flowstate/src/workspace/workspace/top_bar.rs +++ b/crates/flowstate/src/workspace/workspace/top_bar.rs @@ -17,6 +17,11 @@ fn flowstate_top_bar_button(cx: &mut Context) -> impl IntoElement { .on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation()) .child( Button::new("top-flowstate") + .icon( + Icon::default() + .path("logo/flowstate-mark.svg") + .with_size(px(13.0)), + ) .label("Flowstate") .xsmall() .ghost() @@ -106,6 +111,70 @@ fn document_top_bar_button(cx: &mut Context) -> impl IntoElement { ) } +fn collaboration_top_bar_button(cx: &mut Context, has_document: bool, active_collaborating: bool) -> impl IntoElement { + let workspace = cx.entity().downgrade(); + div() + .h_full() + .flex_none() + .flex() + .items_center() + .justify_center() + .on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation()) + .child( + Button::new("top-collaborate") + .label("Collaborate") + .xsmall() + .ghost() + .dropdown_menu(move |menu, _, _| { + menu + .item(file_menu_item( + workspace.clone(), + "Share / Collaborate...", + !has_document, + |workspace, window, cx| { + workspace.open_collaboration_dialog(window, cx); + }, + )) + .item(file_menu_item( + workspace.clone(), + "Copy Invite Ticket", + !has_document, + |workspace, window, cx| { + workspace.copy_active_collaboration_ticket(window, cx); + }, + )) + .item(file_menu_item( + workspace.clone(), + "Comments...", + !has_document, + |workspace, window, cx| { + workspace.open_comment_dialog(window, cx); + }, + )) + .separator() + .item(file_menu_item(workspace.clone(), "Join Session...", false, |workspace, window, cx| { + workspace.open_join_collaboration_dialog(window, cx); + })) + .item(file_menu_item( + workspace.clone(), + "Join from Clipboard", + false, + |workspace, window, cx| { + workspace.join_collaboration_from_clipboard(window, cx); + }, + )) + .item(file_menu_item( + workspace.clone(), + "Leave Shared Session", + !active_collaborating, + |workspace, window, cx| { + workspace.confirm_leave_collaboration_on_active_document(window, cx); + }, + )) + }), + ) +} + #[hotpath::measure] fn file_top_bar_button(has_document: bool, cx: &mut Context) -> impl IntoElement { let workspace = cx.entity().downgrade(); @@ -143,6 +212,18 @@ fn file_top_bar_button(has_document: bool, cx: &mut Context) -> impl workspace.save_active_as(window, cx); })) .separator() + .item(file_menu_item( + workspace.clone(), + "Share Document...", + !has_document, + |workspace, window, cx| { + workspace.open_collaboration_dialog(window, cx); + }, + )) + .item(file_menu_item(workspace.clone(), "Join Session...", false, |workspace, window, cx| { + workspace.open_join_collaboration_dialog(window, cx); + })) + .separator() .item(file_menu_item(workspace.clone(), "Close File", !has_document, |workspace, window, cx| { workspace.close_active_document(window, cx); })) @@ -213,6 +294,38 @@ fn insert_top_bar_button(cx: &mut Context, has_document: bool) -> imp ) } +#[hotpath::measure] +fn share_top_bar_button(cx: &mut Context, has_document: bool, collaborating: bool) -> impl IntoElement { + let workspace = cx.entity().downgrade(); + let button = Button::new("top-share-document") + .label("Share") + .xsmall() + .ghost() + .disabled(!has_document) + .tooltip("Share / Collaborate") + .on_click(move |_, window, cx| { + let _ = workspace.update(cx, |workspace, cx| workspace.open_collaboration_dialog(window, cx)); + }); + + div() + .h_full() + .flex_none() + .flex() + .items_center() + .justify_center() + .on_mouse_down(MouseButton::Left, |_, _, cx| cx.stop_propagation()) + .child(if collaborating { + button.custom( + ButtonCustomVariant::new(cx) + .foreground(cx.theme().success) + .hover(cx.theme().success.opacity(0.12)) + .active(cx.theme().success.opacity(0.18)), + ) + } else { + button + }) +} + #[hotpath::measure] fn insert_image_from_top_bar(workspace: &WeakEntity, cx: &mut App) { let _ = workspace.update(cx, |workspace, cx| { @@ -256,18 +369,22 @@ fn settings_top_bar_button(cx: &mut Context) -> impl IntoElement { .xsmall() .ghost() .dropdown_menu(move |menu, _, _| { - [WorkspaceSettingsSection::General, WorkspaceSettingsSection::Keymap] - .into_iter() - .fold(menu, |menu, section| { - let workspace = workspace.clone(); - menu.item(PopupMenuItem::new(section.title()).on_click(move |_, _, cx| { - let _ = workspace.update(cx, |workspace, cx| { - workspace.settings_section = section; - workspace.settings_overlay = Some(WorkspaceSettingsOverlay::Settings); - cx.notify(); - }); - })) - }) + [ + WorkspaceSettingsSection::General, + WorkspaceSettingsSection::Collaboration, + WorkspaceSettingsSection::Keymap, + ] + .into_iter() + .fold(menu, |menu, section| { + let workspace = workspace.clone(); + menu.item(PopupMenuItem::new(section.title()).on_click(move |_, _, cx| { + let _ = workspace.update(cx, |workspace, cx| { + workspace.settings_section = section; + workspace.settings_overlay = Some(WorkspaceSettingsOverlay::Settings); + cx.notify(); + }); + })) + }) }), ) } @@ -307,7 +424,7 @@ fn view_top_bar_button(cx: &mut Context, outline_open: bool, ribbon_o }), ) .item( - PopupMenuItem::new("Toolkit") + PopupMenuItem::new("Toolkit") .checked(toolkit_open) .on_click(move |_, _, cx| { let _ = toolkit_workspace.update(cx, |workspace, cx| workspace.toggle_toolkit(cx)); diff --git a/crates/flowstate/src/workspace/workspace/traits.rs b/crates/flowstate/src/workspace/workspace/traits.rs index 4111d93b..bfe9caf6 100644 --- a/crates/flowstate/src/workspace/workspace/traits.rs +++ b/crates/flowstate/src/workspace/workspace/traits.rs @@ -1,32 +1,42 @@ #[hotpath::measure_all] impl Render for Workspace { fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + // Window-level mouse listeners may only be registered during the paint + // phase (gpui debug-asserts this; render runs during layout), so the + // ctrl+scroll zoom hook rides a zero-cost canvas child's paint pass. let workspace = cx.entity().downgrade(); - window.on_mouse_event(move |event: &gpui::ScrollWheelEvent, _, window, cx| { - if event.modifiers.control { - let delta = event.delta.pixel_delta(window.line_height()); - if let Some(workspace) = workspace.upgrade() { - workspace.update(cx, |workspace, cx| { - if let Some(editor) = workspace.active_editor.clone() { - if delta.y < px(0.0) { - editor.update(cx, |editor, cx| editor.zoom_in(cx)); - } else { - editor.update(cx, |editor, cx| editor.zoom_out(cx)); - } + let zoom_wheel_hook = gpui::canvas( + |_, _, _| (), + move |_, _, window, _| { + window.on_mouse_event(move |event: &gpui::ScrollWheelEvent, _, window, cx| { + if event.modifiers.control { + let delta = event.delta.pixel_delta(window.line_height()); + if let Some(workspace) = workspace.upgrade() { + workspace.update(cx, |workspace, cx| { + if let Some(editor) = workspace.active_editor.clone() { + if delta.y < px(0.0) { + editor.update(cx, |editor, cx| editor.zoom_in(cx)); + } else { + editor.update(cx, |editor, cx| editor.zoom_out(cx)); + } + } + }); } - }); - } - cx.stop_propagation(); - } - }); + cx.stop_propagation(); + } + }); + }, + ); div() .size_full() .relative() + .child(zoom_wheel_hook) .child( v_flex() .on_action(cx.listener(Self::on_save)) .on_action(cx.listener(Self::on_find_in_document)) + .on_action(cx.listener(Self::on_fidelity_mark)) .on_action(cx.listener(Self::on_zoom_in)) .on_action(cx.listener(Self::on_zoom_out)) .size_full() @@ -74,5 +84,7 @@ impl Render for Workspace { .with_priority(1), ) }) + .when_some(Root::render_dialog_layer(window, cx), |this, layer| this.child(layer)) + .when_some(Root::render_notification_layer(window, cx), |this, layer| this.child(layer)) } } diff --git a/crates/flowstate/src/workspace/workspace/window.rs b/crates/flowstate/src/workspace/workspace/window.rs index 62829fb0..861ffd92 100644 --- a/crates/flowstate/src/workspace/workspace/window.rs +++ b/crates/flowstate/src/workspace/workspace/window.rs @@ -9,8 +9,92 @@ pub fn install_workspace_close_prompt(workspace: Entity, window: &mut return true; } + let collaboration_panels = workspace.read(cx).collaboration_close_panels(cx); + if !collaboration_panels.is_empty() { + if prompt_open.get() { + return false; + } + prompt_open.set(true); + + let collaboration_ids = collaboration_panels + .iter() + .map(|panel| panel.id) + .collect::>(); + let dirty_panels = workspace + .read(cx) + .dirty_panels(cx) + .into_iter() + .filter(|panel| !collaboration_ids.contains(&panel.panel_id(cx))) + .collect::>(); + let answer = if collaboration_panels.len() == 1 { + let phase = crate::collab::phase_for_panel(collaboration_panels[0].id, cx); + window.prompt( + PromptLevel::Warning, + "Leave the collaboration session and quit?", + Some(&collaboration_leave_detail(phase.as_ref())), + &[PromptButton::ok("Leave"), PromptButton::cancel("Cancel")], + cx, + ) + } else { + let detail = format!( + "You're in {} collaboration sessions. Leave all and quit?", + collaboration_panels.len() + ); + window.prompt( + PromptLevel::Warning, + "Leave collaboration sessions?", + Some(&detail), + &[PromptButton::ok("Leave"), PromptButton::cancel("Cancel")], + cx, + ) + }; + let prompt_open = prompt_open.clone(); + let allow_close = allow_close.clone(); + let workspace = workspace.clone(); + + cx.spawn(async move |cx| { + let mut should_close = false; + if matches!(answer.await, Ok(0)) { + let mut discards = Vec::new(); + let mut ok = true; + for panel in collaboration_panels { + match resolve_collaboration_save(panel.panel, panel.save_prompt, window_handle, cx).await { + CollaborationCloseResolution::Proceed { discard } => { + if let Some(discard) = discard { + discards.push(discard); + } + }, + CollaborationCloseResolution::Cancelled => { + ok = false; + break; + }, + } + } + if ok && resolve_dirty_window_close(dirty_panels, window_handle, cx).await { + for panel in discards { + panel.discard(cx); + } + should_close = true; + } + } + + prompt_open.set(false); + if should_close { + allow_close.set(true); + let _ = window_handle.update(cx, |_, window, cx| { + workspace.update(cx, |workspace, cx| workspace.leave_all_collaboration_sessions(cx)); + window.remove_window(); + }); + } + }) + .detach(); + + return false; + } + let dirty_panels = workspace.read(cx).dirty_panels(cx); if dirty_panels.is_empty() { + workspace.update(cx, |workspace, cx| workspace.leave_all_collaboration_sessions(cx)); return true; } @@ -33,6 +117,7 @@ pub fn install_workspace_close_prompt(workspace: Entity, window: &mut ); let prompt_open = prompt_open.clone(); let allow_close = allow_close.clone(); + let workspace = workspace.clone(); cx.spawn(async move |cx| { let should_close = match answer.await { @@ -65,6 +150,7 @@ pub fn install_workspace_close_prompt(workspace: Entity, window: &mut prompt_open.set(false); if should_close { + let _ = workspace.update(cx, |workspace, cx| workspace.leave_all_collaboration_sessions(cx)); allow_close.set(true); let _ = window_handle.update(cx, |_, window, _| window.remove_window()); } @@ -76,11 +162,20 @@ pub fn install_workspace_close_prompt(workspace: Entity, window: &mut } #[hotpath::measure] -pub fn open_workspace_window(document_path: Option, cx: &mut App) { +pub fn open_workspace_window(document_path: Option, cx: &mut App) -> WeakEntity { + let initial_invite = document_path + .as_ref() + .and_then(|path| path.to_str()) + .filter(|text| text.starts_with(flowstate_collab::ticket::INVITE_URL_PREFIX)) + .map(flowstate_collab::ticket::SessionTicket::decode_text); + let document_path = if initial_invite.is_some() { None } else { document_path }; let window_bounds = startup_window_bounds(cx); + let opened_workspace = Rc::new(RefCell::new(None)); + let opened_workspace_slot = opened_workspace.clone(); cx.open_window( WindowOptions { window_bounds, + app_id: Some("dev.flowstate.Flowstate".to_string()), titlebar: Some(TitleBar::title_bar_options()), window_decorations: window_decorations(), ..Default::default() @@ -88,11 +183,27 @@ pub fn open_workspace_window(document_path: Option, cx: &mut App) { |window, cx| { window.set_window_title("Flowstate"); let workspace = cx.new(|cx| Workspace::new(document_path, window, cx)); + if let Some(invite) = initial_invite { + match invite { + Ok(ticket) => workspace.update(cx, |workspace, cx| { + let _ = workspace.join_collaboration_session(ticket, window, cx); + }), + Err(error) => { + let detail = format!("This Flowstate collaboration link is invalid or damaged: {error}"); + std::mem::drop(window.prompt(PromptLevel::Critical, "Invite couldn't be opened", Some(&detail), &[PromptButton::ok("Ok")], cx)); + }, + } + } install_workspace_close_prompt(workspace.clone(), window, cx); + *opened_workspace_slot.borrow_mut() = Some(workspace.downgrade()); cx.new(|cx| Root::new(workspace, window, cx)) }, ) .unwrap(); + opened_workspace + .borrow_mut() + .take() + .expect("workspace window builder must install its workspace") } #[cfg(target_os = "windows")] diff --git a/crates/flowstate/src/workspace/workspace/workspace_state.rs b/crates/flowstate/src/workspace/workspace/workspace_state.rs index b54807c1..4d4a8372 100644 --- a/crates/flowstate/src/workspace/workspace/workspace_state.rs +++ b/crates/flowstate/src/workspace/workspace/workspace_state.rs @@ -324,10 +324,20 @@ impl Workspace { fn active_document_index(&self, cx: &App) -> Option { let active_id = self.active_document_id?; - self - .document_tabs(cx) + // §perf: find the active tab's index by scanning panel ids in tab order (document + // panels then flow panels, then stably pinned-first) without materializing labeled + // tabs (truncate + format! per tab). Mirrors document_tabs/ordered_document_tabs exactly. + let mut ids: Vec = self + .document_panels .iter() - .position(|tab| tab.id == active_id) + .map(|panel| panel.read(cx).id()) + .chain(self.flow_panels.iter().map(|panel| panel.read(cx).id())) + .collect(); + ids.sort_by_key(|id| { + let pin_index = self.pinned_document_ids.iter().position(|pinned| pinned == id); + (pin_index.is_none(), pin_index.unwrap_or(usize::MAX)) + }); + ids.iter().position(|id| *id == active_id) } fn activate_document_at_index(&mut self, index: usize, cx: &mut Context) { @@ -350,7 +360,9 @@ impl Workspace { return; } let target = if offset.is_negative() { - active_index.wrapping_sub(offset.unsigned_abs()) % len + // usize::MAX % len (the wrapping_sub shortcut) only wraps correctly + // when len is a power of two; add len before subtracting instead. + (active_index + len - (offset.unsigned_abs() % len)) % len } else { (active_index + offset as usize) % len }; @@ -400,11 +412,20 @@ impl Workspace { } fn activate_tab_shortcut(&mut self, index: usize, cx: &mut Context) { + // §perf: build the set of live panel ids once instead of rebuilding the entire + // labeled tab Vec for every pinned id. A tab exists for each document/flow panel, + // so membership in this set is equivalent to matching some tab.id. + let live_ids: FxHashSet = self + .document_panels + .iter() + .map(|panel| panel.read(cx).id()) + .chain(self.flow_panels.iter().map(|panel| panel.read(cx).id())) + .collect(); let pinned = self .pinned_document_ids .iter() .copied() - .filter(|id| self.document_tabs(cx).iter().any(|tab| tab.id == *id)) + .filter(|id| live_ids.contains(id)) .collect::>(); if let Some(id) = pinned.get(index).copied() { self.activate_document_id(id, cx); @@ -733,7 +754,7 @@ fn card_paragraph_excluded_from_condense(paragraph: &InputParagraph) -> bool { } fn selected_fragment_or_enclosing_section( - document: &Document, + document: &DocumentProjection, selection: &crate::rich_text_element::EditorSelection, ) -> crate::rich_text_element::RichClipboardFragment { if selection.anchor != selection.head { @@ -763,9 +784,9 @@ fn selected_fragment_or_enclosing_section( ) } -fn enclosing_section_bounds(document: &Document, paragraph_ix: usize, section_slots: &[u8]) -> Option<(usize, usize)> { +fn enclosing_section_bounds(document: &DocumentProjection, paragraph_ix: usize, section_slots: &[u8]) -> Option<(usize, usize)> { document - .sections + .outline .iter() .filter_map(|section| { let SectionKind::Custom(slot) = section.kind; diff --git a/crates/gpui-flowtext/Cargo.toml b/crates/gpui-flowtext/Cargo.toml index 59b84aa0..2297e025 100644 --- a/crates/gpui-flowtext/Cargo.toml +++ b/crates/gpui-flowtext/Cargo.toml @@ -9,18 +9,21 @@ license = "MIT" [lints] workspace = true [dependencies] +blake3 = "1.8.5" crop = { workspace = true } +flowstate-fidelity.workspace = true gpui = { workspace = true } gpui-component = { workspace = true } hotpath = { workspace = true } +im = "15.1.0" imagesize = { workspace = true } mitex = { workspace = true } -postcard = { version = "1.1.3", features = ["use-std"] } resvg = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } +tracing.workspace = true typst = { workspace = true } typst-assets = { workspace = true, features = ["fonts"] } typst-library = { workspace = true } @@ -31,6 +34,11 @@ uuid = { workspace = true } [target.'cfg(any(target_os = "macos", target_os = "linux"))'.dependencies] cosmic-text = { workspace = true } +[dev-dependencies] +flowstate-collab = { path = "../flowstate-collab" } +flowstate-document.workspace = true +gpui = { workspace = true, features = ["test-support"] } +loro = "1.13" [features] hotpath = ["hotpath/hotpath"] diff --git a/crates/gpui-flowtext/README.md b/crates/gpui-flowtext/README.md index 729b45ae..fbcfae50 100644 --- a/crates/gpui-flowtext/README.md +++ b/crates/gpui-flowtext/README.md @@ -26,7 +26,7 @@ This project is early and actively being extracted from a production GPUI applic - DOCX/PDF exporters by default - A prescribed file extension for host applications -The default native extension is `gptx`, but applications can use their own extension when exporting or saving native documents. +Native persistence is supplied by the host application. Flowstate uses its Loro-native `.db8` package adapter. ## API Tiers diff --git a/crates/gpui-flowtext/src/api.rs b/crates/gpui-flowtext/src/api.rs index 6d2694d9..b329c7f8 100644 --- a/crates/gpui-flowtext/src/api.rs +++ b/crates/gpui-flowtext/src/api.rs @@ -2,7 +2,9 @@ use std::{borrow::Cow, path::Path, sync::Arc}; use gpui::Pixels; -use crate::{Document, DocumentExportFormat, EditorSelection, HighlightStyle, ParagraphStyle, RichTextEditorCommand, RunSemanticStyle}; +use crate::{ + DocumentExportFormat, DocumentProjection, EditorSelection, HighlightStyle, ParagraphStyle, RichTextEditorCommand, RunSemanticStyle, +}; #[derive(Clone, Debug, Eq, PartialEq, Hash)] pub struct StyleId(pub Cow<'static, str>); @@ -105,10 +107,26 @@ impl Default for LayoutPolicy { #[derive(Clone, Debug)] pub enum EditorEvent { - Changed { edit_generation: u64 }, - SelectionChanged { selection: EditorSelection }, - CommandDispatched { command: RichTextEditorCommand }, - Exported { format: DocumentExportFormat }, + Changed { + edit_generation: u64, + }, + SelectionChanged { + selection: EditorSelection, + }, + CommandDispatched { + command: RichTextEditorCommand, + }, + Exported { + format: DocumentExportFormat, + }, + /// The editor could not reconcile optimistic local state with the canonical + /// projection and recovered by discarding or rebuilding. Hosts must surface + /// this (telemetry at minimum); it indicates replay/projection divergence. + ReconciliationRecovery { + dropped_batches: usize, + reason: gpui::SharedString, + total_recoveries: u64, + }, } pub trait EditorEventSink: Send + Sync + 'static { @@ -120,10 +138,10 @@ pub trait AssetResolver: Send + Sync + 'static { } pub trait DocumentSerializer: Send + Sync + 'static { - fn read(&self, bytes: &[u8]) -> std::io::Result; - fn write(&self, document: &Document) -> std::io::Result>; + fn read(&self, bytes: &[u8]) -> std::io::Result; + fn write(&self, document: &DocumentProjection) -> std::io::Result>; } pub trait ExternalFormatExporter: Send + Sync + 'static { - fn write_external_format(&self, output_path: &Path, document: &Document, format: DocumentExportFormat) -> std::io::Result<()>; + fn write_external_format(&self, output_path: &Path, document: &DocumentProjection, format: DocumentExportFormat) -> std::io::Result<()>; } diff --git a/crates/gpui-flowtext/src/block_tree.rs b/crates/gpui-flowtext/src/block_tree.rs new file mode 100644 index 00000000..a6120715 --- /dev/null +++ b/crates/gpui-flowtext/src/block_tree.rs @@ -0,0 +1,853 @@ +//! §act-four M3 — the persistent, monoid-annotated block tree. +//! +//! This is the read model's spine: a **persistent** (immutable, structurally +//! shared) sequence of document blocks that replaces `Arc>` + +//! `Arc>` + the Fenwick offset index with ONE structure. It is an +//! **implicit treap** (a Cartesian tree keyed by position, heap-ordered by a +//! well-distributed random priority per node), which gives: +//! +//! - `O(1)` clone — a new version shares every unchanged node (one `Arc` bump); +//! - `O(log N)` `split` / `join` / `splice` — a mutation path-copies only the +//! `O(log N)` nodes root-to-cut and shares the rest, so a new version costs +//! `O(change · log N)` and an old version costs `O(1)` to keep (undo = restore +//! a root); +//! - `O(log N)` monoid queries — each node caches its subtree [`Summary`] +//! (`blocks` count + `bytes` = the document-text byte measure), so +//! `offset ↔ block` and `block → byte offset` are logarithmic descents with +//! NO separate index. +//! +//! Leaves hold `Arc` (shared across versions that didn't touch the +//! block; M4 turns these into lazily-materialized handles). The tree is generic +//! only in spirit — it is concrete to `Block` here so the byte measure and the +//! editor's accessors line up exactly. + +use std::sync::Arc; + +use crate::{Block, Paragraph, paragraph_runs_len}; + +/// The monoid carried by every node. All four components add componentwise +/// (identity `{0,0,0,0}`, associative), so subtree summaries compose bottom-up: +/// +/// - `blocks` — block count (positional split space); +/// - `paragraphs` — paragraph-block count (paragraph-rank space); +/// - `para_text_bytes` — `Σ runs_len` over paragraph blocks only (objects +/// contribute 0) — the `document.text` paragraph-rope coordinate, from which +/// `paragraph_start` is derived (subsuming the Fenwick `ParagraphOffsetIndex`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Summary { + pub blocks: usize, + pub paragraphs: usize, + pub para_text_bytes: usize, +} + +impl Summary { + #[must_use] + pub fn of_block(block: &Block) -> Self { + let (paragraphs, para_text_bytes) = match block { + Block::Paragraph(paragraph) => (1, paragraph_runs_len(paragraph)), + Block::Image(_) | Block::Equation(_) | Block::Table(_) => (0, 0), + }; + Self { + blocks: 1, + paragraphs, + para_text_bytes, + } + } + + #[must_use] + fn combine(self, other: Self) -> Self { + Self { + blocks: self.blocks + other.blocks, + paragraphs: self.paragraphs + other.paragraphs, + para_text_bytes: self.para_text_bytes + other.para_text_bytes, + } + } +} + +/// A persistent treap node. Caches subtree size (positional split) and +/// [`Summary`] (offset queries); `priority` heap-orders the treap (balance). +/// `Clone` backs the copy-on-write in-place maps (`update_at`): +/// `Arc::make_mut` clones a node only when a retained version still shares it, +/// so an owned tree mutates in place. +#[derive(Clone, Debug)] +struct Node { + value: Arc, + left: Option>, + right: Option>, + priority: u64, + size: usize, + summary: Summary, +} + +#[allow( + clippy::ref_option, + reason = "the &Option form threads cleanly through the recursive treap walk; per-call .as_ref() adds noise without benefit" +)] +#[must_use] +fn size(node: &Option>) -> usize { + node.as_ref().map_or(0, |node| node.size) +} + +#[allow( + clippy::ref_option, + reason = "the &Option form threads cleanly through the recursive treap walk; per-call .as_ref() adds noise without benefit" +)] +#[must_use] +fn summary(node: &Option>) -> Summary { + node + .as_ref() + .map_or(Summary::default(), |node| node.summary) +} + +/// Well-distributed priorities from a global counter via splitmix64 — no +/// wall-clock/RNG dependency, so tests are reproducible, yet the values are +/// scrambled enough to keep the treap `O(log N)` in expectation. +fn next_priority() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let mut z = COUNTER + .fetch_add(0x9E37_79B9_7F4A_7C15, Ordering::Relaxed) + .wrapping_add(0x9E37_79B9_7F4A_7C15); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// Assemble a node from a value + priority + subtrees, recomputing the cached +/// size/summary. The caller preserves each existing node's priority through +/// split/merge so the heap order (and thus balance) is stable. +#[must_use] +fn node(value: Arc, priority: u64, left: Option>, right: Option>) -> Arc { + let value_summary = Summary::of_block(&value); + Arc::new(Node { + size: 1 + size(&left) + size(&right), + summary: summary(&left) + .combine(value_summary) + .combine(summary(&right)), + priority, + value, + left, + right, + }) +} + +#[must_use] +fn singleton(value: Arc) -> Arc { + node(value, next_priority(), None, None) +} + +/// Merge two subtrees (every element of `left` precedes every element of +/// `right`) into a valid treap. `O(log N)` expected. +fn merge(left: Option>, right: Option>) -> Option> { + match (left, right) { + (None, right) => right, + (left, None) => left, + (Some(l), Some(r)) => { + if l.priority >= r.priority { + let merged = merge(l.right.clone(), Some(r)); + Some(node(l.value.clone(), l.priority, l.left.clone(), merged)) + } else { + let merged = merge(Some(l), r.left.clone()); + Some(node(r.value.clone(), r.priority, merged, r.right.clone())) + } + }, + } +} + +#[allow( + clippy::ref_option, + reason = "the &Option form threads cleanly through the recursive treap walk; per-call .as_ref() adds noise without benefit" +)] +/// Split a subtree so the left part holds the first `k` elements. `O(log N)`. +fn split(root: &Option>, k: usize) -> (Option>, Option>) { + let Some(current) = root else { + return (None, None); + }; + let left_size = size(¤t.left); + if k <= left_size { + let (ll, lr) = split(¤t.left, k); + (ll, Some(node(current.value.clone(), current.priority, lr, current.right.clone()))) + } else { + let (rl, rr) = split(¤t.right, k - left_size - 1); + (Some(node(current.value.clone(), current.priority, current.left.clone(), rl)), rr) + } +} + +/// Sum of `para_text_bytes` over the first `k` PARAGRAPH blocks of the subtree +/// (objects are skipped — they hold 0 paragraphs). Descends by paragraph-rank +/// using the cached `paragraphs` counts. `O(log N)`. +#[allow( + clippy::ref_option, + reason = "recursive treap walk over &Option children, consistent with split/size/summary" +)] +fn prefix_para_text_bytes(node: &Option>, k: usize) -> usize { + let Some(current) = node else { + return 0; + }; + if k == 0 { + return 0; + } + let left_paragraphs = summary(¤t.left).paragraphs; + if k <= left_paragraphs { + return prefix_para_text_bytes(¤t.left, k); + } + let mut acc = summary(¤t.left).para_text_bytes; + let mut remaining = k - left_paragraphs; + let value = Summary::of_block(¤t.value); + if value.paragraphs == 1 { + // The current block is the (left_paragraphs + 1)-th paragraph; include it. + acc += value.para_text_bytes; + remaining -= 1; + } + acc + prefix_para_text_bytes(¤t.right, remaining) +} + +/// Recompute a node's cached `summary` from its (possibly just-mutated) +/// children and value. `size` is unaffected — these maps never restructure. +fn refresh_summary(node: &mut Node) { + node.summary = summary(&node.left) + .combine(Summary::of_block(&node.value)) + .combine(summary(&node.right)); +} + +/// COW descent to `index`, applying `edit` to that block in place. `make_mut` +/// clones only nodes still shared with a retained version. +fn update_node_at(node: &mut Option>, index: usize, edit: impl FnOnce(&mut Block)) { + let Some(arc) = node else { + return; + }; + let current = Arc::make_mut(arc); + let left_size = size(¤t.left); + match index.cmp(&left_size) { + std::cmp::Ordering::Less => update_node_at(&mut current.left, index, edit), + std::cmp::Ordering::Equal => edit(Arc::make_mut(&mut current.value)), + std::cmp::Ordering::Greater => update_node_at(&mut current.right, index - left_size - 1, edit), + } + refresh_summary(current); +} + +/// Build a valid treap from a slice of blocks. `O(n log n)` via repeated merge +/// (bulk build is not on any hot path). +fn build(blocks: &[Arc]) -> Option> { + let mut root = None; + for block in blocks { + root = merge(root, Some(singleton(block.clone()))); + } + root +} + +/// The persistent block sequence. Cloning is `O(1)`; mutations return a new +/// tree sharing all untouched structure. +#[derive(Clone, Debug, Default)] +pub struct BlockTree { + root: Option>, +} + +impl BlockTree { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + #[must_use] + pub fn from_blocks(blocks: Vec) -> Self { + let leaves: Vec> = blocks.into_iter().map(Arc::new).collect(); + Self { root: build(&leaves) } + } + + #[must_use] + pub fn len(&self) -> usize { + size(&self.root) + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.root.is_none() + } + + /// The block at `index`, or `None` if out of range. `O(log N)`. + #[must_use] + pub fn get(&self, index: usize) -> Option> { + self.node_at(index).map(|node| node.value.clone()) + } + + /// A borrow of the block at `index`, tied to `&self` — the zero-copy form the + /// editor's `document.blocks[i]` / `.get(i)` accessors want. `O(log N)`. + #[must_use] + pub fn get_ref(&self, index: usize) -> Option<&Block> { + self.node_at(index).map(|node| &*node.value) + } + + /// The first / last block by reference. `O(log N)`. + #[must_use] + pub fn first(&self) -> Option<&Block> { + self.get_ref(0) + } + + #[must_use] + pub fn last(&self) -> Option<&Block> { + let len = self.len(); + if len == 0 { None } else { self.get_ref(len - 1) } + } + + /// Descend to the node holding `index` (shared by `get`/`get_ref`). `O(log N)`. + #[must_use] + fn node_at(&self, index: usize) -> Option<&Arc> { + let mut node = self.root.as_ref()?; + let mut index = index; + if index >= node.size { + return None; + } + loop { + let left_size = size(&node.left); + match index.cmp(&left_size) { + std::cmp::Ordering::Less => node = node.left.as_ref()?, + std::cmp::Ordering::Equal => return Some(node), + std::cmp::Ordering::Greater => { + index -= left_size + 1; + node = node.right.as_ref()?; + }, + } + } + } + + /// The number of paragraph blocks in the tree. `O(1)`. + #[must_use] + pub fn paragraph_count(&self) -> usize { + summary(&self.root).paragraphs + } + + /// A borrow of the `paragraph_ix`-th PARAGRAPH block (objects are skipped in + /// paragraph-rank space). The zero-copy accessor behind a tree-derived + /// `document.paragraphs` view (§act-four Slice 4). `O(log N)`. + #[must_use] + pub fn paragraph_ref(&self, paragraph_ix: usize) -> Option<&Paragraph> { + let mut node = self.root.as_ref()?; + let mut rank = paragraph_ix; + loop { + let left_paragraphs = summary(&node.left).paragraphs; + if rank < left_paragraphs { + node = node.left.as_ref()?; + continue; + } + rank -= left_paragraphs; + if let Block::Paragraph(paragraph) = &*node.value { + if rank == 0 { + return Some(paragraph); + } + rank -= 1; + } + node = node.right.as_ref()?; + } + } + + /// The block-row index (objects INCLUDED) of the `paragraph_ix`-th paragraph + /// block — the inverse of `paragraph_ref` in row space. `O(log N)`. + /// + /// §perf-heaven T7.11: the tree-native replacement for the O(blocks) + /// `block_ix_for_paragraph` linear scan. It walks the same + /// `summary().paragraphs` rank the read model already maintains (updated + /// `O(change)` by `splice`/`update_at`), accumulating the total block count of + /// fully-passed subtrees to land on the row. Returns `None` if `paragraph_ix` + /// is past the last paragraph. + #[must_use] + pub fn block_row_for_paragraph_ix(&self, paragraph_ix: usize) -> Option { + let mut node = self.root.as_ref()?; + let mut rank = paragraph_ix; + let mut row_base = 0usize; + loop { + let left_paragraphs = summary(&node.left).paragraphs; + if rank < left_paragraphs { + node = node.left.as_ref()?; + continue; + } + rank -= left_paragraphs; + row_base += size(&node.left); + if let Block::Paragraph(_) = &*node.value { + if rank == 0 { + return Some(row_base); + } + rank -= 1; + } + row_base += 1; + node = node.right.as_ref()?; + } + } + + /// The paragraph-rank of the paragraph block at block row `row`, i.e. the count + /// of paragraph blocks strictly before `row`. `Some(rank)` iff the block at + /// `row` is itself a paragraph; `None` for an object row or out of range. + /// `O(log N)`. + /// + /// §perf-heaven T7.12/T7.13: the tree-native replacement for the object-doc + /// scans in `document_offset_for_position` and `paragraph_ix_for_block_row` + /// (both formerly counted `Block::Paragraph` before `row`). + #[must_use] + pub fn paragraph_ix_for_block_row(&self, row: usize) -> Option { + let mut node = self.root.as_ref()?; + let mut index = row; + let mut paragraphs_before = 0usize; + loop { + let left_size = size(&node.left); + if index < left_size { + node = node.left.as_ref()?; + continue; + } + paragraphs_before += summary(&node.left).paragraphs; + index -= left_size; + if index == 0 { + return matches!(&*node.value, Block::Paragraph(_)).then_some(paragraphs_before); + } + if let Block::Paragraph(_) = &*node.value { + paragraphs_before += 1; + } + index -= 1; + node = node.right.as_ref()?; + } + } + + /// The count of paragraph blocks at rows strictly before `row` (any block + /// kind at `row`; `row >= len` returns the total paragraph count). `O(log N)`. + /// + /// §act-ten A10.8: backs the editor's object-selection collapse queries + /// (`paragraph_before_block`/`paragraph_after_block`), which formerly walked + /// every block per call — O(visible items x blocks) per layout pass on + /// object-heavy docs via `paragraph_range_for_item_range`. + #[must_use] + pub fn paragraphs_before_row(&self, row: usize) -> usize { + let Some(mut node) = self.root.as_ref() else { + return 0; + }; + let mut index = row; + let mut paragraphs_before = 0usize; + loop { + let left_size = size(&node.left); + if index < left_size { + match node.left.as_ref() { + Some(left) => { + node = left; + continue; + }, + None => return paragraphs_before, + } + } + paragraphs_before += summary(&node.left).paragraphs; + index -= left_size; + if index == 0 { + return paragraphs_before; + } + if let Block::Paragraph(_) = &*node.value { + paragraphs_before += 1; + } + index -= 1; + match node.right.as_ref() { + Some(right) => node = right, + None => return paragraphs_before, + } + } + } + + /// Iterate the paragraph blocks in order, by reference (skipping objects) — + /// backs `document.paragraphs.iter()`. `O(N)`. + pub fn paragraphs_iter(&self) -> impl Iterator + '_ { + self.iter_blocks().filter_map(|block| match block { + Block::Paragraph(paragraph) => Some(paragraph), + Block::Image(_) | Block::Equation(_) | Block::Table(_) => None, + }) + } + + /// The `document.text` (paragraph-rope) byte offset where paragraph + /// `paragraph_ix` starts — the tree-native replacement for + /// `ParagraphOffsetIndex::paragraph_start`. Paragraphs are joined by `\n`, so + /// `start(p) = (Σ runs_len of paragraphs before p) + newlines_before(p)`, + /// where `newlines_before(p) = min(p, paragraph_count - 1)` (every paragraph + /// but the first is preceded by one separator; the last has no trailing one). + /// `O(log N)`. + #[must_use] + pub fn paragraph_start(&self, paragraph_ix: usize) -> usize { + let total = summary(&self.root).paragraphs; + prefix_para_text_bytes(&self.root, paragraph_ix) + paragraph_ix.min(total.saturating_sub(1)) + } + + /// Mutate the block at `index` in place. Copy-on-write: `Arc::make_mut` + /// clones only the `O(log N)` path nodes still shared with a retained + /// version, so on a uniquely-owned tree this is an in-place edit with no + /// allocation. Recomputes the path summaries. `O(log N)`. + pub fn update_at(&mut self, index: usize, edit: impl FnOnce(&mut Block)) { + if index < self.len() { + update_node_at(&mut self.root, index, edit); + } + } + + /// Replace blocks `range` with `replacement`, returning a NEW tree that + /// shares every node outside the cut. `O((log N) + |replacement|)`. + #[must_use] + pub fn splice(&self, range: std::ops::Range, replacement: Vec) -> Self { + let len = self.len(); + let start = range.start.min(len); + let end = range.end.min(len).max(start); + let (left, rest) = split(&self.root, start); + let (_removed, right) = split(&rest, end - start); + let middle = build(&replacement.into_iter().map(Arc::new).collect::>()); + let root = merge(merge(left, middle), right); + Self { root } + } + + /// Iterate the blocks in order by reference (in-order traversal, zero-copy). + /// This is the workhorse behind `document.blocks.iter()`. `O(N)`. + pub fn iter_blocks(&self) -> impl Iterator + '_ { + let mut stack: Vec<&Node> = Vec::new(); + let mut current = self.root.as_deref(); + std::iter::from_fn(move || { + while let Some(node) = current { + stack.push(node); + current = node.left.as_deref(); + } + let node = stack.pop()?; + current = node.right.as_deref(); + Some(&*node.value) + }) + } + + /// Iterate the blocks in `range` by reference. `O(range.end)` (skips a prefix + /// of the in-order walk); used by the handful of slice-index read sites. The + /// range is clamped to `len()`. + pub fn range(&self, range: std::ops::Range) -> impl Iterator + '_ { + let len = self.len(); + let start = range.start.min(len); + let end = range.end.min(len).max(start); + self.iter_blocks().skip(start).take(end - start) + } + + /// Iterate the blocks in order (in-order traversal). `O(N)`. + pub fn iter(&self) -> impl Iterator> + '_ { + let mut stack = Vec::new(); + let mut current = self.root.clone(); + std::iter::from_fn(move || { + while let Some(node) = current.clone() { + stack.push(node.clone()); + current = node.left.clone(); + } + let node = stack.pop()?; + current = node.right.clone(); + Some(node.value.clone()) + }) + } + + /// Collect the blocks into a `Vec` (for equivalence checks / migration bridge). + #[must_use] + pub fn to_vec(&self) -> Vec { + self.iter().map(|block| (*block).clone()).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Paragraph, ParagraphStyle, TextRun}; + + fn paragraph(byte_len: usize) -> Block { + Block::Paragraph(Paragraph { + style: ParagraphStyle::Normal, + runs: if byte_len == 0 { + Vec::new() + } else { + vec![TextRun { + len: byte_len, + styles: crate::RunStyles::default(), + }] + }, + version: 0, + }) + } + + fn blocks(lens: &[usize]) -> Vec { + lens.iter().map(|&len| paragraph(len)).collect() + } + + /// Check the treap invariants: heap order by priority, and cached + /// size/summary equal the recomputed values. Also returns the height so the + /// caller can assert it stays `O(log N)`. + #[allow(clippy::ref_option, reason = "recursive tree walk over &Option children")] + fn assert_invariants(node: &Option>) -> usize { + match node { + None => 0, + Some(node) => { + for child in [&node.left, &node.right].into_iter().flatten() { + assert!(node.priority >= child.priority, "treap heap order violated"); + } + assert_eq!(node.size, 1 + size(&node.left) + size(&node.right), "size annotation stale"); + let recomputed = summary(&node.left) + .combine(Summary::of_block(&node.value)) + .combine(summary(&node.right)); + assert_eq!(node.summary, recomputed, "summary annotation stale"); + 1 + assert_invariants(&node.left).max(assert_invariants(&node.right)) + }, + } + } + + #[test] + fn build_and_accessors() { + let tree = BlockTree::from_blocks(blocks(&[3, 5, 2, 8])); + assert_eq!(tree.len(), 4); + assert_invariants(&tree.root); + } + + #[test] + fn borrowing_accessors_match_reference() { + let reference = blocks(&[3, 5, 0, 8, 2]); + let tree = BlockTree::from_blocks(reference.clone()); + // get_ref matches indexing the reference Vec. + for (ix, block) in reference.iter().enumerate() { + assert_eq!(tree.get_ref(ix), Some(block), "get_ref[{ix}]"); + } + assert_eq!(tree.get_ref(reference.len()), None); + assert_eq!(tree.first(), reference.first()); + assert_eq!(tree.last(), reference.last()); + // Borrowing in-order iteration matches the reference Vec exactly. + assert_eq!(tree.iter_blocks().collect::>(), reference.iter().collect::>()); + // Range iteration matches slice iteration, incl. clamping. + assert_eq!(tree.range(1..4).collect::>(), reference[1..4].iter().collect::>()); + assert_eq!(tree.range(3..100).collect::>(), reference[3..].iter().collect::>()); + assert_eq!(tree.range(2..2).count(), 0); + // Empty tree. + let empty = BlockTree::new(); + assert_eq!(empty.first(), None); + assert_eq!(empty.last(), None); + assert_eq!(empty.iter_blocks().count(), 0); + } + + /// Reference `paragraph_start`, computed the `ParagraphOffsetIndex` way over + /// the paragraph blocks only: `width(i) = runs_len(i) + separator`, cumulative. + fn reference_paragraph_start(blocks: &[Block], paragraph_ix: usize) -> usize { + let widths: Vec = blocks + .iter() + .filter_map(|block| match block { + Block::Paragraph(paragraph) => Some(paragraph.runs.iter().map(|run| run.len).sum::()), + _ => None, + }) + .collect(); + let mut start = 0; + for (i, width) in widths.iter().enumerate().take(paragraph_ix) { + let separator = usize::from(i + 1 < widths.len()); + start += width + separator; + } + start + } + + fn image_block() -> Block { + Block::Image(crate::ImageBlock { + asset_id: crate::AssetId(7), + alt_text: "x".into(), + caption: None, + sizing: crate::ImageSizing::Intrinsic, + alignment: crate::BlockAlignment::Center, + external_url: None, + version: 0, + }) + } + + #[test] + fn paragraph_view_accessors_skip_objects_and_match_reference() { + let mixed = vec![paragraph(4), image_block(), paragraph(0), paragraph(6), image_block(), paragraph(2)]; + let tree = BlockTree::from_blocks(mixed.clone()); + let reference: Vec<&Paragraph> = mixed + .iter() + .filter_map(|block| match block { + Block::Paragraph(paragraph) => Some(paragraph), + _ => None, + }) + .collect(); + assert_eq!(tree.paragraph_count(), reference.len()); + for (rank, expected) in reference.iter().enumerate() { + assert_eq!(tree.paragraph_ref(rank), Some(*expected), "paragraph_ref[{rank}]"); + } + assert_eq!(tree.paragraph_ref(reference.len()), None); + assert_eq!(tree.paragraphs_iter().collect::>(), reference); + } + + #[test] + fn paragraph_start_matches_offset_index_reference() { + // Paragraph-only doc. + let plain = blocks(&[4, 0, 7, 3, 5]); + let tree = BlockTree::from_blocks(plain.clone()); + assert_eq!(tree.paragraph_count(), 5); + for p in 0..=5 { + assert_eq!( + tree.paragraph_start(p), + reference_paragraph_start(&plain, p), + "plain paragraph_start[{p}]" + ); + } + // Object-interleaved doc: paragraph rank must skip images. + let mixed = vec![paragraph(4), image_block(), paragraph(0), paragraph(6), image_block(), paragraph(2)]; + let tree = BlockTree::from_blocks(mixed.clone()); + assert_eq!(tree.paragraph_count(), 4); + for p in 0..=4 { + assert_eq!( + tree.paragraph_start(p), + reference_paragraph_start(&mixed, p), + "mixed paragraph_start[{p}]" + ); + } + // Splices keep the paragraph monoid correct. + let spliced = tree.splice(1..3, blocks(&[9, 9])); + let spliced_ref = { + let mut v = mixed.clone(); + v.splice(1..3, blocks(&[9, 9])); + v + }; + for p in 0..=spliced.paragraph_count() { + assert_eq!( + spliced.paragraph_start(p), + reference_paragraph_start(&spliced_ref, p), + "spliced paragraph_start[{p}]" + ); + } + } + + #[test] + fn height_stays_logarithmic() { + // A large tree built + spliced must keep O(log N) height (treap balance), + // so every query/mutation is logarithmic, not linear. + let n = 20_000usize; + let mut tree = BlockTree::from_blocks(blocks(&(0..n).map(|i| 1 + i % 5).collect::>())); + let mut height = assert_invariants(&tree.root); + // log2(20000) ≈ 14.3; a treap's expected height is ~1.4·log2(n). Allow 4×. + let bound = 4 * (usize::BITS - (n as u32).leading_zeros()) as usize; + assert!(height <= bound, "initial height {height} exceeds {bound}"); + // A pile of interior splices must not degrade balance. + let mut rng = Rng::new(12345); + for _ in 0..500 { + let start = rng.below(tree.len()); + let end = (start + rng.below(5)).min(tree.len()); + tree = tree.splice(start..end, blocks(&[2, 3])); + } + height = assert_invariants(&tree.root); + let bound = 4 * (usize::BITS - (tree.len() as u32).leading_zeros()) as usize; + assert!(height <= bound, "post-splice height {height} exceeds {bound} for len {}", tree.len()); + } + + #[test] + fn cow_maps_match_reference_and_preserve_persistence() { + let base = BlockTree::from_blocks(blocks(&[1, 2, 3, 4, 5])); + // Retain a version to prove COW persistence (it must NOT change). + let retained = base.clone(); + let mut tree = base.clone(); + + // update_at: replace one block. + tree.update_at(2, |block| *block = paragraph(9)); + let mut reference = blocks(&[1, 2, 3, 4, 5]); + reference[2] = paragraph(9); + assert_eq!(tree.to_vec(), reference); + assert_invariants(&tree.root); + + // The retained version is byte-for-byte unchanged (persistence held). + assert_eq!(retained.to_vec(), blocks(&[1, 2, 3, 4, 5])); + } + + #[test] + fn splice_shares_and_matches_reference() { + let mut reference: Vec = blocks(&[1, 2, 3, 4, 5]); + let mut tree = BlockTree::from_blocks(reference.clone()); + // Replace [1,3) with two blocks. + let repl = blocks(&[9, 8]); + reference.splice(1..3, repl.clone()); + tree = tree.splice(1..3, repl); + assert_eq!(tree.to_vec(), reference); + assert_invariants(&tree.root); + } + + #[test] + fn clone_is_structural_and_undo_is_pointer_restore() { + let v0 = BlockTree::from_blocks(blocks(&[1, 2, 3, 4, 5, 6, 7, 8])); + let v1 = v0.splice(2..5, blocks(&[100])); + // v0 is untouched (persistence): undo = keep v0. + assert_eq!(v0.to_vec(), blocks(&[1, 2, 3, 4, 5, 6, 7, 8])); + assert_eq!(v1.len(), 6); + // Structural sharing: cloning v1 is O(1) — same root Arc. + let v1b = v1.clone(); + assert!(Arc::ptr_eq(v1.root.as_ref().unwrap(), v1b.root.as_ref().unwrap())); + } + + // ---- Property fuzz: the tree is behaviorally a persistent Vec ------------- + + struct Rng(u64); + impl Rng { + fn new(seed: u64) -> Self { + Self(seed.max(1)) + } + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn below(&mut self, bound: usize) -> usize { + if bound == 0 { 0 } else { (self.next() % bound as u64) as usize } + } + } + + #[test] + fn property_matches_reference_vec_under_random_splices() { + for seed in 1..200u64 { + let mut rng = Rng::new(seed); + let mut reference: Vec = blocks(&[1, 2, 3]); + let mut tree = BlockTree::from_blocks(reference.clone()); + // Retain a mid-history version to check persistence (it must not change). + let mut snapshot: Option<(Vec, BlockTree)> = None; + + for step in 0..80 { + let len = reference.len(); + let start = rng.below(len + 1); + let end = (start + rng.below(len - start + 1)).min(len); + let insert_count = rng.below(4); + let repl = blocks( + &(0..insert_count) + .map(|i| 1 + ((step + i) % 7)) + .collect::>(), + ); + + reference.splice(start..end, repl.clone()); + tree = tree.splice(start..end, repl); + + // (I1) content equals the reference Vec. + assert_eq!(tree.to_vec(), reference, "seed {seed} step {step}: content diverged"); + // (I2) balance + annotation invariants hold. + assert_invariants(&tree.root); + // (I3) the paragraph-rope monoid agrees with a cumulative scan + // (§act-ten A9.5: the block-space `bytes` monoid was removed — dead + // outside the tree, maintained on every splice; `paragraph_start` is + // the surviving load-bearing offset space). + let paragraph_count = reference + .iter() + .filter(|block| matches!(block, Block::Paragraph(_))) + .count(); + let mut cum = 0usize; + let mut paragraph_rank = 0usize; + for block in &reference { + if let Block::Paragraph(paragraph) = block { + let expected = cum + paragraph_rank.min(paragraph_count.saturating_sub(1)); + assert_eq!( + tree.paragraph_start(paragraph_rank), + expected, + "seed {seed} step {step}: paragraph_start[{paragraph_rank}]" + ); + cum += paragraph_runs_len(paragraph); + paragraph_rank += 1; + } + } + + // (I4) persistence: a snapshot taken earlier is byte-for-byte unchanged. + if let Some((snap_vec, snap_tree)) = &snapshot { + assert_eq!(snap_tree.to_vec(), *snap_vec, "seed {seed} step {step}: snapshot mutated"); + } + if step == 40 { + snapshot = Some((reference.clone(), tree.clone())); + } + } + } + } +} diff --git a/crates/gpui-flowtext/src/collaboration.rs b/crates/gpui-flowtext/src/collaboration.rs index 308b31a8..7737a3cd 100644 --- a/crates/gpui-flowtext/src/collaboration.rs +++ b/crates/gpui-flowtext/src/collaboration.rs @@ -1,56 +1,31 @@ -use std::ops::Range; +use std::sync::Arc; -use serde::{Deserialize, Serialize}; - -use super::{Block, BlockId, Document, DocumentSpan, ParagraphId, ParagraphStyle, RunStyles, new_block_id, new_paragraph_id}; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -pub struct TableCellId(pub u128); +use super::{AssetId, AssetRecord, BlockId, DocumentProjection, InputBlock, InputParagraph, ParagraphId, ParagraphStyle, TextRun}; +use rustc_hash::FxHashMap; #[derive(Clone, Debug, Default)] pub struct DocumentIdentityMap { paragraph_ids: Vec, block_ids: Vec, - table_cell_ids: Vec>>, + paragraph_index_by_id: FxHashMap, } #[hotpath::measure_all] impl DocumentIdentityMap { #[must_use] - pub fn new(document: &Document) -> Self { + pub fn new(document: &DocumentProjection) -> Self { let mut this = Self::default(); this.reconcile(document); this } - pub fn reconcile(&mut self, document: &Document) { + pub fn reconcile(&mut self, document: &DocumentProjection) { self.paragraph_ids.clone_from(&document.ids.paragraph_ids); - self.block_ids.clone_from(&document.ids.block_ids); - self - .table_cell_ids - .resize_with(document.blocks.len(), Vec::new); - self.table_cell_ids.truncate(document.blocks.len()); - for (block_ix, block) in document.blocks.iter().enumerate() { - let Block::Table(table) = block else { - self.table_cell_ids[block_ix].clear(); - continue; - }; - let rows = &mut self.table_cell_ids[block_ix]; - rows.resize_with(table.rows.len(), Vec::new); - rows.truncate(table.rows.len()); - for (row_ix, row) in table.rows.iter().enumerate() { - resize_ids(&mut rows[row_ix], row.cells.len(), TableCellId); - } + self.paragraph_index_by_id.clear(); + for (ix, id) in self.paragraph_ids.iter().enumerate() { + self.paragraph_index_by_id.insert(*id, ix); } - } - - pub fn insert_split_paragraph(&mut self, paragraph_ix: usize, block_ix: usize) { - self - .paragraph_ids - .insert((paragraph_ix + 1).min(self.paragraph_ids.len()), new_paragraph_id()); - let block_insert_ix = (block_ix + 1).min(self.block_ids.len()); - self.block_ids.insert(block_insert_ix, new_block_id()); - self.table_cell_ids.insert(block_insert_ix, Vec::new()); + self.block_ids.clone_from(&document.ids.block_ids); } #[must_use] @@ -63,243 +38,142 @@ impl DocumentIdentityMap { self.block_ids.get(block_ix).copied() } - #[must_use] - pub fn table_cell_id(&self, block_ix: usize, row_ix: usize, cell_ix: usize) -> Option { - self - .table_cell_ids - .get(block_ix)? - .get(row_ix)? - .get(cell_ix) - .copied() - } - #[must_use] pub fn paragraph_index(&self, id: ParagraphId) -> Option { - self - .paragraph_ids - .iter() - .position(|candidate| *candidate == id) + self.paragraph_index_by_id.get(&id).copied() } } -#[hotpath::measure] -fn resize_ids(ids: &mut Vec, len: usize, wrap: impl Fn(u128) -> T) -where - T: std::marker::Copy, -{ - while ids.len() < len { - ids.push(wrap(uuid::Uuid::new_v4().as_u128())); - } - ids.truncate(len); +// Loro-first (spec invariant 6): the raw projection-space command surface is +// DELETED — not deprecated, not quarantined. Local editing enters exclusively +// as typed intents (`crate::local_intents`) resolved against live Loro state; +// wrong-position operations are unrepresentable. The CI guard +// (tools/check_raw_authority.sh) rejects any reintroduction by name. + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProjectionTextDelta { + Retain(usize), + Insert(usize), + Delete(usize), } #[derive(Clone, Debug)] -pub enum CanonicalOperation { - InsertText { - paragraph: ParagraphId, - byte: usize, - text: String, - styles: RunStyles, - }, - DeleteRange { - start_paragraph: ParagraphId, - start_byte: usize, - end_paragraph: ParagraphId, - end_byte: usize, - }, - SplitParagraph { - paragraph: ParagraphId, - byte: usize, - new_paragraph: ParagraphId, - }, - JoinParagraphs { - first: ParagraphId, - second: ParagraphId, - }, - SetParagraphStyle { - paragraph: ParagraphId, - style: ParagraphStyle, - }, - SetRunStyles { - paragraph: ParagraphId, - range: Range, - styles: RunStyles, - }, - InsertBlock { - block: BlockId, - block_ix: usize, - }, - DeleteBlock { - block: BlockId, - }, - MoveBlock { - block: BlockId, - new_block_ix: usize, - }, - ReplaceParagraphSpan { - start_paragraph: Option, - before: DocumentSpan, - after: DocumentSpan, - }, - ReplaceBlock { - block: Option, - }, - ReplaceDocument, +pub struct ProjectionStructuralBlock { + pub block_id: BlockId, + pub paragraph_id: Option, + pub block: InputBlock, } -#[derive(Clone, Debug, Serialize, Deserialize)] -enum WireCanonicalOperation { - InsertText { - paragraph: ParagraphId, - byte: usize, - text: String, - styles: RunStyles, - }, - DeleteRange { - start_paragraph: ParagraphId, - start_byte: usize, - end_paragraph: ParagraphId, - end_byte: usize, - }, - SplitParagraph { - paragraph: ParagraphId, - byte: usize, - new_paragraph: ParagraphId, - }, - JoinParagraphs { - first: ParagraphId, - second: ParagraphId, - }, - SetParagraphStyle { - paragraph: ParagraphId, - style: ParagraphStyle, - }, - SetRunStyles { - paragraph: ParagraphId, - range: Range, - styles: RunStyles, - }, +#[derive(Clone, Debug)] +pub struct ProjectionPatchBatch { + /// Unique id for diagnostics and duplicate-delivery detection. It is not part + /// of document semantics; the frontier pair is the authoritative ordering. + pub transaction_id: u128, + /// Frontier of the materialized document this delta must be applied to. + pub base_frontier: Vec, + /// Frontier represented after every patch has been applied successfully. + pub new_frontier: Vec, + // §perf-heaven T8.19: `Arc<[…]>` so the recorded-inverse undo/redo can SHARE + // its (possibly huge, mass-op) patch set with the emitted event instead of + // cloning it — the `patches.clone()` in `try_fast_step` becomes an O(1) refcount + // bump. Not serialized (`ProjectionPatchBatch` is `Clone, Debug` only), so this + // is purely an in-process representation change; every consumer reads by `&`. + pub patches: Arc<[ProjectionPatch]>, } -impl WireCanonicalOperation { - fn from_canonical(operation: &CanonicalOperation) -> Option { - match operation { - CanonicalOperation::InsertText { - paragraph, - byte, - text, - styles, - } => Some(Self::InsertText { - paragraph: *paragraph, - byte: *byte, - text: text.clone(), - styles: *styles, - }), - CanonicalOperation::DeleteRange { - start_paragraph, - start_byte, - end_paragraph, - end_byte, - } => Some(Self::DeleteRange { - start_paragraph: *start_paragraph, - start_byte: *start_byte, - end_paragraph: *end_paragraph, - end_byte: *end_byte, - }), - CanonicalOperation::SplitParagraph { - paragraph, - byte, - new_paragraph, - } => Some(Self::SplitParagraph { - paragraph: *paragraph, - byte: *byte, - new_paragraph: *new_paragraph, - }), - CanonicalOperation::JoinParagraphs { first, second } => Some(Self::JoinParagraphs { - first: *first, - second: *second, - }), - CanonicalOperation::SetParagraphStyle { paragraph, style } => Some(Self::SetParagraphStyle { - paragraph: *paragraph, - style: *style, - }), - CanonicalOperation::SetRunStyles { paragraph, range, styles } => Some(Self::SetRunStyles { - paragraph: *paragraph, - range: range.clone(), - styles: *styles, - }), - CanonicalOperation::InsertBlock { .. } - | CanonicalOperation::DeleteBlock { .. } - | CanonicalOperation::MoveBlock { .. } - | CanonicalOperation::ReplaceParagraphSpan { .. } - | CanonicalOperation::ReplaceBlock { .. } - | CanonicalOperation::ReplaceDocument => None, - } +impl ProjectionPatchBatch { + #[must_use] + pub fn is_empty(&self) -> bool { + self.patches.is_empty() } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ProjectionApplyError { + StaleFrontier { expected: Vec, actual: Vec }, + MissingBlock { block_id: BlockId, row_hint: usize }, + WrongBlockKind { block_id: BlockId, expected: &'static str }, + MissingParagraph { paragraph_id: ParagraphId, block_id: BlockId }, + DuplicateBlockId(BlockId), + DuplicateParagraphId(ParagraphId), + InvalidAnchor(BlockId), + InvalidStructuralPatch(&'static str), + UnexpectedTransaction { expected: Option, actual: u128 }, +} - fn into_canonical(self) -> CanonicalOperation { +impl std::fmt::Display for ProjectionApplyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::InsertText { - paragraph, - byte, - text, - styles, - } => CanonicalOperation::InsertText { - paragraph, - byte, - text, - styles, + Self::StaleFrontier { expected, actual } => write!( + f, + "materialized document frontier mismatch (expected {} bytes, actual {} bytes)", + expected.len(), + actual.len() + ), + Self::MissingBlock { block_id, row_hint } => write!(f, "projection patch target block {} is missing (row hint {row_hint})", block_id.0), + Self::WrongBlockKind { block_id, expected } => write!(f, "projection patch target block {} is not {expected}", block_id.0), + Self::MissingParagraph { paragraph_id, block_id } => { + write!(f, "projection paragraph {} is not attached to block {}", paragraph_id.0, block_id.0) }, - Self::DeleteRange { - start_paragraph, - start_byte, - end_paragraph, - end_byte, - } => CanonicalOperation::DeleteRange { - start_paragraph, - start_byte, - end_paragraph, - end_byte, + Self::DuplicateBlockId(id) => write!(f, "projection patch would create duplicate block id {}", id.0), + Self::DuplicateParagraphId(id) => write!(f, "projection patch would create duplicate paragraph id {}", id.0), + Self::InvalidAnchor(id) => write!(f, "projection insertion anchor block {} is missing", id.0), + Self::InvalidStructuralPatch(reason) => f.write_str(reason), + Self::UnexpectedTransaction { expected, actual } => { + write!(f, "runtime transaction acknowledgement mismatch (expected {expected:?}, actual {actual})") }, - Self::SplitParagraph { - paragraph, - byte, - new_paragraph, - } => CanonicalOperation::SplitParagraph { - paragraph, - byte, - new_paragraph, - }, - Self::JoinParagraphs { first, second } => CanonicalOperation::JoinParagraphs { first, second }, - Self::SetParagraphStyle { paragraph, style } => CanonicalOperation::SetParagraphStyle { paragraph, style }, - Self::SetRunStyles { paragraph, range, styles } => CanonicalOperation::SetRunStyles { paragraph, range, styles }, } } } -pub fn encode_canonical_operations(operations: &[CanonicalOperation]) -> Option> { - let wire_operations: Vec<_> = operations - .iter() - .filter_map(WireCanonicalOperation::from_canonical) - .collect(); - if wire_operations.is_empty() && !operations.is_empty() { - return None; - } - postcard::to_stdvec(&wire_operations).ok() -} - -pub fn decode_canonical_operations(bytes: &[u8]) -> Option> { - postcard::from_bytes::>(bytes) - .ok() - .map(|operations| { - operations - .into_iter() - .map(WireCanonicalOperation::into_canonical) - .collect() - }) -} +impl std::error::Error for ProjectionApplyError {} -#[derive(Clone, Debug, Default)] -pub struct CollaborationEdit { - pub operations: Vec, +#[derive(Clone, Debug)] +pub enum ProjectionPatch { + ParagraphText { + block_id: BlockId, + paragraph_id: ParagraphId, + row_hint: usize, + new: InputParagraph, + delta_utf8: Vec, + }, + ParagraphStyle { + block_id: BlockId, + paragraph_id: ParagraphId, + row_hint: usize, + style: ParagraphStyle, + }, + ParagraphRuns { + block_id: BlockId, + paragraph_id: ParagraphId, + row_hint: usize, + runs: Vec, + }, + ReplaceObjectBlock { + block_id: BlockId, + row_hint: usize, + block: ProjectionStructuralBlock, + }, + InsertBlocks { + /// Insert immediately before this stable block id, or append when `None`. + before: Option, + row_hint: usize, + blocks: Vec, + }, + DeleteBlocks { + block_ids: Vec, + row_hint: usize, + }, + MoveBlock { + block_id: BlockId, + /// Place the moved block immediately before this id, or at the end when + /// `None`. The anchor is interpreted after removing `block_id`. + before: Option, + from_hint: usize, + to_hint: usize, + }, + AssetArrived { + id: AssetId, + record: AssetRecord, + }, } diff --git a/crates/gpui-flowtext/src/demo.rs b/crates/gpui-flowtext/src/demo.rs index 730da68e..9ac624ef 100644 --- a/crates/gpui-flowtext/src/demo.rs +++ b/crates/gpui-flowtext/src/demo.rs @@ -3,14 +3,14 @@ use std::sync::Arc; use crop::Rope; use super::{ - AssetStore, Document, DocumentParagraphInput, DocumentTheme, InputParagraph, InputRun, Paragraph, ParagraphOffsetIndex, ParagraphStyle, - RunStyle, RunStyles, TextRun, document_ids_for_shape, merge_adjacent_runs, paragraph_blocks_from_paragraphs, rebuild_document_sections, + AssetStore, DocumentParagraphInput, DocumentProjection, DocumentTheme, InputParagraph, InputRun, Paragraph, ParagraphStyle, RunStyle, + RunStyles, TextRun, document_ids_for_shape, merge_adjacent_runs, paragraph_blocks_from_paragraphs, rebuild_document_sections, reconcile_document_ids, }; #[hotpath::measure] #[must_use] -pub fn document_from_input(theme: DocumentTheme, paragraphs: Vec) -> Document { +pub fn document_from_input(theme: DocumentTheme, paragraphs: Vec) -> DocumentProjection { let text_capacity = paragraphs .iter() .map(|paragraph| { @@ -28,17 +28,14 @@ pub fn document_from_input(theme: DocumentTheme, paragraphs: Vec if ix > 0 { text.push('\n'); } - let start = text.len(); let mut runs = Vec::with_capacity(paragraph.runs.len()); for run in paragraph.runs { let len = run.text.len(); text.push_str(&run.text); runs.push(TextRun { len, styles: run.styles }); } - let end = text.len(); stored_paragraphs.push(Paragraph { style: paragraph.style, - byte_range: start..end, runs: merge_adjacent_runs(runs), version: 0, }); @@ -48,7 +45,7 @@ pub fn document_from_input(theme: DocumentTheme, paragraphs: Vec #[hotpath::measure] #[must_use] -pub fn document_from_paragraphs(theme: DocumentTheme, paragraphs: Vec) -> Document { +pub fn document_from_paragraphs(theme: DocumentTheme, paragraphs: Vec) -> DocumentProjection { let text_capacity = paragraphs .iter() .map(|paragraph| { @@ -66,17 +63,14 @@ pub fn document_from_paragraphs(theme: DocumentTheme, paragraphs: Vec 0 { text.push('\n'); } - let start = text.len(); let mut runs = Vec::with_capacity(paragraph.runs.len()); for run in paragraph.runs { let len = run.text.len(); text.push_str(&run.text); runs.push(TextRun { len, styles: run.styles }); } - let end = text.len(); stored_paragraphs.push(Paragraph { style: paragraph.style, - byte_range: start..end, runs: merge_adjacent_runs(runs), version: 0, }); @@ -85,27 +79,26 @@ pub fn document_from_paragraphs(theme: DocumentTheme, paragraphs: Vec) -> Document { +fn document_from_stored_paragraphs(theme: DocumentTheme, text: String, mut stored_paragraphs: Vec) -> DocumentProjection { if stored_paragraphs.is_empty() { stored_paragraphs.push(Paragraph { style: ParagraphStyle::Normal, - byte_range: 0..0, runs: Vec::new(), version: 0, }); } - let offset_index = ParagraphOffsetIndex::new(&stored_paragraphs); let blocks = paragraph_blocks_from_paragraphs(&stored_paragraphs); let paragraph_count = stored_paragraphs.len(); let block_count = blocks.len(); - let mut document = Document { + let mut document = DocumentProjection { + frontier: Vec::new(), text: Rope::from(text), - paragraphs: Arc::new(stored_paragraphs), + paragraphs: crate::ParagraphSeq::from_vec(stored_paragraphs), ids: document_ids_for_shape(paragraph_count, block_count), - blocks: Arc::new(blocks), + blocks: crate::BlockSeq::from_vec(blocks), assets: AssetStore::default(), sections: Arc::new(Vec::new()), - offset_index, + outline: Arc::new(Vec::new()), theme, }; reconcile_document_ids(&mut document); @@ -115,13 +108,13 @@ fn document_from_stored_paragraphs(theme: DocumentTheme, text: String, mut store #[hotpath::measure] #[must_use] -pub fn blank_document() -> Document { +pub fn blank_document() -> DocumentProjection { document_from_input(DocumentTheme::default(), Vec::new()) } #[hotpath::measure] #[must_use] -pub fn demo_document() -> Document { +pub fn demo_document() -> DocumentProjection { let mut paragraphs = vec![ InputParagraph { style: ParagraphStyle::Custom(0), diff --git a/crates/gpui-flowtext/src/document/block_seq.rs b/crates/gpui-flowtext/src/document/block_seq.rs new file mode 100644 index 00000000..870edee4 --- /dev/null +++ b/crates/gpui-flowtext/src/document/block_seq.rs @@ -0,0 +1,386 @@ +// §act-four M3 migration — `BlockSeq`: the tree-backed block sequence that +// replaces `Arc>` on `DocumentProjection`. +// +// Read sites keep their shape — `document.blocks[i]`, `.get(i)`, `.iter()`, +// `.len()`, `.first()` all resolve through the persistent `BlockTree` and hand +// back `&Block` (zero copy). Mutation sites go through the `O(log N)` `splice` +// / `set` / `insert` / `remove` fast paths, or — for the few complex sites that +// run several `Vec` ops at once — the `BlockSeq::make_mut` guard (materialize → +// mutate → rebuild). Cloning is `O(1)` (the tree root is one `Arc`), which is +// exactly what lets a version's blocks be a pointer to restore on undo instead +// of a rebuilt `Vec`. + +use crate::block_tree::BlockTree; + +/// A persistent, tree-backed block sequence. Drop-in for the old +/// `Arc>` field: `O(1)` clone, `O(log N)` random access + byte +/// queries, structural sharing across document versions. +#[derive(Clone, Debug, Default)] +pub struct BlockSeq { + tree: BlockTree, +} + +impl BlockSeq { + #[must_use] + pub fn from_vec(blocks: Vec) -> Self { + Self { + tree: BlockTree::from_blocks(blocks), + } + } + + #[must_use] + pub fn from_tree(tree: BlockTree) -> Self { + Self { tree } + } + + /// The backing tree — for the version graph (store a root) and byte queries. + #[must_use] + pub fn tree(&self) -> &BlockTree { + &self.tree + } + + #[must_use] + pub fn len(&self) -> usize { + self.tree.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.tree.is_empty() + } + + /// The block at `index`, by reference. `O(log N)`. + #[must_use] + pub fn get(&self, index: usize) -> Option<&Block> { + self.tree.get_ref(index) + } + + #[must_use] + pub fn first(&self) -> Option<&Block> { + self.tree.first() + } + + #[must_use] + pub fn last(&self) -> Option<&Block> { + self.tree.last() + } + + /// In-order iteration by reference — backs `document.blocks.iter()`. `O(N)`. + pub fn iter(&self) -> impl Iterator + '_ { + self.tree.iter_blocks() + } + + /// Iterate `range` by reference — backs the `blocks[a..b]` read sites. The + /// range is clamped to `len()`. + pub fn range(&self, range: std::ops::Range) -> impl Iterator + '_ { + self.tree.range(range) + } + + /// Bounds-checked range iteration mirroring slice `get(a..b)`: `None` if the + /// range is inverted or past the end (preserves the `?`-bail read sites). + pub fn get_range(&self, range: std::ops::Range) -> Option + '_> { + if range.start > range.end || range.end > self.len() { + return None; + } + Some(self.tree.range(range)) + } + + /// Materialize the whole sequence into an owned `Vec` (equivalence checks, + /// serialization, and the few sites that genuinely need a contiguous copy). + #[must_use] + pub fn to_vec(&self) -> Vec { + self.tree.to_vec() + } + + + + /// The number of paragraph blocks. `O(1)`. + #[must_use] + pub fn paragraph_count(&self) -> usize { + self.tree.paragraph_count() + } + + /// The `paragraph_ix`-th paragraph block by reference (objects skipped) — the + /// zero-copy accessor behind a tree-derived `document.paragraphs` view + /// (§act-four Slice 4). `O(log N)`. + #[must_use] + pub fn paragraph_ref(&self, paragraph_ix: usize) -> Option<&Paragraph> { + self.tree.paragraph_ref(paragraph_ix) + } + + /// Iterate the paragraph blocks in order, by reference. `O(N)`. + pub fn paragraphs_iter(&self) -> impl Iterator + '_ { + self.tree.paragraphs_iter() + } + + /// The `document.text` byte offset where paragraph `paragraph_ix` starts — + /// the tree-native replacement for `ParagraphOffsetIndex::paragraph_start`. + /// `O(log N)`. + #[must_use] + pub fn paragraph_start(&self, paragraph_ix: usize) -> usize { + self.tree.paragraph_start(paragraph_ix) + } + + /// The block-row index of the `paragraph_ix`-th paragraph (objects included in + /// row space). `O(log N)` — the tree-native replacement for the O(blocks) + /// `block_ix_for_paragraph` scan. §perf-heaven T7.11. + #[must_use] + pub fn block_row_for_paragraph_ix(&self, paragraph_ix: usize) -> Option { + self.tree.block_row_for_paragraph_ix(paragraph_ix) + } + + /// The paragraph-rank of the paragraph at block row `row` (count of paragraphs + /// before it); `None` if `row` is an object row or out of range. `O(log N)` — + /// the tree-native replacement for the object-doc paragraph-count scans. + /// §perf-heaven T7.12/T7.13. + #[must_use] + pub fn paragraph_ix_for_block_row(&self, row: usize) -> Option { + self.tree.paragraph_ix_for_block_row(row) + } + + /// The count of paragraphs at rows strictly before `row` (any block kind at + /// `row`). `O(log N)`. §act-ten A10.8. + #[must_use] + pub fn paragraphs_before_row(&self, row: usize) -> usize { + self.tree.paragraphs_before_row(row) + } + + // -- `O(log N)` fast-path mutations (path-copy, share the rest) ----------- + + /// Replace `range` with `replacement`, sharing every untouched node. + pub fn splice(&mut self, range: std::ops::Range, replacement: Vec) { + self.tree = self.tree.splice(range, replacement); + } + + /// Replace the single block at `index` (no-op if out of range). `O(log N)`. + pub fn set(&mut self, index: usize, block: Block) { + if index < self.len() { + self.tree = self.tree.splice(index..index + 1, vec![block]); + } + } + + /// Insert `block` before `index` (clamped to `len`). `O(log N)`. + pub fn insert(&mut self, index: usize, block: Block) { + let at = index.min(self.len()); + self.tree = self.tree.splice(at..at, vec![block]); + } + + /// Remove the block at `index` (no-op if out of range). `O(log N)`. + pub fn remove(&mut self, index: usize) { + if index < self.len() { + self.tree = self.tree.splice(index..index + 1, Vec::new()); + } + } + + /// Append `block`. `O(log N)`. + pub fn push(&mut self, block: Block) { + let len = self.len(); + self.tree = self.tree.splice(len..len, vec![block]); + } + + /// Mutate one block in place (copy-on-write). `O(log N)`, no allocation on a + /// uniquely-owned tree. + pub fn update_at(&mut self, index: usize, edit: impl FnOnce(&mut Block)) { + self.tree.update_at(index, edit); + } + + + /// Mutable access as a `Vec` for the complex sites that run several ops at + /// once: materialize now, rebuild the tree on the guard's `Drop`. `O(N)` — + /// hot single-block sites should prefer `set`/`insert`/`remove`/`splice`. + #[must_use] + pub fn make_mut(&mut self) -> BlockSeqMut<'_> { + let vec = self.tree.to_vec(); + BlockSeqMut { seq: self, vec } + } +} + +impl std::ops::Index for BlockSeq { + type Output = Block; + fn index(&self, index: usize) -> &Block { + self.get(index).expect("block index out of bounds") + } +} + +impl From> for BlockSeq { + fn from(blocks: Vec) -> Self { + Self::from_vec(blocks) + } +} + +impl FromIterator for BlockSeq { + fn from_iter>(iter: I) -> Self { + Self::from_vec(iter.into_iter().collect()) + } +} + +/// A materialize→mutate→rebuild guard. Derefs to the underlying `Vec` +/// so existing `Arc::make_mut(&mut …blocks)` call sites keep their body +/// verbatim; on `Drop` the mutated `Vec` is rebuilt into the persistent tree. +pub struct BlockSeqMut<'a> { + seq: &'a mut BlockSeq, + vec: Vec, +} + +impl std::ops::Deref for BlockSeqMut<'_> { + type Target = Vec; + fn deref(&self) -> &Vec { + &self.vec + } +} + +impl std::ops::DerefMut for BlockSeqMut<'_> { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.vec + } +} + +impl Drop for BlockSeqMut<'_> { + fn drop(&mut self) { + self.seq.tree = BlockTree::from_blocks(std::mem::take(&mut self.vec)); + } +} + +#[cfg(test)] +mod block_seq_tests { + use super::*; + use crate::{ParagraphStyle, TextRun}; + + fn para(byte_len: usize) -> Block { + Block::Paragraph(Paragraph { + style: ParagraphStyle::Normal, + runs: if byte_len == 0 { + Vec::new() + } else { + vec![TextRun { + len: byte_len, + styles: crate::RunStyles::default(), + }] + }, + version: 0, + }) + } + + fn seq_and_ref(lens: &[usize]) -> (BlockSeq, Vec) { + let reference: Vec = lens.iter().map(|&l| para(l)).collect(); + (BlockSeq::from_vec(reference.clone()), reference) + } + + #[test] + fn read_api_matches_reference_vec() { + let (seq, reference) = seq_and_ref(&[3, 5, 0, 8, 2]); + assert_eq!(seq.len(), reference.len()); + assert!(!seq.is_empty()); + for ix in 0..reference.len() { + assert_eq!(seq.get(ix), Some(&reference[ix])); + assert_eq!(&seq[ix], &reference[ix]); + } + assert_eq!(seq.get(reference.len()), None); + assert_eq!(seq.first(), reference.first()); + assert_eq!(seq.last(), reference.last()); + assert_eq!(seq.iter().collect::>(), reference.iter().collect::>()); + assert_eq!(seq.range(1..4).collect::>(), reference[1..4].iter().collect::>()); + assert_eq!(seq.to_vec(), reference); + } + + #[test] + fn fast_path_mutations_match_vec_semantics() { + let (mut seq, mut reference) = seq_and_ref(&[1, 2, 3, 4]); + seq.set(1, para(9)); + reference[1] = para(9); + assert_eq!(seq.to_vec(), reference); + seq.insert(2, para(7)); + reference.insert(2, para(7)); + assert_eq!(seq.to_vec(), reference); + seq.remove(0); + reference.remove(0); + assert_eq!(seq.to_vec(), reference); + seq.push(para(5)); + reference.push(para(5)); + assert_eq!(seq.to_vec(), reference); + seq.splice(1..3, vec![para(6), para(6), para(6)]); + reference.splice(1..3, vec![para(6), para(6), para(6)]); + assert_eq!(seq.to_vec(), reference); + } + + #[test] + fn make_mut_guard_rebuilds_on_drop() { + let (mut seq, mut reference) = seq_and_ref(&[1, 2, 3]); + let mut guard = seq.make_mut(); + guard.insert(1, para(8)); + guard[0] = para(4); + guard.remove(3); + drop(guard); + reference.insert(1, para(8)); + reference[0] = para(4); + reference.remove(3); + assert_eq!(seq.to_vec(), reference); + } + + fn image() -> Block { + Block::Image(crate::ImageBlock { + asset_id: crate::AssetId(1), + alt_text: "x".into(), + caption: None, + sizing: crate::ImageSizing::Intrinsic, + alignment: crate::BlockAlignment::Left, + external_url: None, + version: 0, + }) + } + + /// §perf-heaven T7.11-T7.13 NET: the O(log N) tree queries must equal the + /// O(blocks) linear scans they replace, for EVERY paragraph rank and EVERY + /// block row, on an object-bearing sequence (where the aligned fast path + /// misses). If a tree walk ever disagrees with the scan, this trips. + #[test] + fn paragraph_row_queries_match_linear_scan() { + // Interleave paragraphs and objects so blocks.len() != paragraph_count. + let blocks = vec![image(), para(1), para(2), image(), image(), para(3), para(4), image()]; + let seq = BlockSeq::from_vec(blocks.clone()); + + // Reference: block row of the k-th paragraph, by linear scan. + let scan_block_row = |k: usize| -> Option { + let mut seen = 0; + for (row, b) in blocks.iter().enumerate() { + if matches!(b, Block::Paragraph(_)) { + if seen == k { + return Some(row); + } + seen += 1; + } + } + None + }; + // Reference: paragraph rank at block row, by linear scan (None for objects). + let scan_para_ix = |row: usize| -> Option { + matches!(blocks.get(row), Some(Block::Paragraph(_))) + .then(|| blocks.iter().take(row).filter(|b| matches!(b, Block::Paragraph(_))).count()) + }; + + let paragraph_count = blocks.iter().filter(|b| matches!(b, Block::Paragraph(_))).count(); + assert_eq!(seq.paragraph_count(), paragraph_count); + for k in 0..=paragraph_count + 1 { + assert_eq!(seq.block_row_for_paragraph_ix(k), scan_block_row(k), "block_row_for_paragraph_ix({k})"); + } + for row in 0..=blocks.len() + 1 { + assert_eq!(seq.paragraph_ix_for_block_row(row), scan_para_ix(row), "paragraph_ix_for_block_row({row})"); + } + + // The queries are also consistent inverses on paragraph rows. + for k in 0..paragraph_count { + let row = seq.block_row_for_paragraph_ix(k).unwrap(); + assert_eq!(seq.paragraph_ix_for_block_row(row), Some(k)); + } + } + + #[test] + fn clone_shares_and_is_persistent() { + let (seq, _) = seq_and_ref(&[1, 2, 3, 4, 5]); + let mut edited = seq.clone(); + edited.set(2, para(99)); + // The clone diverges; the original is untouched (persistence → undo target). + assert_eq!(seq.to_vec(), seq_and_ref(&[1, 2, 3, 4, 5]).1); + assert_ne!(edited.to_vec(), seq.to_vec()); + } +} diff --git a/crates/gpui-flowtext/src/document/blocks.rs b/crates/gpui-flowtext/src/document/blocks.rs index de88397a..ca8fa382 100644 --- a/crates/gpui-flowtext/src/document/blocks.rs +++ b/crates/gpui-flowtext/src/document/blocks.rs @@ -24,6 +24,26 @@ pub struct AssetRecord { pub bytes: Arc>, } +pub const IMAGE_LOADING_PLACEHOLDER_WIDTH_PX: f32 = 240.0; +pub const IMAGE_LOADING_PLACEHOLDER_HEIGHT_PX: f32 = 160.0; + +impl AssetRecord { + #[must_use] + pub fn stable_content_hash(bytes: &[u8]) -> u64 { + let digest = blake3::hash(bytes); + u64::from_le_bytes( + digest.as_bytes()[..8] + .try_into() + .expect("BLAKE3 digest always contains at least eight bytes"), + ) + } + + #[must_use] + pub fn is_loading_placeholder(&self) -> bool { + self.bytes.is_empty() + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct ImageBlock { pub asset_id: AssetId, @@ -31,6 +51,12 @@ pub struct ImageBlock { pub caption: Option, pub sizing: ImageSizing, pub alignment: BlockAlignment, + /// §A11.9: a genuinely-LINKED image's external target URL (`a:blip r:link` / + /// VML `r:href` resolving to a `TargetMode="External"` relationship). Such an + /// image has no embedded media part — `asset_id` is derived from the URL + /// bytes and no [`AssetRecord`] carries bytes for it. `None` for embedded + /// images (the overwhelmingly common case). + pub external_url: Option, pub version: u64, } @@ -73,18 +99,32 @@ pub enum EquationDisplay { #[derive(Clone, Debug, PartialEq, Eq)] pub struct TableBlock { pub rows: Vec, - pub column_widths: Vec, + /// Ordered columns, each with its durable [`ColumnId`] and width (§P2b). + /// Replaces the id-less `column_widths` list; read a width as + /// `columns[i].width`. + pub columns: Vec, pub style: TableStyle, pub version: u64, } +/// A table column carrying its durable identity and rendered width (§P2b). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TableColumn { + pub id: ColumnId, + pub width: TableColumnWidth, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct TableRow { + pub id: RowId, pub cells: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] pub struct TableCell { + pub id: CellId, + pub row_id: RowId, + pub column_id: ColumnId, pub blocks: Vec, pub row_span: u16, pub col_span: u16, diff --git a/crates/gpui-flowtext/src/document/core.rs b/crates/gpui-flowtext/src/document/core.rs index 29034961..b64ec17a 100644 --- a/crates/gpui-flowtext/src/document/core.rs +++ b/crates/gpui-flowtext/src/document/core.rs @@ -1,13 +1,11 @@ -use std::{ops::Range, sync::Arc}; +use std::{cell::Cell, ops::Range, sync::Arc}; use crop::Rope; use gpui::{Hsla, Pixels, SharedString, black, px, rgb}; use rustc_hash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; -// `paragraph_widths` and `paragraph_width` are free helpers that still live in -// the parent module. `ParagraphOffsetIndex`'s methods invoke them. -use super::{paragraph_text_len, paragraph_width, paragraph_widths}; +use super::paragraph_text_len; pub const SOFT_LINE_BREAK: char = '\u{2028}'; pub const SOFT_LINE_BREAK_STR: &str = "\u{2028}"; @@ -34,26 +32,34 @@ pub struct RichClipboardFragment { pub assets: Vec, } -// -- Document and paragraphs --------------------------------------------- +// -- DocumentProjection and paragraphs --------------------------------------------- #[derive(Clone, Debug)] -pub struct Document { +pub struct DocumentProjection { + /// Encoded canonical frontier this disposable projection was built from. + /// Empty for standalone projections that are not backed by a CRDT runtime. + pub frontier: Vec, pub text: Rope, - pub paragraphs: Arc>, - pub blocks: Arc>, + pub paragraphs: ParagraphSeq, + pub blocks: BlockSeq, pub assets: AssetStore, pub ids: DocumentIds, + /// Canonical page/document sections projected from the CRDT. pub sections: Arc>, - // Auxiliary Fenwick-tree index over per-paragraph byte widths. Kept in sync - // with `paragraphs` by the edit helpers in `edit_ops`. Not part of the - // public API. - pub offset_index: ParagraphOffsetIndex, + /// Disposable heading hierarchy derived from paragraph styles. + pub outline: Arc>, pub theme: DocumentTheme, } -#[hotpath::measure] -pub fn paragraphs_mut(document: &mut Document) -> &mut Vec { - Arc::make_mut(&mut document.paragraphs) +// §act-four M3 Slice 4: `document.paragraphs` is a persistent `ParagraphSeq`. +// `paragraphs_mut` hands out `&mut ParagraphSeq` — NOT a `Drop` guard — so its +// borrow of `document` is released by NLL at last use, letting the mutation +// sites interleave a following block edit exactly as they did with the old +// `&mut Vec`. Mutations go through the sequence's copy-on-write methods +// (`set`/`get_mut`/`insert`/`remove`/`iter_mut`/`splice`), applied in place. +#[inline] +pub fn paragraphs_mut(document: &mut DocumentProjection) -> &mut ParagraphSeq { + &mut document.paragraphs } #[hotpath::measure] @@ -61,9 +67,74 @@ pub fn paragraph_blocks_from_paragraphs(paragraphs: &[Paragraph]) -> Vec paragraphs.iter().cloned().map(Block::Paragraph).collect() } +thread_local! { + static DOCUMENT_SECTION_REBUILD_DEFERRAL_DEPTH: Cell = const { Cell::new(0) }; + static DOCUMENT_SECTION_REBUILD_DEFERRED_DIRTY: Cell = const { Cell::new(false) }; +} + +#[derive(Debug)] +pub struct DocumentSectionRebuildDeferral; + +impl Drop for DocumentSectionRebuildDeferral { + fn drop(&mut self) { + DOCUMENT_SECTION_REBUILD_DEFERRAL_DEPTH.with(|depth| { + depth.set(depth.get().saturating_sub(1)); + }); + } +} + +#[hotpath::measure] +#[must_use] +pub fn defer_document_section_rebuilds() -> DocumentSectionRebuildDeferral { + DOCUMENT_SECTION_REBUILD_DEFERRAL_DEPTH.with(|depth| { + if depth.get() == 0 { + DOCUMENT_SECTION_REBUILD_DEFERRED_DIRTY.with(|dirty| dirty.set(false)); + } + depth.set(depth.get().saturating_add(1)); + }); + DocumentSectionRebuildDeferral +} + +#[hotpath::measure] +#[must_use] +pub fn document_section_rebuilds_deferred() -> bool { + DOCUMENT_SECTION_REBUILD_DEFERRAL_DEPTH.with(|depth| depth.get() > 0) +} + +#[hotpath::measure] +#[must_use] +pub fn deferred_document_section_rebuild_requested() -> bool { + DOCUMENT_SECTION_REBUILD_DEFERRED_DIRTY.with(Cell::get) +} + +thread_local! { + // §perf-heaven T2 tripwire: counts how many times `block_ix_for_paragraph` + // fell through past the aligned fast path. The fallthrough is an O(log N) + // tree rank query since T7.11 (the counter predates that and kept its role): + // a hot path calling it once per paragraph still shows up here as O(paragraphs) + // growth, which is why hot loops hoist a single `paragraph_block_rows` pass + // instead. Regression tests reset it, drive a mass op, and assert it stayed + // bounded. + static BLOCK_IX_SCAN_COUNT: Cell = const { Cell::new(0) }; +} + +/// Number of fallthrough (non-aligned) lookups performed by +/// [`block_ix_for_paragraph`] since the last reset — O(log N) each since +/// T7.11. A per-paragraph caller on an object-bearing doc drives this to +/// O(paragraphs); the batched [`paragraph_block_rows`] path leaves it at zero. +#[must_use] +pub fn block_ix_scan_count() -> u64 { + BLOCK_IX_SCAN_COUNT.with(Cell::get) +} + +/// Reset the [`block_ix_scan_count`] tripwire (test/measurement harness use). +pub fn reset_block_ix_scan_count() { + BLOCK_IX_SCAN_COUNT.with(|count| count.set(0)); +} + #[hotpath::measure] #[must_use] -pub fn block_ix_for_paragraph(document: &Document, target_paragraph_ix: usize) -> Option { +pub fn block_ix_for_paragraph(document: &DocumentProjection, target_paragraph_ix: usize) -> Option { if document.blocks.len() == document.paragraphs.len() && document .blocks @@ -73,21 +144,36 @@ pub fn block_ix_for_paragraph(document: &Document, target_paragraph_ix: usize) - return Some(target_paragraph_ix); } - let mut paragraph_ix = 0; - for (block_ix, block) in document.blocks.iter().enumerate() { - if matches!(block, Block::Paragraph(_)) { - if paragraph_ix == target_paragraph_ix { - return Some(block_ix); - } - paragraph_ix += 1; - } + // §perf-heaven T7.11: an O(log N) tree rank-select replaces the former + // O(blocks) linear scan. The counter still records this object-doc, per-call + // resolution so the intent-complexity guard keeps proving that mass ops hoist + // `paragraph_block_rows` once instead of resolving per paragraph (the per-call + // cost is now O(log N), but calling it O(paragraphs) times is still worse than + // one O(blocks) pass, so the guard remains meaningful). + BLOCK_IX_SCAN_COUNT.with(|count| count.set(count.get().saturating_add(1))); + document.blocks.block_row_for_paragraph_ix(target_paragraph_ix) +} + +/// Block row for EVERY paragraph in one pass — the batched sibling of +/// [`block_ix_for_paragraph`]. Callers that need rows for many paragraphs on +/// an object-bearing doc (where the aligned fast path misses) would otherwise +/// pay an O(blocks) scan per paragraph. +#[must_use] +pub fn paragraph_block_rows(document: &DocumentProjection) -> Vec { + if document.blocks.len() == document.paragraphs.len() { + return (0..document.blocks.len()).collect(); } - None + document + .blocks + .iter() + .enumerate() + .filter_map(|(row, block)| matches!(block, Block::Paragraph(_)).then_some(row)) + .collect() } #[hotpath::measure] #[must_use] -pub fn document_position_for_offset(document: &Document, offset: DocumentOffset) -> Option { +pub fn document_position_for_offset(document: &DocumentProjection, offset: DocumentOffset) -> Option { let paragraph = document.paragraphs.get(offset.paragraph)?; if offset.byte > paragraph_text_len(paragraph) { return None; @@ -100,7 +186,7 @@ pub fn document_position_for_offset(document: &Document, offset: DocumentOffset) #[hotpath::measure] #[must_use] -pub fn document_offset_for_position(document: &Document, position: &DocumentPosition) -> Option { +pub fn document_offset_for_position(document: &DocumentProjection, position: &DocumentPosition) -> Option { match position { DocumentPosition::Text { block_ix, byte } => { if document.blocks.len() == document.paragraphs.len() @@ -115,48 +201,60 @@ pub fn document_offset_for_position(document: &Document, position: &DocumentPosi return None; } - let mut paragraph_ix = 0_usize; - for (ix, block) in document.blocks.iter().enumerate() { - match block { - Block::Paragraph(paragraph) => { - if ix == *block_ix { - if *byte <= paragraph_text_len(paragraph) { - return Some(DocumentOffset { - paragraph: paragraph_ix, - byte: *byte, - }); - } - return None; - } - paragraph_ix += 1; - }, - Block::Image(_) | Block::Equation(_) | Block::Table(_) => { - if ix == *block_ix { - return None; - } - }, - } - } - None + // §perf-heaven T7.12: O(log N) rank query replaces the O(blocks) scan. + // `paragraph_ix_for_block_row` returns `Some` only when the row is a + // paragraph (object rows and out-of-range → `None`), so the following + // `get` is guaranteed to hand back that paragraph for the byte-bound check. + let paragraph_ix = document.blocks.paragraph_ix_for_block_row(*block_ix)?; + let Some(Block::Paragraph(paragraph)) = document.blocks.get(*block_ix) else { + return None; + }; + (*byte <= paragraph_text_len(paragraph)).then_some(DocumentOffset { + paragraph: paragraph_ix, + byte: *byte, + }) }, DocumentPosition::Object { .. } | DocumentPosition::TableCell { .. } => None, } } #[hotpath::measure] -pub fn update_paragraph_block(document: &mut Document, paragraph_ix: usize) { +pub fn update_paragraph_block(document: &mut DocumentProjection, paragraph_ix: usize) { + if let Some(block_ix) = block_ix_for_paragraph(document, paragraph_ix) { + update_paragraph_block_at(document, paragraph_ix, block_ix); + } +} + +/// Row-aware variant of [`update_paragraph_block`] for callers that have +/// ALREADY resolved the paragraph's block row (the projection patch-apply +/// carries an accurate `row_hint`). Skips the O(blocks) `block_ix_for_paragraph` +/// re-scan — the §perf-heaven T2 quadratic behind mass restyle: one scan per +/// patch over an object-bearing doc was O(paragraphs²). +pub fn update_paragraph_block_at(document: &mut DocumentProjection, paragraph_ix: usize, block_ix: usize) { let Some(paragraph) = document.paragraphs.get(paragraph_ix).cloned() else { return; }; - if let Some(block_ix) = block_ix_for_paragraph(document, paragraph_ix) - && let Some(block) = Arc::make_mut(&mut document.blocks).get_mut(block_ix) - { - *block = Block::Paragraph(paragraph); + if matches!(document.blocks.get(block_ix), Some(Block::Paragraph(_))) { + document.blocks.set(block_ix, Block::Paragraph(paragraph)); } } #[hotpath::measure] -pub fn replace_paragraph_blocks(document: &mut Document, start_paragraph: usize, old_count: usize, replacements: &[Paragraph]) { +pub fn replace_paragraph_blocks(document: &mut DocumentProjection, start_paragraph: usize, old_count: usize, replacements: &[Paragraph]) { + // Fast path: a single in-place paragraph update in a paragraph-only-aligned + // document. Block ids and order are unchanged, so we replace just that one + // block instead of rebuilding the whole block vector. + if old_count == 1 + && replacements.len() == 1 + && document.blocks.len() == document.paragraphs.len() + && matches!(document.blocks.get(start_paragraph), Some(Block::Paragraph(_))) + { + document.blocks.set(start_paragraph, Block::Paragraph(replacements[0].clone())); + reconcile_document_ids(document); + rebuild_document_sections(document); + return; + } + let block_start = block_ix_for_paragraph(document, start_paragraph).unwrap_or(document.blocks.len()); let mut paragraph_ix = 0; let mut output = Vec::with_capacity(document.blocks.len() + replacements.len()); @@ -192,7 +290,7 @@ pub fn replace_paragraph_blocks(document: &mut Document, start_paragraph: usize, output.push(Block::Paragraph(paragraph.clone())); } - document.blocks = Arc::new(output); + document.blocks = BlockSeq::from_vec(output); let block_end = (block_start + old_count).min(document.ids.block_ids.len()); let replacement_ids = if old_count == replacements.len() { document.ids.block_ids[block_start..block_end].to_vec() @@ -206,10 +304,7 @@ pub fn replace_paragraph_blocks(document: &mut Document, start_paragraph: usize, } ids }; - document - .ids - .block_ids - .splice(block_start..block_end, replacement_ids); + std::sync::Arc::make_mut(&mut document.ids.block_ids).splice(block_start..block_end, replacement_ids); reconcile_document_ids(document); rebuild_document_sections(document); } @@ -243,38 +338,37 @@ pub fn new_section_id() -> SectionId { pub fn document_ids_for_shape(paragraph_count: usize, block_count: usize) -> DocumentIds { DocumentIds { document_id: new_document_id(), - paragraph_ids: std::iter::repeat_with(new_paragraph_id) - .take(paragraph_count) - .collect(), - block_ids: std::iter::repeat_with(new_block_id) - .take(block_count) - .collect(), + paragraph_ids: std::sync::Arc::new(std::iter::repeat_with(new_paragraph_id).take(paragraph_count).collect()), + block_ids: std::sync::Arc::new(std::iter::repeat_with(new_block_id).take(block_count).collect()), } } #[hotpath::measure] -pub fn reconcile_document_ids(document: &mut Document) { +pub fn reconcile_document_ids(document: &mut DocumentProjection) { if document.ids.document_id == 0 { document.ids.document_id = new_document_id(); } - while document.ids.paragraph_ids.len() < document.paragraphs.len() { - document.ids.paragraph_ids.push(new_paragraph_id()); + if document.ids.paragraph_ids.len() != document.paragraphs.len() { + let ids = std::sync::Arc::make_mut(&mut document.ids.paragraph_ids); + while ids.len() < document.paragraphs.len() { + ids.push(new_paragraph_id()); + } + ids.truncate(document.paragraphs.len()); } - document - .ids - .paragraph_ids - .truncate(document.paragraphs.len()); - while document.ids.block_ids.len() < document.blocks.len() { - document.ids.block_ids.push(new_block_id()); + if document.ids.block_ids.len() != document.blocks.len() { + let ids = std::sync::Arc::make_mut(&mut document.ids.block_ids); + while ids.len() < document.blocks.len() { + ids.push(new_block_id()); + } + ids.truncate(document.blocks.len()); } - document.ids.block_ids.truncate(document.blocks.len()); } #[hotpath::measure] #[must_use] -pub fn paragraph_index_for_id(document: &Document, id: ParagraphId) -> Option { +pub fn paragraph_index_for_id(document: &DocumentProjection, id: ParagraphId) -> Option { document .ids .paragraph_ids @@ -282,61 +376,94 @@ pub fn paragraph_index_for_id(document: &Document, id: ParagraphId) -> Option Option { +pub fn paragraph_id_at(document: &DocumentProjection, paragraph_ix: usize) -> Option { document.ids.paragraph_ids.get(paragraph_ix).copied() } -#[hotpath::measure] +#[inline] #[must_use] -pub fn block_id_at(document: &Document, block_ix: usize) -> Option { +pub fn block_id_at(document: &DocumentProjection, block_ix: usize) -> Option { document.ids.block_ids.get(block_ix).copied() } #[hotpath::measure] -pub fn insert_paragraph_id(document: &mut Document, paragraph_ix: usize) -> ParagraphId { +pub fn insert_paragraph_id(document: &mut DocumentProjection, paragraph_ix: usize) -> ParagraphId { let id = new_paragraph_id(); - document - .ids - .paragraph_ids - .insert(paragraph_ix.min(document.ids.paragraph_ids.len()), id); + let at = paragraph_ix.min(document.ids.paragraph_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.paragraph_ids).insert(at, id); id } #[hotpath::measure] -pub fn insert_block_id(document: &mut Document, block_ix: usize) -> BlockId { +pub fn insert_block_id(document: &mut DocumentProjection, block_ix: usize) -> BlockId { let id = new_block_id(); - document - .ids - .block_ids - .insert(block_ix.min(document.ids.block_ids.len()), id); + let at = block_ix.min(document.ids.block_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.block_ids).insert(at, id); id } #[hotpath::measure] -pub fn remove_paragraph_ids(document: &mut Document, range: Range) { +pub fn remove_paragraph_ids(document: &mut DocumentProjection, range: Range) { let start = range.start.min(document.ids.paragraph_ids.len()); let end = range.end.min(document.ids.paragraph_ids.len()); if start < end { - document.ids.paragraph_ids.drain(start..end); + std::sync::Arc::make_mut(&mut document.ids.paragraph_ids).drain(start..end); } } #[hotpath::measure] -pub fn remove_block_ids(document: &mut Document, range: Range) { +pub fn remove_block_ids(document: &mut DocumentProjection, range: Range) { let start = range.start.min(document.ids.block_ids.len()); let end = range.end.min(document.ids.block_ids.len()); if start < end { - document.ids.block_ids.drain(start..end); + std::sync::Arc::make_mut(&mut document.ids.block_ids).drain(start..end); } } #[hotpath::measure] -pub fn rebuild_document_sections(document: &mut Document) { +pub fn rebuild_document_outline(document: &mut DocumentProjection) { + if document_section_rebuilds_deferred() { + DOCUMENT_SECTION_REBUILD_DEFERRED_DIRTY.with(|dirty| dirty.set(true)); + return; + } + rebuild_document_outline_now(document); +} + +#[hotpath::measure] +pub fn rebuild_document_outline_now(document: &mut DocumentProjection) { reconcile_document_ids(document); - let mut sections: Vec = Vec::new(); - let mut stack: Vec<(usize, SectionId)> = Vec::new(); + document.outline = Arc::new(document_outline(document)); +} + +/// Compatibility name retained for existing edit primitives. This rebuilds +/// only the derived heading outline; canonical `document.sections` are never +/// modified. +#[hotpath::measure] +pub fn rebuild_document_sections(document: &mut DocumentProjection) { + rebuild_document_outline(document); +} + +#[hotpath::measure] +pub fn rebuild_document_sections_now(document: &mut DocumentProjection) { + DOCUMENT_SECTION_REBUILD_DEFERRED_DIRTY.with(|dirty| dirty.set(false)); + rebuild_document_outline_now(document); +} + +/// Computes the disposable heading hierarchy purely from paragraph styles, +/// order, and stable paragraph ids. +#[hotpath::measure] +#[must_use] +pub fn document_outline(document: &DocumentProjection) -> Vec { + let mut outline: Vec = Vec::new(); + // §perf: the stack holds each open node's index into `outline` rather than its + // SectionId. `outline` is append-only (nodes are never removed), so the index + // stays valid, and closing a section becomes an O(1) index instead of the former + // O(outline) linear `find` per pop — turning this from O(headings²) to O(headings). + let mut stack: Vec<(usize, usize)> = Vec::new(); for (paragraph_ix, paragraph) in document.paragraphs.iter().enumerate() { let Some((level, kind)) = section_level_and_kind(document, paragraph.style) else { @@ -346,39 +473,51 @@ pub fn rebuild_document_sections(document: &mut Document) { .last() .is_some_and(|(ancestor_level, _)| *ancestor_level >= level) { - if let Some((_, section_id)) = stack.pop() { - for section in sections - .iter_mut() - .filter(|section| section.id == section_id) - { - section.end_paragraph_exclusive = paragraph_id_at(document, paragraph_ix); - } + if let Some((_, node_index)) = stack.pop() { + outline[node_index].end_paragraph_exclusive = paragraph_id_at(document, paragraph_ix); } } let paragraph_id = paragraph_id_at(document, paragraph_ix).unwrap_or_else(new_paragraph_id); - let parent_id = stack.last().map(|(_, id)| *id); + let parent_id = stack.last().map(|(_, node_index)| outline[*node_index].id); let id = section_id_for_heading(paragraph_id, kind); - sections.push(DocumentSection { + let node_index = outline.len(); + outline.push(DocumentOutlineNode { id, parent_id, kind, - heading_paragraph: Some(paragraph_id), + heading_paragraph: paragraph_id, start_paragraph: paragraph_id, end_paragraph_exclusive: None, }); - stack.push((level, id)); + stack.push((level, node_index)); } - for (_, section_id) in stack { - if let Some(section) = sections.iter_mut().find(|section| section.id == section_id) { - section.end_paragraph_exclusive = None; - } - } - document.sections = Arc::new(sections); + outline +} + +/// Whether the paragraph at `paragraph_ix` carries a heading (section) style. +#[hotpath::measure] +#[must_use] +pub fn paragraph_is_heading(document: &DocumentProjection, paragraph_ix: usize) -> bool { + document + .paragraphs + .get(paragraph_ix) + .is_some_and(|paragraph| section_level_and_kind(document, paragraph.style).is_some()) } +/// Whether any paragraph in `range` (clamped to the paragraph count) is a heading. +/// Lets callers decide whether a content edit can skip [`rebuild_document_outline`]. #[hotpath::measure] -fn section_level_and_kind(document: &Document, style: ParagraphStyle) -> Option<(usize, SectionKind)> { +#[must_use] +pub fn range_contains_heading(document: &DocumentProjection, range: Range) -> bool { + let end = range.end.min(document.paragraphs.len()); + (range.start..end).any(|paragraph_ix| paragraph_is_heading(document, paragraph_ix)) +} + +// §perf: not hotpath-measured — a map probe per paragraph per outline pass; +// the hooks dominated it at scale. +#[inline] +fn section_level_and_kind(document: &DocumentProjection, style: ParagraphStyle) -> Option<(usize, SectionKind)> { match style { ParagraphStyle::Normal => None, ParagraphStyle::Custom(slot) => { @@ -398,73 +537,7 @@ const fn section_id_for_heading(paragraph_id: ParagraphId, kind: SectionKind) -> SectionId(paragraph_id.0 ^ (kind_slot << 120)) } -/// Fenwick-tree (binary indexed tree) over the byte widths of each paragraph, -/// plus the raw widths. Lets us compute the absolute byte offset of any -/// paragraph in O(log N) and update it incrementally as the document is -/// edited. -#[derive(Clone, Debug)] -pub struct ParagraphOffsetIndex { - pub widths: Vec, - pub tree: Vec, -} - -#[hotpath::measure_all] -impl ParagraphOffsetIndex { - #[must_use] - pub fn new(paragraphs: &[Paragraph]) -> Self { - let mut index = Self { - widths: paragraph_widths(paragraphs), - tree: vec![0; paragraphs.len() + 1], - }; - for ix in 0..index.widths.len() { - index.add(ix, index.widths[ix] as isize); - } - index - } - - pub fn rebuild(&mut self, paragraphs: &[Paragraph]) { - *self = Self::new(paragraphs); - } - - #[must_use] - pub fn paragraph_start(&self, paragraph_ix: usize) -> usize { - self.prefix_sum(paragraph_ix) - } - - pub fn update_paragraph_width(&mut self, paragraph_ix: usize, paragraphs: &[Paragraph]) { - if paragraph_ix >= self.widths.len() || self.tree.len() != self.widths.len() + 1 { - self.rebuild(paragraphs); - return; - } - let Some(width) = paragraph_width(paragraphs, paragraph_ix) else { - return; - }; - let old_width = self.widths[paragraph_ix]; - if old_width == width { - return; - } - self.widths[paragraph_ix] = width; - self.add(paragraph_ix, width as isize - old_width as isize); - } - - fn add(&mut self, paragraph_ix: usize, delta: isize) { - if delta == 0 { - return; - } - let mut ix = paragraph_ix + 1; - while ix < self.tree.len() { - self.tree[ix] = self.tree[ix].saturating_add_signed(delta); - ix += ix & (!ix + 1); - } - } - - fn prefix_sum(&self, paragraph_count: usize) -> usize { - let mut ix = paragraph_count.min(self.widths.len()); - let mut sum = 0; - while ix > 0 { - sum += self.tree[ix]; - ix &= ix - 1; - } - sum - } -} +// §act-four M3 Slice 2b: the Fenwick `ParagraphOffsetIndex` has been removed. +// Paragraph offsets are now derived from the block tree's paragraph-space +// monoid (`BlockSeq::paragraph_start`), which subsumes it at `O(log N)` with no +// separate structure to keep in sync or snapshot per version. diff --git a/crates/gpui-flowtext/src/document/mod.rs b/crates/gpui-flowtext/src/document/mod.rs index 89f892c7..9e45d37d 100644 --- a/crates/gpui-flowtext/src/document/mod.rs +++ b/crates/gpui-flowtext/src/document/mod.rs @@ -1,4 +1,6 @@ include!("core.rs"); +include!("block_seq.rs"); +include!("paragraph_seq.rs"); include!("text.rs"); include!("blocks.rs"); include!("run_styles.rs"); diff --git a/crates/gpui-flowtext/src/document/paragraph_seq.rs b/crates/gpui-flowtext/src/document/paragraph_seq.rs new file mode 100644 index 00000000..c5fa381c --- /dev/null +++ b/crates/gpui-flowtext/src/document/paragraph_seq.rs @@ -0,0 +1,272 @@ +// §act-four M3 Slice 4 — `ParagraphSeq`: the persistent, structurally-shared +// paragraph sequence that replaces `Arc>` on +// `DocumentProjection`. +// +// Backed by a persistent RRB vector (`im::Vector`): `O(1)` clone with +// structural sharing (so a retained version's paragraphs cost `O(change)`, not +// `O(document)`), `O(log N)` random access + splice. Read sites keep their +// shape — `document.paragraphs[i]`, `.get(i)`, `.iter()`, `.len()`, `.first()` +// all hand back `&Paragraph` tied to the field (a field, not a temporary view, +// so borrows live as long as the projection). The complex mutation sites go +// through `paragraphs_mut(...)`'s materialize→mutate→rebuild guard; hot single- +// paragraph edits use the `O(log N)` `set`/`insert`/`remove` fast paths. + +/// A persistent, tree-backed paragraph sequence. Drop-in for the old +/// `Arc>` field. +#[derive(Clone, Debug, Default)] +pub struct ParagraphSeq { + paragraphs: im::Vector, +} + +impl ParagraphSeq { + #[must_use] + pub fn from_vec(paragraphs: Vec) -> Self { + Self { + paragraphs: paragraphs.into_iter().collect(), + } + } + + #[must_use] + pub fn len(&self) -> usize { + self.paragraphs.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.paragraphs.is_empty() + } + + #[must_use] + pub fn get(&self, index: usize) -> Option<&Paragraph> { + self.paragraphs.get(index) + } + + #[must_use] + pub fn first(&self) -> Option<&Paragraph> { + self.paragraphs.front() + } + + #[must_use] + pub fn last(&self) -> Option<&Paragraph> { + self.paragraphs.back() + } + + /// In-order iteration by reference — backs `document.paragraphs.iter()`. + pub fn iter(&self) -> im::vector::Iter<'_, Paragraph> { + self.paragraphs.iter() + } + + #[must_use] + pub fn to_vec(&self) -> Vec { + self.paragraphs.iter().cloned().collect() + } + + /// Owned copy of the paragraphs in `range` (replaces `paragraphs[a..b].to_vec()`). + #[must_use] + pub fn range_to_vec(&self, range: std::ops::Range) -> Vec { + let len = self.len(); + let start = range.start.min(len); + let end = range.end.min(len).max(start); + self.paragraphs.iter().skip(start).take(end - start).cloned().collect() + } + + // -- `O(log N)` fast-path mutations (path-copy, share the rest) ----------- + + /// Replace the paragraph at `index` (no-op if out of range). `O(log N)`. + pub fn set(&mut self, index: usize, paragraph: Paragraph) { + if index < self.len() { + self.paragraphs.set(index, paragraph); + } + } + + /// Insert `paragraph` before `index` (clamped to `len`). `O(log N)`. + pub fn insert(&mut self, index: usize, paragraph: Paragraph) { + self.paragraphs.insert(index.min(self.len()), paragraph); + } + + /// Remove the paragraph at `index` (no-op if out of range). `O(log N)`. + pub fn remove(&mut self, index: usize) { + if index < self.len() { + self.paragraphs.remove(index); + } + } + + /// Append `paragraph`. `O(log N)`. + pub fn push(&mut self, paragraph: Paragraph) { + self.paragraphs.push_back(paragraph); + } + + /// Copy-on-write mutable access to one paragraph — clones the shared node + /// only if a retained version still holds it. `O(log N)`. Borrows only + /// `self`, so callers can interleave a following block edit (NLL releases the + /// borrow at last use — no `Drop` guard to pin `&mut document`). + pub fn get_mut(&mut self, index: usize) -> Option<&mut Paragraph> { + self.paragraphs.get_mut(index) + } + + /// Copy-on-write mutable iteration over the paragraphs. `O(N)`. + pub fn iter_mut(&mut self) -> im::vector::IterMut<'_, Paragraph> { + self.paragraphs.iter_mut() + } + + /// Replace `range` with `replacement` (the `Vec::splice` shape used by a few + /// mutation sites). `O(log N + |replacement|)`. + pub fn splice(&mut self, range: std::ops::Range, replacement: Vec) { + let len = self.len(); + let start = range.start.min(len); + let end = range.end.min(len).max(start); + let tail = self.paragraphs.split_off(end); + let _removed = self.paragraphs.split_off(start); + for paragraph in replacement { + self.paragraphs.push_back(paragraph); + } + self.paragraphs.append(tail); + } + + /// Mutable access as a `Vec` for the complex sites that run several ops at + /// once: materialize now, rebuild the persistent vector on the guard's + /// `Drop`. `O(N)` — hot single-paragraph sites should prefer the fast paths. + #[must_use] + pub fn make_mut(&mut self) -> ParagraphSeqMut<'_> { + let vec = self.to_vec(); + ParagraphSeqMut { seq: self, vec } + } +} + +impl std::ops::Index for ParagraphSeq { + type Output = Paragraph; + fn index(&self, index: usize) -> &Paragraph { + &self.paragraphs[index] + } +} + +impl From> for ParagraphSeq { + fn from(paragraphs: Vec) -> Self { + Self::from_vec(paragraphs) + } +} + +impl FromIterator for ParagraphSeq { + fn from_iter>(iter: I) -> Self { + Self { + paragraphs: iter.into_iter().collect(), + } + } +} + +impl<'a> IntoIterator for &'a ParagraphSeq { + type Item = &'a Paragraph; + type IntoIter = im::vector::Iter<'a, Paragraph>; + fn into_iter(self) -> Self::IntoIter { + self.paragraphs.iter() + } +} + +impl<'a> IntoIterator for &'a mut ParagraphSeq { + type Item = &'a mut Paragraph; + type IntoIter = im::vector::IterMut<'a, Paragraph>; + fn into_iter(self) -> Self::IntoIter { + self.paragraphs.iter_mut() + } +} + +/// A materialize→mutate→rebuild guard. Derefs to the underlying `Vec` +/// so existing `paragraphs_mut(...)` bodies keep their `Vec` operations +/// verbatim; on `Drop` the mutated `Vec` is rebuilt into the persistent vector. +pub struct ParagraphSeqMut<'a> { + seq: &'a mut ParagraphSeq, + vec: Vec, +} + +impl std::ops::Deref for ParagraphSeqMut<'_> { + type Target = Vec; + fn deref(&self) -> &Vec { + &self.vec + } +} + +impl std::ops::DerefMut for ParagraphSeqMut<'_> { + fn deref_mut(&mut self) -> &mut Vec { + &mut self.vec + } +} + +impl Drop for ParagraphSeqMut<'_> { + fn drop(&mut self) { + self.seq.paragraphs = std::mem::take(&mut self.vec).into_iter().collect(); + } +} + +#[cfg(test)] +mod paragraph_seq_tests { + use super::*; + use crate::{ParagraphStyle, TextRun}; + + fn para(byte_len: usize) -> Paragraph { + Paragraph { + style: ParagraphStyle::Normal, + runs: if byte_len == 0 { + Vec::new() + } else { + vec![TextRun { + len: byte_len, + styles: crate::RunStyles::default(), + }] + }, + version: 0, + } + } + + fn seq_and_ref(lens: &[usize]) -> (ParagraphSeq, Vec) { + let reference: Vec = lens.iter().map(|&l| para(l)).collect(); + (ParagraphSeq::from_vec(reference.clone()), reference) + } + + #[test] + fn read_api_matches_reference_vec() { + let (seq, reference) = seq_and_ref(&[3, 5, 0, 8, 2]); + assert_eq!(seq.len(), reference.len()); + assert!(!seq.is_empty()); + for (ix, expected) in reference.iter().enumerate() { + assert_eq!(seq.get(ix), Some(expected)); + assert_eq!(&seq[ix], expected); + } + assert_eq!(seq.get(reference.len()), None); + assert_eq!(seq.first(), reference.first()); + assert_eq!(seq.last(), reference.last()); + assert_eq!(seq.iter().collect::>(), reference.iter().collect::>()); + assert_eq!(seq.to_vec(), reference); + } + + #[test] + fn mutations_match_vec_semantics() { + let (mut seq, mut reference) = seq_and_ref(&[1, 2, 3, 4]); + seq.set(1, para(9)); + reference[1] = para(9); + seq.insert(2, para(7)); + reference.insert(2, para(7)); + seq.remove(0); + reference.remove(0); + seq.push(para(5)); + reference.push(para(5)); + assert_eq!(seq.to_vec(), reference); + let mut guard = seq.make_mut(); + guard.insert(1, para(8)); + guard[0] = para(4); + guard.remove(2); + drop(guard); + reference.insert(1, para(8)); + reference[0] = para(4); + reference.remove(2); + assert_eq!(seq.to_vec(), reference); + } + + #[test] + fn clone_shares_and_is_persistent() { + let (seq, original) = seq_and_ref(&[1, 2, 3, 4, 5]); + let mut edited = seq.clone(); + edited.set(2, para(99)); + assert_eq!(seq.to_vec(), original); + assert_ne!(edited.to_vec(), seq.to_vec()); + } +} diff --git a/crates/gpui-flowtext/src/document/position.rs b/crates/gpui-flowtext/src/document/position.rs index d1b41e3a..e9937914 100644 --- a/crates/gpui-flowtext/src/document/position.rs +++ b/crates/gpui-flowtext/src/document/position.rs @@ -12,6 +12,65 @@ pub enum ObjectAffinity { After, } +/// §16 selection affinity: which side of a position a selection endpoint +/// "sticks" to. +/// +/// This is the genuine, stored intent behind a caret/endpoint — it is **not** +/// re-derived from selection direction. It records *why* a side was chosen so +/// the caret behaves correctly across concurrent inserts at the same position, +/// undo/redo restoration, object/paragraph boundaries, and selection extension: +/// +/// * [`SelectionAffinity::Before`] — the endpoint belongs to the content +/// *before* the offset (e.g. the caret arrived by moving Left, or sits at the +/// trailing edge of the preceding glyph). Stable cursors anchor to that +/// preceding glyph when one exists. +/// * [`SelectionAffinity::After`] — the endpoint belongs to the content *after* +/// the offset (e.g. the caret arrived by moving Right, or sits at the leading +/// edge of the following glyph). Stable cursors anchor to that following glyph +/// when one exists. +/// * [`SelectionAffinity::Neutral`] — no strong side was expressed (fresh edit +/// result, mouse placement, programmatic selection). Maps to a middle cursor +/// side. +/// +/// The collaboration runtime maps these onto Loro stable cursor boundaries when +/// encoding presence/undo cursors instead of guessing from selection direction. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub enum SelectionAffinity { + Before, + After, + #[default] + Neutral, +} + +/// §16 visual gravity: which visual line a soft-wrap-seam offset renders on. +/// +/// At a soft wrap, one byte offset is shared by the end of the upper visual line +/// and the start of the lower visual line. Affinity alone cannot disambiguate +/// this purely visual choice, so gravity is tracked separately and **consumed** +/// by caret layout (see `caret_bounds`/`locate_line`): +/// +/// * [`VisualGravity::Upstream`] — render at the trailing edge of the visual +/// line that *ends* at the offset (end of the upper line). Produced by +/// "move/extend to line end". +/// * [`VisualGravity::Downstream`] — render at the leading edge of the visual +/// line that *starts* at the offset (start of the lower line). Produced by +/// "move/extend to line start". +/// * [`VisualGravity::Neutral`] — editor default. Preserves the historical +/// "caret-at-start-of-next-line" wrap-seam bias (i.e. behaves like +/// `Downstream` at a seam), used for arrow/word/vertical motion, fresh edits, +/// mouse placement, and remote carets. +/// +/// Bidi/RTL reordering is **not** modeled here; gravity currently disambiguates +/// only soft-wrap seams in left-to-right text. RTL caret placement remains a +/// defined-but-simplified behavior (treated as `Neutral`). +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub enum VisualGravity { + Upstream, + Downstream, + #[default] + Neutral, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub enum DocumentPosition { Text { diff --git a/crates/gpui-flowtext/src/document/run_styles.rs b/crates/gpui-flowtext/src/document/run_styles.rs index ae9da5d1..f49c9f24 100644 --- a/crates/gpui-flowtext/src/document/run_styles.rs +++ b/crates/gpui-flowtext/src/document/run_styles.rs @@ -29,6 +29,57 @@ impl HighlightStyle { } } +/// Vertical alignment of a run's glyphs — OOXML `w:vertAlign` (superscript / +/// subscript). Orthogonal to [`RunSemanticStyle`]: a run can be, e.g., +/// emphasized *and* superscript, so this is a separate field on [`RunStyles`] +/// rather than a semantic slot (mirrors `strikethrough`). +/// +/// PHASE-3 NOTE (deferred — not done here): as of this change super/subscript is +/// **pure data**. It is captured on `.docx` import, persisted as a Loro mark, and +/// re-emitted on `.docx` export, but it is **not rendered in the editor and not +/// user-toggleable yet**. To finish it: +/// * Rendering — `gpui-flowtext/src/rich_text/layout/shaping.rs`: apply a +/// baseline shift + ~0.65× font scale for `Superscript`/`Subscript` (this is +/// NOT a decoration line like `strikethrough` in `paint.rs`; it is a +/// glyph-metrics change, closer to how font size is handled). +/// * Editor toggle + selection state — follow `strikethrough` across +/// `rich_text/tools.rs`, `rich_text/editor/formatting.rs`, +/// `rich_text/editor/style_state.rs`, `rich_text/editor/commands.rs`. +/// * Collab runtime — mirror `MARK_STRIKETHROUGH` in the collab write/clear +/// paths (`flowstate-collab/src/crdt_runtime.rs`, +/// `crdt_runtime/projection_patch.rs`, `local_write/commit.rs`) so live +/// edits read/clear the mark; the batch import/export pipeline already does. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub enum VertAlign { + #[default] + Baseline, + Superscript, + Subscript, +} + +impl VertAlign { + /// Loro mark value for this alignment; `None` for `Baseline` (the mark is + /// simply absent). Mirrors the `slot()` pattern of the other run-style marks. + #[must_use] + pub const fn mark_value(self) -> Option { + match self { + Self::Baseline => None, + Self::Superscript => Some(1), + Self::Subscript => Some(2), + } + } + + /// Decode a Loro mark value back into an alignment (unknown ⇒ `Baseline`). + #[must_use] + pub const fn from_mark_value(value: i64) -> Self { + match value { + 1 => Self::Superscript, + 2 => Self::Subscript, + _ => Self::Baseline, + } + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum RunStyle { Plain, @@ -76,6 +127,13 @@ impl RunStyles { self.strikethrough = true; self } + + #[must_use] + #[hotpath::skip] + pub const fn with_vert_align(mut self, vert_align: VertAlign) -> Self { + self.vert_align = vert_align; + self + } } // -- Theme ---------------------------------------------------------------- diff --git a/crates/gpui-flowtext/src/document/text.rs b/crates/gpui-flowtext/src/document/text.rs index e56f08d6..20640a3a 100644 --- a/crates/gpui-flowtext/src/document/text.rs +++ b/crates/gpui-flowtext/src/document/text.rs @@ -1,10 +1,16 @@ #[derive(Clone, Debug, PartialEq, Eq)] pub struct Paragraph { pub style: ParagraphStyle, - pub byte_range: Range, pub runs: Vec, pub version: u64, } +// §perf-heaven T8.6: `byte_range` was a per-paragraph ABSOLUTE-offset cache that +// `update_paragraph_offsets_after_len_change` shifted across the whole tail on +// EVERY content edit (im::Vector + block-tree tail COW = ~3.8 MB/keystroke on a +// 6k-para doc, O(tail)). It is redundant: the block tree's `paragraph_start` +// already derives the offset from run lengths (O(log N), maintained), and the +// length is `paragraph_text_len`. Removed the field; read sites derive via +// `paragraph_byte_range(document, ix)`. The per-edit tail shift is gone. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct ParagraphId(pub u128); @@ -12,14 +18,71 @@ pub struct ParagraphId(pub u128); #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct BlockId(pub u128); +/// Durable identity of a table row (§P2b). Minted once at row creation and +/// carried through canonical Loro (`row_order` / `rows_by_id`), the projection, +/// and every editor command so concurrent structural table edits converge on +/// the same row instead of resolving a stale positional index against a merged +/// order. +/// +/// INVARIANT: like every durable id ([`ParagraphId`], [`BlockId`], `AssetId`), +/// a `RowId` must be GLOBALLY UNIQUE within a document — never reused across two +/// tables. A cell's identity is [`CellId::from_coordinate`] of `(row_id, +/// column_id)`, and cell text flows share one global registry keyed by that id, +/// so two tables reusing a `(row, column)` coordinate would collide their flows +/// and lose text. Mint from `uuid::Uuid::new_v4().as_u128()` (editors/paste) or +/// a per-document-unique seed (importers); only [`ColumnId`] may be table-local +/// because a globally-unique `RowId` already makes every coordinate unique. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct RowId(pub u128); + +/// Durable identity of a table column (§P2b). See [`RowId`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct ColumnId(pub u128); + +/// Durable identity of a table cell (§P2b). Deterministic from its +/// `(RowId, ColumnId)` coordinate via [`CellId::from_coordinate`] so two peers +/// that concurrently materialize the same coordinate address the identical Loro +/// container (last-writer-wins merge, never a duplicate). +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct CellId(pub u128); + +impl CellId { + /// Deterministic cell identity for a `(row, column)` coordinate (§P2b). + /// + /// The id is a BLAKE3 mix of the two u128 ids — deliberately NOT the string + /// `"{row}.{column}"`, because the canonical `loro_id_u128` decoder parses + /// only the trailing numeric segment of a dotted id and would collapse every + /// cell in a column onto the same id. Mixing both halves keeps every + /// coordinate distinct while staying a pure function of the coordinate, so + /// any peer (or the topology-repair pass) that needs cell `(R, C)` derives the + /// identical [`CellId`] and `ensure`s the same Loro container. + #[must_use] + pub fn from_coordinate(row: RowId, column: ColumnId) -> Self { + let mut input = [0_u8; 32]; + input[..16].copy_from_slice(&row.0.to_le_bytes()); + input[16..].copy_from_slice(&column.0.to_le_bytes()); + let digest = blake3::hash(&input); + Self(u128::from_le_bytes( + digest.as_bytes()[..16] + .try_into() + .expect("BLAKE3 digest always contains at least sixteen bytes"), + )) + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct SectionId(pub u128); +/// §act-ten A10.1: the id vectors are `Arc`-COW. A projection clone is taken on +/// every publish/event; with plain `Vec`s each clone memcpy'd O(paragraphs) +/// (~200 KB/keystroke on a 6k-para doc) even though ids only change on +/// STRUCTURAL ops. `Arc` makes the per-keystroke clone a refcount bump; the +/// rare structural edit pays one copy-on-write via [`std::sync::Arc::make_mut`]. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct DocumentIds { pub document_id: u128, - pub paragraph_ids: Vec, - pub block_ids: Vec, + pub paragraph_ids: std::sync::Arc>, + pub block_ids: std::sync::Arc>, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] @@ -27,7 +90,7 @@ pub enum SectionKind { Custom(u8), } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct DocumentSection { pub id: SectionId, pub parent_id: Option, @@ -35,6 +98,145 @@ pub struct DocumentSection { pub heading_paragraph: Option, pub start_paragraph: ParagraphId, pub end_paragraph_exclusive: Option, + /// §11 page-structure payload for this section, when known. + /// + /// The heading-outline computation ([`document_sections`]) cannot derive page + /// structure from paragraph styles, so it leaves this `None`. The canonical + /// values live in Loro and are populated by the `flowstate-document` + /// projector. `#[serde(default)]` keeps older cached projections (which had no + /// such field) deserializable. + #[serde(default)] + pub page: Option, +} + +/// Derived heading hierarchy for navigation and collapse UI. +/// +/// Unlike [`DocumentSection`], this type is never canonical document state and +/// deliberately carries no page-layout payload. It may be rebuilt at any time +/// from paragraph styles and stable paragraph ids. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DocumentOutlineNode { + pub id: SectionId, + pub parent_id: Option, + pub kind: SectionKind, + pub heading_paragraph: ParagraphId, + pub start_paragraph: ParagraphId, + pub end_paragraph_exclusive: Option, +} + +/// US Letter width in twips (8.5in x 1440). Mirrors the `flowstate-document` +/// Loro encoding so projection mapping is a trivial field-by-field copy (§11). +const DEFAULT_PAGE_WIDTH_TWIPS: i64 = 12_240; +/// US Letter height in twips (11in x 1440). +const DEFAULT_PAGE_HEIGHT_TWIPS: i64 = 15_840; +/// One-inch margin in twips. +const DEFAULT_MARGIN_TWIPS: i64 = 1_440; + +/// §11 page-structure attributes carried by a [`DocumentSection`]. +/// +/// This is the gpui-flowtext-native projection mirror of +/// `flowstate_document::loro_schema::SectionPageAttrs`. gpui-flowtext must not +/// depend on `flowstate-document` (that crate depends on this one), so the type +/// is defined here and the projector maps Loro values onto it field-for-field. +/// All units/semantics match the canonical Loro encoding: lengths are in twips +/// (1/1440 inch), `columns` is a count, and header/footer flow ids reference +/// independent text flows. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SectionPageAttrs { + pub page_size: SectionPageSize, + pub margins: SectionMargins, + pub columns: i64, + pub orientation: SectionOrientation, + pub page_numbering: SectionPageNumbering, + pub header_flow_id: Option, + pub footer_flow_id: Option, +} + +impl Default for SectionPageAttrs { + fn default() -> Self { + Self { + page_size: SectionPageSize::default(), + margins: SectionMargins::default(), + columns: 1, + orientation: SectionOrientation::Portrait, + page_numbering: SectionPageNumbering::default(), + header_flow_id: None, + footer_flow_id: None, + } + } +} + +/// §11 page size in twips. Defaults to US Letter. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SectionPageSize { + pub width_twips: i64, + pub height_twips: i64, +} + +impl Default for SectionPageSize { + fn default() -> Self { + Self { + width_twips: DEFAULT_PAGE_WIDTH_TWIPS, + height_twips: DEFAULT_PAGE_HEIGHT_TWIPS, + } + } +} + +/// §11 section margins in twips. Defaults to one-inch margins. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SectionMargins { + pub top_twips: i64, + pub right_twips: i64, + pub bottom_twips: i64, + pub left_twips: i64, +} + +impl Default for SectionMargins { + fn default() -> Self { + Self { + top_twips: DEFAULT_MARGIN_TWIPS, + right_twips: DEFAULT_MARGIN_TWIPS, + bottom_twips: DEFAULT_MARGIN_TWIPS, + left_twips: DEFAULT_MARGIN_TWIPS, + } + } +} + +/// §11 section page orientation. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum SectionOrientation { + #[default] + Portrait, + Landscape, +} + +/// §11 page-numbering descriptor for a section. Defaults to no numbering +/// starting at 1. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SectionPageNumbering { + pub format: PageNumberFormat, + pub start: i64, +} + +impl Default for SectionPageNumbering { + fn default() -> Self { + Self { + format: PageNumberFormat::None, + start: 1, + } + } +} + +/// §11 page-number rendering format for a section. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum PageNumberFormat { + #[default] + None, + Decimal, + LowerRoman, + UpperRoman, + LowerAlpha, + UpperAlpha, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] @@ -75,19 +277,29 @@ pub struct DocumentParagraphInput { pub struct DocumentSpan { pub start_paragraph: usize, pub paragraphs: Vec, + /// Durable paragraph ids parallel to `paragraphs` (one id per paragraph). + /// + /// Carried so the optimistic replay, the runtime prediction, and the canonical + /// Loro apply of a `ReplaceParagraphSpan` all keep the SAME identities instead + /// of each side re-deriving ids (positionally on the client, by Loro metadata + /// survival canonically) and diverging on which id lives at a merged boundary. + pub paragraph_ids: Vec, + /// Durable block ids for the paragraph-blocks, parallel to `paragraphs`. Each + /// [`Paragraph`] is exactly one block, so there is one block id per paragraph. + pub block_ids: Vec, pub text: String, } /// Input-shape used by document builders (demo data, clipboard fragments). /// Carries explicit run text instead of byte offsets so the higher-level /// helpers can splice in arbitrary content. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputRun { pub text: String, pub styles: RunStyles, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputParagraph { pub style: ParagraphStyle, pub runs: Vec, @@ -102,7 +314,7 @@ pub struct InputAsset { pub bytes: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputBlock { Paragraph(InputParagraph), Image(InputImageBlock), @@ -110,80 +322,98 @@ pub enum InputBlock { Table(InputTableBlock), } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputImageBlock { pub asset_id: AssetId, pub alt_text: String, pub caption: Option, pub sizing: InputImageSizing, pub alignment: InputBlockAlignment, + /// §A11.9: external target URL for a genuinely-LINKED image (no embedded + /// media part). See [`ImageBlock::external_url`]. `serde(default)` keeps + /// pre-existing serialized payloads (clipboard fragments) decodable. + #[serde(default)] + pub external_url: Option, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputImageSizing { Intrinsic, FitWidth, Fixed { width_px: u32, height_px: Option }, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputBlockAlignment { Left, Center, Right, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputEquationBlock { pub source: String, pub syntax: InputEquationSyntax, pub display: InputEquationDisplay, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputEquationSyntax { Latex, } -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputEquationDisplay { Display, InlineLikeParagraph, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputTableBlock { pub rows: Vec, - pub column_widths: Vec, + /// Ordered columns, each carrying its durable [`ColumnId`] and width (§P2b). + /// Replaces the id-less `column_widths` list so column identity survives + /// concurrent insert/remove/reorder. + pub columns: Vec, pub style: InputTableStyle, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct InputTableColumn { + pub id: ColumnId, + pub width: InputTableColumnWidth, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputTableRow { + pub id: RowId, pub cells: Vec, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputTableCell { + pub id: CellId, + pub row_id: RowId, + pub column_id: ColumnId, pub blocks: Vec, pub row_span: u16, pub col_span: u16, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputTableCellBlock { Paragraph(InputParagraph), Table(InputTableBlock), } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub enum InputTableColumnWidth { Auto, FixedPx(u32), Fraction(u32), } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct InputTableStyle { pub header_row: bool, } @@ -194,4 +424,7 @@ pub struct RunStyles { pub direct_underline: bool, pub strikethrough: bool, pub highlight: Option, + /// Superscript/subscript. Orthogonal to `semantic`; imported/exported and + /// persisted, but not yet rendered or user-toggleable — see [`VertAlign`]. + pub vert_align: VertAlign, } diff --git a/crates/gpui-flowtext/src/document/theme.rs b/crates/gpui-flowtext/src/document/theme.rs index 820509e4..695ce061 100644 --- a/crates/gpui-flowtext/src/document/theme.rs +++ b/crates/gpui-flowtext/src/document/theme.rs @@ -181,4 +181,4 @@ impl DocumentTheme { } } -// -- Document offset ------------------------------------------------------ +// -- DocumentProjection offset ------------------------------------------------------ diff --git a/crates/gpui-flowtext/src/edit_ops/fragments.rs b/crates/gpui-flowtext/src/edit_ops/fragments.rs index e732820a..27eec49f 100644 --- a/crates/gpui-flowtext/src/edit_ops/fragments.rs +++ b/crates/gpui-flowtext/src/edit_ops/fragments.rs @@ -1,6 +1,6 @@ #[hotpath::measure] #[must_use] -pub fn selected_rich_fragment(document: &Document, range: Range) -> RichClipboardFragment { +pub fn selected_rich_fragment(document: &DocumentProjection, range: Range) -> RichClipboardFragment { if document.paragraphs.is_empty() || range.start.paragraph > range.end.paragraph { return RichClipboardFragment { format: RICH_TEXT_CLIPBOARD_FORMAT.to_owned(), diff --git a/crates/gpui-flowtext/src/edit_ops/insert_delete.rs b/crates/gpui-flowtext/src/edit_ops/insert_delete.rs index efb68bad..4a7f1a84 100644 --- a/crates/gpui-flowtext/src/edit_ops/insert_delete.rs +++ b/crates/gpui-flowtext/src/edit_ops/insert_delete.rs @@ -1,13 +1,16 @@ #[hotpath::measure] -pub fn insert_text_at(document: &mut Document, paragraph_ix: usize, byte: usize, text: &str, styles: RunStyles) { +pub fn insert_text_at(document: &mut DocumentProjection, paragraph_ix: usize, byte: usize, text: &str, styles: RunStyles) { if text.is_empty() { return; } + let byte = clamp_paragraph_byte_to_char_boundary(document, paragraph_ix, byte); let insert_len = text.len(); let paragraph_start = paragraph_byte_range(document, paragraph_ix).start; document.text.insert(paragraph_start + byte, text); { - let paragraph = &mut paragraphs_mut(document)[paragraph_ix]; + let Some(paragraph) = paragraphs_mut(document).get_mut(paragraph_ix) else { + return; + }; bump_paragraph_version(paragraph); if paragraph.runs.is_empty() { paragraph.runs.push(TextRun { len: insert_len, styles }); @@ -88,16 +91,21 @@ pub fn insert_text_at(document: &mut Document, paragraph_ix: usize, byte: usize, // `paragraph`. Runs are split or dropped as needed; remaining runs are re- // merged so adjacent same-style fragments coalesce. #[hotpath::measure] -pub fn delete_range_in_paragraph(document: &mut Document, paragraph_ix: usize, range: Range) { - if range.start >= range.end { +pub fn delete_range_in_paragraph(document: &mut DocumentProjection, paragraph_ix: usize, range: Range) { + let start = clamp_paragraph_byte_to_char_boundary(document, paragraph_ix, range.start); + let end = clamp_paragraph_byte_to_char_boundary(document, paragraph_ix, range.end); + if start >= end { return; } + let range = start..end; let paragraph_start = paragraph_byte_range(document, paragraph_ix).start; document .text .delete(paragraph_start + range.start..paragraph_start + range.end); { - let paragraph = &mut paragraphs_mut(document)[paragraph_ix]; + let Some(paragraph) = paragraphs_mut(document).get_mut(paragraph_ix) else { + return; + }; bump_paragraph_version(paragraph); let mut offset = 0; let mut new_runs: Vec = Vec::with_capacity(paragraph.runs.len()); diff --git a/crates/gpui-flowtext/src/edit_ops/mutate.rs b/crates/gpui-flowtext/src/edit_ops/mutate.rs index a71d832f..09999f4a 100644 --- a/crates/gpui-flowtext/src/edit_ops/mutate.rs +++ b/crates/gpui-flowtext/src/edit_ops/mutate.rs @@ -1,12 +1,14 @@ #[hotpath::measure] -pub fn mutate_runs_in_range(document: &mut Document, range: Range, mut mutate: impl FnMut(&mut RunStyles)) { +pub fn mutate_runs_in_range(document: &mut DocumentProjection, range: Range, mut mutate: impl FnMut(&mut RunStyles)) { let paragraph_count = paragraphs_mut(document).len(); if paragraph_count == 0 || range.start.paragraph >= paragraph_count || range.start.paragraph > range.end.paragraph { return; } let last_paragraph = range.end.paragraph.min(paragraph_count - 1); for paragraph_ix in range.start.paragraph..=last_paragraph { - let paragraph = &mut paragraphs_mut(document)[paragraph_ix]; + let Some(paragraph) = paragraphs_mut(document).get_mut(paragraph_ix) else { + continue; + }; let start = if paragraph_ix == range.start.paragraph { range.start.byte } else { 0 }; let end = if paragraph_ix == range.end.paragraph { range.end.byte diff --git a/crates/gpui-flowtext/src/edit_ops/offsets.rs b/crates/gpui-flowtext/src/edit_ops/offsets.rs index f930d8b7..919a0835 100644 --- a/crates/gpui-flowtext/src/edit_ops/offsets.rs +++ b/crates/gpui-flowtext/src/edit_ops/offsets.rs @@ -1,4 +1,6 @@ -#[hotpath::measure] +// §perf: not hotpath-measured — a field sum called on every offset/length +// query; the measurement hooks dwarfed the work and skewed profiles. +#[inline] #[must_use] pub fn paragraph_runs_len(paragraph: &Paragraph) -> usize { paragraph.runs.iter().map(|run| run.len).sum() @@ -24,55 +26,59 @@ pub fn paragraph_width(paragraphs: &[Paragraph], paragraph_ix: usize) -> Option< #[hotpath::measure] #[must_use] -pub fn paragraph_byte_range(document: &Document, paragraph_ix: usize) -> Range { - let start = document.offset_index.paragraph_start(paragraph_ix); +pub fn paragraph_byte_range(document: &DocumentProjection, paragraph_ix: usize) -> Range { + // §act-four M3 Slice 2b: paragraph offsets come from the block tree's + // paragraph-space monoid (`paragraph_start`), which subsumes the Fenwick + // `ParagraphOffsetIndex`. Valid because the block tree mirrors the paragraph + // runs, and `paragraph_start` depends only on run lengths (not byte_range). + let start = document.blocks.paragraph_start(paragraph_ix); start..start + paragraph_text_len(&document.paragraphs[paragraph_ix]) } +/// Clamp a paragraph-local UTF-8 byte offset to the nearest valid character +/// boundary at or before it. +/// +/// Editor coordinates are byte-based. Keeping this normalization at the +/// document boundary prevents a stale or transformed caret from ever reaching +/// `crop::Rope` with an interior UTF-8 code-unit offset. #[hotpath::measure] -pub fn refresh_paragraph_range(document: &mut Document, paragraph_ix: usize) { - let range = paragraph_byte_range(document, paragraph_ix); - paragraphs_mut(document)[paragraph_ix].byte_range = range; -} - -#[hotpath::measure] -pub fn refresh_paragraph_ranges(document: &mut Document) { - for paragraph_ix in 0..document.paragraphs.len() { - refresh_paragraph_range(document, paragraph_ix); - } -} - -#[hotpath::measure] -fn refresh_paragraph_ranges_from(document: &mut Document, start_paragraph: usize) { - let paragraph_count = document.paragraphs.len(); - for paragraph_ix in start_paragraph.min(paragraph_count)..paragraph_count { - refresh_paragraph_range(document, paragraph_ix); +#[must_use] +pub fn clamp_paragraph_byte_to_char_boundary(document: &DocumentProjection, paragraph_ix: usize, byte: usize) -> usize { + let Some(paragraph) = document.paragraphs.get(paragraph_ix) else { + return 0; + }; + let paragraph_start = document.blocks.paragraph_start(paragraph_ix); + let mut byte = byte.min(paragraph_text_len(paragraph)); + while byte > 0 && !document.text.is_char_boundary(paragraph_start + byte) { + byte -= 1; } + byte } +// Per-keystroke hot path: a single paragraph's text length changed (paragraph +// COUNT is unchanged). §perf-heaven T8.6: `byte_range` is now DERIVED (the block +// tree's `paragraph_start` prefix-sum from run lengths + `paragraph_text_len`), +// so there is NO cached absolute offset to shift across the tail — the former +// O(tail) `im::Vector` + block-tree COW (~3.8 MB/keystroke on a 6k-para doc) is +// gone. The paragraph sequence already holds the edited runs; the only remaining +// work is mirroring the edited paragraph's content into its ONE block copy (the +// two representations hold separate `Paragraph` copies), found in O(log N) via +// the tree rank query; `update_at` refreshes the tree's paragraph-rope summary +// from the new run lengths. #[hotpath::measure] -pub fn rebuild_document_offset_index(document: &mut Document) { - document.offset_index.rebuild(&document.paragraphs); - refresh_paragraph_ranges(document); -} - -#[hotpath::measure] -pub fn update_paragraph_offsets_after_len_change(document: &mut Document, paragraph_ix: usize) { +pub fn update_paragraph_offsets_after_len_change(document: &mut DocumentProjection, paragraph_ix: usize) { if paragraph_ix >= document.paragraphs.len() { return; } - document - .offset_index - .update_paragraph_width(paragraph_ix, &document.paragraphs); - refresh_paragraph_ranges_from(document, paragraph_ix); - let paragraph_count = document.paragraphs.len(); - let replacements = document.paragraphs[paragraph_ix..].to_vec(); - replace_paragraph_blocks( - document, - paragraph_ix, - paragraph_count.saturating_sub(paragraph_ix), - &replacements, - ); + let source = document.paragraphs[paragraph_ix].clone(); + let Some(row) = crate::block_ix_for_paragraph(document, paragraph_ix) else { + return; + }; + document.blocks.update_at(row, |block| { + if let Block::Paragraph(paragraph) = block { + paragraph.clone_from(&source); + } + }); } // Returns `(run_index, local_byte)` for the given absolute byte offset within @@ -128,8 +134,12 @@ mod offsets_tests { insert_text_at(&mut document, 0, "alpha".len(), " beta", RunStyles::default()); - assert_eq!(document.paragraphs[1].byte_range, "alpha beta\n".len().."alpha beta\nKepe et al. ‘23".len()); - assert!(matches!(&document.blocks[1], Block::Paragraph(paragraph) if paragraph.byte_range == document.paragraphs[1].byte_range)); + // §perf-heaven T8.6: the byte range is now DERIVED from the block tree's + // `paragraph_start` prefix-sum. Verify the derive tracks the insert, and + // that the block-tree mirror's runs stayed in sync (which is what makes the + // derive correct). + assert_eq!(paragraph_byte_range(&document, 1), "alpha beta\n".len().."alpha beta\nKepe et al. ‘23".len()); + assert!(matches!(&document.blocks[1], Block::Paragraph(paragraph) if paragraph.runs == document.paragraphs[1].runs)); } } diff --git a/crates/gpui-flowtext/src/edit_ops/rich_fragment.rs b/crates/gpui-flowtext/src/edit_ops/rich_fragment.rs index 287f72a1..6c450648 100644 --- a/crates/gpui-flowtext/src/edit_ops/rich_fragment.rs +++ b/crates/gpui-flowtext/src/edit_ops/rich_fragment.rs @@ -1,5 +1,5 @@ #[hotpath::measure] -pub fn insert_rich_fragment_at(document: &mut Document, offset: DocumentOffset, fragment: &RichClipboardFragment) -> DocumentOffset { +pub fn insert_rich_fragment_at(document: &mut DocumentProjection, offset: DocumentOffset, fragment: &RichClipboardFragment) -> DocumentOffset { let Some(first_paragraph) = fragment.paragraphs.first() else { return offset; }; @@ -10,7 +10,7 @@ pub fn insert_rich_fragment_at(document: &mut Document, offset: DocumentOffset, } #[hotpath::measure] -fn insert_single_paragraph_fragment_at(document: &mut Document, offset: DocumentOffset, paragraph: &InputParagraph) -> DocumentOffset { +fn insert_single_paragraph_fragment_at(document: &mut DocumentProjection, offset: DocumentOffset, paragraph: &InputParagraph) -> DocumentOffset { let text = input_paragraph_text(paragraph); if text.is_empty() { return offset; @@ -18,7 +18,7 @@ fn insert_single_paragraph_fragment_at(document: &mut Document, offset: Document let Some(target) = document.paragraphs.get(offset.paragraph) else { return offset; }; - let byte = offset.byte.min(paragraph_text_len(target)); + let byte = clamp_paragraph_byte_to_char_boundary(document, offset.paragraph, offset.byte); let paragraph_start = paragraph_byte_range(document, offset.paragraph).start; let (left_runs, right_runs) = split_runs_at(&target.runs, byte); document.text.insert(paragraph_start + byte, &text); @@ -26,12 +26,11 @@ fn insert_single_paragraph_fragment_at(document: &mut Document, offset: Document runs.extend(left_runs); runs.extend(input_paragraph_text_runs(paragraph)); runs.extend(right_runs); - { - let target = &mut paragraphs_mut(document)[offset.paragraph]; + if let Some(target) = paragraphs_mut(document).get_mut(offset.paragraph) { target.style = paragraph.style; target.runs = merge_adjacent_runs(runs); bump_paragraph_version(target); - }; + } update_paragraph_offsets_after_len_change(document, offset.paragraph); rebuild_document_sections(document); DocumentOffset { @@ -41,11 +40,11 @@ fn insert_single_paragraph_fragment_at(document: &mut Document, offset: Document } #[hotpath::measure] -fn insert_multi_paragraph_fragment_at(document: &mut Document, offset: DocumentOffset, fragment: &RichClipboardFragment) -> DocumentOffset { +fn insert_multi_paragraph_fragment_at(document: &mut DocumentProjection, offset: DocumentOffset, fragment: &RichClipboardFragment) -> DocumentOffset { let Some(target) = document.paragraphs.get(offset.paragraph).cloned() else { return offset; }; - let byte = offset.byte.min(paragraph_text_len(&target)); + let byte = clamp_paragraph_byte_to_char_boundary(document, offset.paragraph, offset.byte); let paragraph_start = paragraph_byte_range(document, offset.paragraph).start; let (left_runs, right_runs) = split_runs_at(&target.runs, byte); let inserted_text = fragment @@ -70,22 +69,17 @@ fn insert_multi_paragraph_fragment_at(document: &mut Document, offset: DocumentO } replacements.push(Paragraph { style: if ix == 0 && !left_runs.is_empty() { target.style } else { paragraph.style }, - byte_range: 0..0, runs: merge_adjacent_runs(runs), version: target.version.wrapping_add(1), }); } let replacement_count = replacements.len(); - { - let paragraphs = paragraphs_mut(document); - paragraphs.splice(offset.paragraph..=offset.paragraph, replacements) - }; + paragraphs_mut(document).splice(offset.paragraph..offset.paragraph + 1, replacements); for insert_ix in 1..replacement_count { insert_paragraph_id(document, offset.paragraph + insert_ix); } - rebuild_document_offset_index(document); - let block_replacements = document.paragraphs[offset.paragraph..offset.paragraph + replacement_count].to_vec(); + let block_replacements = document.paragraphs.range_to_vec(offset.paragraph..offset.paragraph + replacement_count); replace_paragraph_blocks(document, offset.paragraph, 1, &block_replacements); rebuild_document_sections(document); diff --git a/crates/gpui-flowtext/src/edit_ops/selection_styles.rs b/crates/gpui-flowtext/src/edit_ops/selection_styles.rs index 1ba0b21b..68bd530e 100644 --- a/crates/gpui-flowtext/src/edit_ops/selection_styles.rs +++ b/crates/gpui-flowtext/src/edit_ops/selection_styles.rs @@ -1,14 +1,23 @@ #[hotpath::measure] #[must_use] -pub fn selection_run_styles(document: &Document, range: Range) -> Vec { +pub fn selection_run_styles(document: &DocumentProjection, range: Range) -> Vec { let mut styles = Vec::new(); - for paragraph_ix in range.start.paragraph..=range.end.paragraph { + if document.paragraphs.is_empty() || range.start.paragraph >= document.paragraphs.len() { + return styles; + } + let last_paragraph = range.end.paragraph.min(document.paragraphs.len() - 1); + for paragraph_ix in range.start.paragraph..=last_paragraph { let paragraph = &document.paragraphs[paragraph_ix]; - let start = if paragraph_ix == range.start.paragraph { range.start.byte } else { 0 }; + let paragraph_len = paragraph_text_len(paragraph); + let start = if paragraph_ix == range.start.paragraph { + range.start.byte.min(paragraph_len) + } else { + 0 + }; let end = if paragraph_ix == range.end.paragraph { - range.end.byte + range.end.byte.min(paragraph_len) } else { - paragraph_text_len(paragraph) + paragraph_len }; let mut offset = 0; for run in ¶graph.runs { @@ -25,20 +34,57 @@ pub fn selection_run_styles(document: &Document, range: Range) - #[hotpath::measure] #[must_use] -pub fn selection_prefers_direct_underline(document: &Document, range: Range) -> bool { - (range.start.paragraph..=range.end.paragraph) +pub fn selection_prefers_direct_underline(document: &DocumentProjection, range: Range) -> bool { + if document.paragraphs.is_empty() || range.start.paragraph >= document.paragraphs.len() { + return false; + } + let last_paragraph = range.end.paragraph.min(document.paragraphs.len() - 1); + (range.start.paragraph..=last_paragraph) .any(|paragraph_ix| matches!(document.paragraphs[paragraph_ix].style, ParagraphStyle::Custom(3) | ParagraphStyle::Custom(4))) } +// §perf: scan the intersecting runs directly with an early exit instead of +// allocating a Vec via selection_run_styles just to fold it away. +// This feeds toolbar/formatting state that is recomputed on every selection change. #[hotpath::measure] -pub fn selection_all_run_styles(document: &Document, range: Range, predicate: impl Fn(RunStyles) -> bool) -> bool { - let styles = selection_run_styles(document, range); - !styles.is_empty() && styles.into_iter().all(predicate) +pub fn selection_all_run_styles(document: &DocumentProjection, range: Range, predicate: impl Fn(RunStyles) -> bool) -> bool { + if document.paragraphs.is_empty() || range.start.paragraph >= document.paragraphs.len() { + return false; + } + let last_paragraph = range.end.paragraph.min(document.paragraphs.len() - 1); + let mut saw_any = false; + for paragraph_ix in range.start.paragraph..=last_paragraph { + let paragraph = &document.paragraphs[paragraph_ix]; + let paragraph_len = paragraph_text_len(paragraph); + let start = if paragraph_ix == range.start.paragraph { + range.start.byte.min(paragraph_len) + } else { + 0 + }; + let end = if paragraph_ix == range.end.paragraph { + range.end.byte.min(paragraph_len) + } else { + paragraph_len + }; + let mut offset = 0; + for run in ¶graph.runs { + let run_start = offset; + let run_end = offset + run.len; + offset = run_end; + if run_start < end && run_end > start { + saw_any = true; + if !predicate(run.styles) { + return false; + } + } + } + } + saw_any } #[hotpath::measure] #[must_use] -pub fn selection_all_underline_kind(document: &Document, range: Range, direct: bool) -> bool { +pub fn selection_all_underline_kind(document: &DocumentProjection, range: Range, direct: bool) -> bool { selection_all_run_styles(document, range, |styles| { if direct { styles.direct_underline @@ -50,20 +96,29 @@ pub fn selection_all_underline_kind(document: &Document, range: Range) -> bool { - (range.start.paragraph..=range.end.paragraph).any(|paragraph_ix| { - let start = if paragraph_ix == range.start.paragraph { range.start.byte } else { 0 }; +pub fn selection_contains_whole_paragraph(document: &DocumentProjection, range: Range) -> bool { + if document.paragraphs.is_empty() || range.start.paragraph >= document.paragraphs.len() { + return false; + } + let last_paragraph = range.end.paragraph.min(document.paragraphs.len() - 1); + (range.start.paragraph..=last_paragraph).any(|paragraph_ix| { + let paragraph_len = paragraph_text_len(&document.paragraphs[paragraph_ix]); + let start = if paragraph_ix == range.start.paragraph { + range.start.byte.min(paragraph_len) + } else { + 0 + }; let end = if paragraph_ix == range.end.paragraph { - range.end.byte + range.end.byte.min(paragraph_len) } else { - paragraph_text_len(&document.paragraphs[paragraph_ix]) + paragraph_len }; - start == 0 && end == paragraph_text_len(&document.paragraphs[paragraph_ix]) + start == 0 && end == paragraph_len }) } #[hotpath::measure] -pub fn clear_whole_paragraph_formatting(document: &mut Document, paragraph_ix: usize) { +pub fn clear_whole_paragraph_formatting(document: &mut DocumentProjection, paragraph_ix: usize) { let Some(paragraph) = paragraphs_mut(document).get_mut(paragraph_ix) else { return; }; diff --git a/crates/gpui-flowtext/src/edit_ops/split_delete.rs b/crates/gpui-flowtext/src/edit_ops/split_delete.rs index 57314c4f..d2fbeee3 100644 --- a/crates/gpui-flowtext/src/edit_ops/split_delete.rs +++ b/crates/gpui-flowtext/src/edit_ops/split_delete.rs @@ -37,39 +37,39 @@ pub fn split_runs_at(runs: &[TextRun], byte: usize) -> (Vec, Vec) { +pub fn delete_cross_paragraph_range(document: &mut DocumentProjection, range: Range) { if range.start.paragraph >= range.end.paragraph { delete_range_in_paragraph(document, range.start.paragraph, range.start.byte..range.end.byte); return; @@ -77,29 +77,31 @@ pub fn delete_cross_paragraph_range(document: &mut Document, range: Range, style: RunStyle) { +pub fn apply_style_to_paragraph_range(document: &mut DocumentProjection, paragraph_ix: usize, range: Range, style: RunStyle) { if range.start >= range.end { return; } @@ -52,22 +52,30 @@ pub fn apply_style_to_paragraph_range(document: &mut Document, paragraph_ix: usi } } +// §perf: compact the runs in place (write index never exceeds the read index) +// so the caller's existing allocation is reused instead of allocating a second +// Vec and freeing the input. This runs on nearly every text edit +// (insert/delete/style/split), so it saves one alloc+free per edit op. #[hotpath::measure] #[must_use] -pub fn merge_adjacent_runs(runs: Vec) -> Vec { - let mut merged: Vec = Vec::with_capacity(runs.len()); - for run in runs { - if run.len == 0 { +pub fn merge_adjacent_runs(mut runs: Vec) -> Vec { + let mut write = 0usize; + let mut read = 0usize; + while read < runs.len() { + let len = runs[read].len; + let styles = runs[read].styles; + read += 1; + if len == 0 { continue; } - if let Some(last) = merged.last_mut() - && last.styles == run.styles - { - last.len += run.len; + if write > 0 && runs[write - 1].styles == styles { + runs[write - 1].len += len; continue; } - merged.push(run); + runs[write] = TextRun { len, styles }; + write += 1; } - merged + runs.truncate(write); + runs } diff --git a/crates/gpui-flowtext/src/edit_ops/text.rs b/crates/gpui-flowtext/src/edit_ops/text.rs index e87f622a..eab57464 100644 --- a/crates/gpui-flowtext/src/edit_ops/text.rs +++ b/crates/gpui-flowtext/src/edit_ops/text.rs @@ -1,10 +1,11 @@ #[hotpath::measure] #[must_use] -pub fn paragraph_text(document: &Document, paragraph_ix: usize) -> String { +pub fn paragraph_text(document: &DocumentProjection, paragraph_ix: usize) -> String { document_text_slice(document, paragraph_byte_range(document, paragraph_ix)) } -#[hotpath::measure] +// §perf: not hotpath-measured — see `paragraph_runs_len`. +#[inline] #[must_use] pub fn paragraph_text_len(paragraph: &Paragraph) -> usize { paragraph_runs_len(paragraph) @@ -12,7 +13,7 @@ pub fn paragraph_text_len(paragraph: &Paragraph) -> usize { #[hotpath::measure] #[must_use] -pub fn document_text_slice(document: &Document, range: Range) -> String { +pub fn document_text_slice(document: &DocumentProjection, range: Range) -> String { let len = document.text.byte_len(); let start = range.start.min(len); let end = range.end.min(len); @@ -25,7 +26,7 @@ pub fn document_text_slice(document: &Document, range: Range) -> String { } #[hotpath::measure] -pub fn push_document_text_slice(document: &Document, range: Range, text: &mut String) { +pub fn push_document_text_slice(document: &DocumentProjection, range: Range, text: &mut String) { let len = document.text.byte_len(); let start = range.start.min(len); let end = range.end.min(len); @@ -39,13 +40,13 @@ pub fn push_document_text_slice(document: &Document, range: Range, text: #[hotpath::measure] #[must_use] -pub fn paragraph_char_count(document: &Document, paragraph_ix: usize, needle: char) -> usize { +pub fn paragraph_char_count(document: &DocumentProjection, paragraph_ix: usize, needle: char) -> usize { document_text_slice_char_count(document, paragraph_byte_range(document, paragraph_ix), needle) } #[hotpath::measure] #[must_use] -pub fn document_text_slice_char_count(document: &Document, range: Range, needle: char) -> usize { +pub fn document_text_slice_char_count(document: &DocumentProjection, range: Range, needle: char) -> usize { document .text .byte_slice(range) @@ -56,7 +57,7 @@ pub fn document_text_slice_char_count(document: &Document, range: Range, #[hotpath::measure] #[must_use] -pub fn capture_document_span(document: &Document, range: Range) -> DocumentSpan { +pub fn capture_document_span(document: &DocumentProjection, range: Range) -> DocumentSpan { let start = range.start.min(document.paragraphs.len()); let end = range.end.min(document.paragraphs.len()).max(start); let text = if start < end { @@ -65,15 +66,38 @@ pub fn capture_document_span(document: &Document, range: Range) -> Docume } else { String::new() }; + // Capture the durable ids the span currently owns (one paragraph id and one + // paragraph-block id per paragraph) so a later replacement can force the + // optimistic, predicted, and canonical apply to preserve the same identities. + let paragraph_ids = (start..end) + .filter_map(|paragraph_ix| document.ids.paragraph_ids.get(paragraph_ix).copied()) + .collect::>(); + let block_ids = (start..end) + .filter_map(|paragraph_ix| { + block_ix_for_paragraph(document, paragraph_ix).and_then(|block_ix| document.ids.block_ids.get(block_ix).copied()) + }) + .collect::>(); DocumentSpan { start_paragraph: start, - paragraphs: document.paragraphs[start..end].to_vec(), + paragraphs: document.paragraphs.range_to_vec(start..end), + paragraph_ids, + block_ids, text, } } #[hotpath::measure] -pub fn apply_document_span_replacement(document: &mut Document, current: &DocumentSpan, replacement: &DocumentSpan) { +pub fn apply_document_span_replacement(document: &mut DocumentProjection, current: &DocumentSpan, replacement: &DocumentSpan) { + debug_assert_eq!( + replacement.paragraph_ids.len(), + replacement.paragraphs.len(), + "DocumentSpan replacement must carry exactly one paragraph id per paragraph", + ); + debug_assert_eq!( + replacement.block_ids.len(), + replacement.paragraphs.len(), + "DocumentSpan replacement must carry exactly one block id per paragraph", + ); let byte_range = paragraph_span_byte_range(document, current.start_paragraph, current.paragraphs.len()); document.text.delete(byte_range.clone()); document.text.insert(byte_range.start, &replacement.text); @@ -81,24 +105,35 @@ pub fn apply_document_span_replacement(document: &mut Document, current: &Docume .start_paragraph .saturating_add(current.paragraphs.len()) .min(document.paragraphs.len()); - remove_paragraph_ids(document, current.start_paragraph..paragraph_end); + let before_count = paragraph_end.saturating_sub(current.start_paragraph); + // Splice the editor-captured durable ids verbatim rather than re-deriving them + // positionally (keep-first/mint-rest). Using the exact ids the command carried + // is what keeps this optimistic replay, the runtime prediction, and the + // canonical Loro apply from disagreeing on which paragraph/block id survives a + // merge (e.g. a join dropping a middle boundary) — a disagreement that would + // strand later pending edits referencing the dropped id and lose text. + let paragraph_id_end = paragraph_end.min(document.ids.paragraph_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.paragraph_ids) + .splice(current.start_paragraph.min(paragraph_id_end)..paragraph_id_end, replacement.paragraph_ids.iter().copied()); paragraphs_mut(document).splice(current.start_paragraph..paragraph_end, replacement.paragraphs.clone()); - for relative_ix in 0..replacement.paragraphs.len() { - insert_paragraph_id(document, current.start_paragraph + relative_ix); + replace_paragraph_blocks(document, current.start_paragraph, before_count, &replacement.paragraphs); + // `replace_paragraph_blocks` derives the span's block ids positionally; overwrite + // them with the editor-captured block ids so block identities match canonical + // too. The replacement paragraph-blocks are contiguous from `block_start`. + if let Some(block_start) = block_ix_for_paragraph(document, current.start_paragraph) { + let block_end = block_start + .saturating_add(replacement.block_ids.len()) + .min(document.ids.block_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.block_ids) + .splice(block_start.min(block_end)..block_end, replacement.block_ids.iter().copied()); } - replace_paragraph_blocks( - document, - current.start_paragraph, - paragraph_end.saturating_sub(current.start_paragraph), - &replacement.paragraphs, - ); - rebuild_document_offset_index(document); - rebuild_document_sections(document); + // `replace_paragraph_blocks` already rebuilt the section outline; only the + // byte-offset index still needs refreshing after the splice. } #[hotpath::measure] #[must_use] -pub fn paragraph_span_byte_range(document: &Document, start_paragraph: usize, paragraph_count: usize) -> Range { +pub fn paragraph_span_byte_range(document: &DocumentProjection, start_paragraph: usize, paragraph_count: usize) -> Range { if paragraph_count == 0 || start_paragraph >= document.paragraphs.len() { let byte = document .paragraphs @@ -118,12 +153,12 @@ pub fn paragraph_span_byte_range(document: &Document, start_paragraph: usize, pa )] #[hotpath::measure] #[must_use] -pub fn full_document_text(document: &Document) -> String { +pub fn full_document_text(document: &DocumentProjection) -> String { document_text_slice(document, 0..document.text.byte_len()) } #[hotpath::measure] -pub fn document_end(document: &Document) -> DocumentOffset { +pub fn document_end(document: &DocumentProjection) -> DocumentOffset { let paragraph = document.paragraphs.len().saturating_sub(1); DocumentOffset { paragraph, @@ -140,14 +175,14 @@ pub fn document_end(document: &Document) -> DocumentOffset { )] #[hotpath::measure] #[must_use] -pub fn global_byte(document: &Document, offset: DocumentOffset) -> usize { +pub fn global_byte(document: &DocumentProjection, offset: DocumentOffset) -> usize { paragraph_byte_range(document, offset.paragraph).start + offset.byte } #[allow(dead_code, reason = "Global-to-document offset conversion is retained for file/search integrations.")] #[hotpath::measure] #[must_use] -pub fn global_to_document_offset(document: &Document, byte: usize) -> DocumentOffset { +pub fn global_to_document_offset(document: &DocumentProjection, byte: usize) -> DocumentOffset { let byte = byte.min(document.text.byte_len()); let mut low = 0; let mut high = document.paragraphs.len(); @@ -172,42 +207,45 @@ pub fn global_to_document_offset(document: &Document, byte: usize) -> DocumentOf #[hotpath::measure] #[must_use] -pub fn find_text_ranges(document: &Document, query: &str) -> Vec> { +pub fn find_text_ranges(document: &DocumentProjection, query: &str) -> Vec> { find_text_ranges_with_case(document, query, true) } #[hotpath::measure] #[must_use] -pub fn find_text_ranges_with_case(document: &Document, query: &str, case_sensitive: bool) -> Vec> { +pub fn find_text_ranges_with_case(document: &DocumentProjection, query: &str, case_sensitive: bool) -> Vec> { find_text_ranges_with_options(document, query, case_sensitive, false) } #[hotpath::measure] #[must_use] -pub fn find_text_ranges_with_options(document: &Document, query: &str, case_sensitive: bool, whole_words: bool) -> Vec> { +pub fn find_text_ranges_with_options(document: &DocumentProjection, query: &str, case_sensitive: bool, whole_words: bool) -> Vec> { if query.is_empty() { return Vec::new(); } let text = full_document_text(document); - let matches = if case_sensitive { + // §perf: filter+map the match_indices iterator straight into the result instead of + // collecting a throwaway intermediate Vec and re-iterating it. ASCII-lowercase + // preserves byte offsets, so whole-word checks still use the original `text`. + let to_offsets = + |range: Range| global_to_document_offset(document, range.start)..global_to_document_offset(document, range.end); + if case_sensitive { text .match_indices(query) .map(|(start, matched)| start..start + matched.len()) - .collect::>() + .filter(|range| !whole_words || is_whole_word_match(&text, range.clone())) + .map(to_offsets) + .collect() } else { let lower_text = text.to_ascii_lowercase(); let lower_query = query.to_ascii_lowercase(); lower_text .match_indices(lower_query.as_str()) .map(|(start, matched)| start..start + matched.len()) - .collect::>() - }; - - matches - .into_iter() - .filter(|range| !whole_words || is_whole_word_match(&text, range.clone())) - .map(|range| global_to_document_offset(document, range.start)..global_to_document_offset(document, range.end)) - .collect() + .filter(|range| !whole_words || is_whole_word_match(&text, range.clone())) + .map(to_offsets) + .collect() + } } #[hotpath::measure] @@ -233,7 +271,7 @@ fn next_char_is_word_like(text: &str, byte: usize) -> bool { #[hotpath::measure] #[must_use] -pub fn selected_plain_text(document: &Document, range: Range) -> String { +pub fn selected_plain_text(document: &DocumentProjection, range: Range) -> String { if document.paragraphs.is_empty() { return String::new(); } diff --git a/crates/gpui-flowtext/src/lib.rs b/crates/gpui-flowtext/src/lib.rs index ecf63458..fab0bced 100644 --- a/crates/gpui-flowtext/src/lib.rs +++ b/crates/gpui-flowtext/src/lib.rs @@ -1,23 +1,27 @@ mod api; +pub mod block_tree; mod collaboration; mod demo; mod document; mod edit_ops; +pub mod local_intents; mod persistence; mod rich_text; pub use api::*; +pub use block_tree::{BlockTree, Summary}; pub use collaboration::*; pub use demo::*; pub use document::*; pub use edit_ops::*; +pub use local_intents::*; pub use persistence::*; pub use rich_text::*; pub mod prelude { pub use crate::{ - Document, DocumentTheme, EditorSelection, HighlightStyle, Paragraph, ParagraphStyle, RichTextDocumentElement, RichTextEditor, - RichTextEditorCommand, RunSemanticStyle, RunStyle, RunStyles, TextRun, + DocumentProjection, DocumentTheme, EditorSelection, HighlightStyle, Paragraph, ParagraphStyle, RichTextDocumentElement, RichTextEditor, + RichTextEditorCommand, RunSemanticStyle, RunStyle, RunStyles, TextRun, VertAlign, }; } @@ -35,17 +39,10 @@ pub mod editor_api { }; } -pub mod persistence_api { - pub use crate::{ - DEFAULT_DOCUMENT_EXTENSION, document_bytes, load_or_create_document, read_document, read_document_bytes, recovery_path_for_document, - write_document, - }; -} - pub mod host { pub use crate::{ - AssetResolver, BlockKindId, DocumentExportAdapter, DocumentExportFormat, DocumentSerializer, ExternalFormatExporter, - set_document_export_adapter, + AssetResolver, BlockKindId, DocumentExportAdapter, DocumentExportFormat, DocumentRecoveryAdapter, DocumentSerializer, + ExternalFormatExporter, set_document_export_adapter, set_document_recovery_adapter, }; } @@ -53,19 +50,3 @@ pub mod advanced { pub use crate::collaboration::*; pub use crate::edit_ops::*; } - -use std::time::Instant; - -const TIMING_ENV: &str = "DEBATEPROCESSOR_TIMING"; - -#[hotpath::measure] -fn timing_enabled() -> bool { - std::env::var_os(TIMING_ENV).is_some() -} - -#[hotpath::measure] -pub(crate) fn log_timing_lazy(label: &str, start: Instant, detail: impl FnOnce() -> String) { - if timing_enabled() { - eprintln!("[timing] {label}: {:?} {}", start.elapsed(), detail()); - } -} diff --git a/crates/gpui-flowtext/src/local_intents.rs b/crates/gpui-flowtext/src/local_intents.rs new file mode 100644 index 00000000..a083f61f --- /dev/null +++ b/crates/gpui-flowtext/src/local_intents.rs @@ -0,0 +1,572 @@ +//! Typed local-write intents (spec §4, D2). +//! +//! An intent is the ONLY way editing enters the document. Each intent carries +//! stable identity (paragraph/block/row/column ids, optionally an encoded Loro +//! cursor) plus raw offsets strictly as *hints*. The write path resolves +//! identity against live Loro state inside the write gate before any mutation; +//! if identity cannot be resolved the intent is rejected before the doc is +//! touched (spec I-2, I-15). Raw projection offsets are never authority. +//! +//! One intent = one Loro commit = one undo-group member (spec §4). Intents are +//! deliberately plain data (no Loro types beyond encoded cursor bytes) so the +//! editor crate can construct them without depending on `loro`. + +use std::ops::Range; + +use crate::{ + BlockId, ColumnId, DocumentOffset, DocumentProjection, EditorSelection, InputBlock, InputBlockAlignment, InputImageSizing, InputParagraph, + InputTableCell, InputTableColumnWidth, InputTableRow, ParagraphId, ParagraphStyle, ProjectionPatchBatch, RowId, RunStyles, SelectionAffinity, + VisualGravity, +}; + +/// A position in body text addressed by stable identity. +/// +/// Resolution law (spec §4): the encoded `cursor` (when present) is the +/// preferred basis and wins over `byte_hint`; otherwise the paragraph id +/// resolves via its durable paragraph record and `byte_hint` is clamped into +/// the resolved paragraph's current range. `byte_hint` is a UTF-8 byte offset +/// within the paragraph's text (projection space); it is never applied as a +/// raw document index. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TextAnchor { + pub paragraph: ParagraphId, + pub byte_hint: usize, + /// Encoded `loro::cursor::Cursor` captured earlier (e.g. at composition + /// start or selection time). Optional; identity resolution works without it. + pub cursor: Option>, +} + +impl TextAnchor { + #[must_use] + pub fn new(paragraph: ParagraphId, byte_hint: usize) -> Self { + Self { + paragraph, + byte_hint, + cursor: None, + } + } + + #[must_use] + pub fn with_cursor(mut self, cursor: Vec) -> Self { + self.cursor = Some(cursor); + self + } +} + +/// Insert plain text at an anchored caret position. +/// +/// Style inheritance is Loro's job (expand-`After` run marks, spec §9): a +/// plain insert emits NO style operations. `style_override` is present only +/// when the caret's style state differs from what inheritance would produce +/// (e.g. bold toggled off at the end of a bold run) and marks exactly the +/// inserted range. +#[derive(Clone, Debug)] +pub struct InsertTextIntent { + pub at: TextAnchor, + pub text: String, + pub style_override: Option, +} + +/// Delete the anchored range. Marks collapse naturally; no style ops. +#[derive(Clone, Debug)] +pub struct DeleteRangeIntent { + pub start: TextAnchor, + pub end: TextAnchor, +} + +/// Split the anchored paragraph at the anchor position. New paragraph/block +/// identities are minted by the write path (not the editor) and surface to the +/// UI through the returned projection patches. +/// +/// Sentinel hygiene (spec §9): the split unmarks all run-style keys on the +/// inserted `\n` so expand-`After` marks never bleed across the paragraph +/// boundary; cross-split style continuation is the caret-style-override path. +#[derive(Clone, Debug)] +pub struct SplitParagraphIntent { + pub at: TextAnchor, + pub inherited_style: ParagraphStyle, +} + +/// Join `second` into `first` (both by durable identity). +#[derive(Clone, Debug)] +pub struct JoinParagraphsIntent { + pub first: ParagraphId, + pub second: ParagraphId, +} + +/// Apply run styles over an anchored range (minimal mark change over the +/// changed range only — never a run-list restatement, spec §9). +#[derive(Clone, Debug)] +pub struct SetMarksIntent { + pub start: TextAnchor, + pub end: TextAnchor, + pub styles: RunStyles, +} + +/// Set a paragraph's style mark (anchored on its boundary sentinel). +#[derive(Clone, Debug)] +pub struct SetParagraphStyleIntent { + pub paragraph: ParagraphId, + pub style: ParagraphStyle, +} + +/// Set MANY paragraphs to one style in a single commit — one gate hold, one +/// Loro commit, one undo member, one projection patch batch (§11 +/// anti-amplification). A selection-wide restyle previously amplified into one +/// intent per paragraph: N gate holds, N commits, N stream drains (measured +/// 64.7s for a select-all restyle on the reference doc), and an undo group +/// whose N members fragmented Loro undo into N full-state checkouts. +/// Stale or non-representable targets (object-following boundary-less rows) +/// are skipped, not rejected — the surviving targets carry the user's intent. +#[derive(Clone, Debug)] +pub struct SetParagraphStylesIntent { + pub paragraphs: Vec, + pub style: ParagraphStyle, +} + +/// Insert an object block (image/equation/table/…) at an anchored position. +/// The block's durable identity is minted by the write path. +#[derive(Clone, Debug)] +pub struct InsertObjectIntent { + pub at: TextAnchor, + pub block: InputBlock, +} + +/// Replace an object block's content in place (same identity). +#[derive(Clone, Debug)] +pub struct ReplaceObjectIntent { + pub block: BlockId, + pub after: InputBlock, +} + +/// Delete object blocks by identity. +#[derive(Clone, Debug)] +pub struct DeleteBlocksIntent { + pub blocks: Vec, +} + +/// Move an object block so it sits immediately before `before` (or at the end +/// of the document when `None`). Identity-anchored — never a raw index. +#[derive(Clone, Debug)] +pub struct MoveBlockIntent { + pub block: BlockId, + pub before: Option, +} + +/// One block of a rich pasted fragment. +#[derive(Clone, Debug)] +pub enum FragmentBlock { + Paragraph(InputParagraph), + Object(InputBlock), +} + +/// Insert a rich multi-block fragment (paste) at an anchored position. This is +/// a compound intent: it still executes as one gate hold and one Loro commit, +/// and a mid-apply failure is compensated per spec I-10. +#[derive(Clone, Debug)] +pub struct InsertRichFragmentIntent { + pub at: TextAnchor, + pub blocks: Vec, +} + +/// One same-paragraph find/replace match: an anchored range plus the styles +/// the replacement should carry (the styles at the match start in the +/// editor's projection — the mirror of `InsertTextIntent::style_override`, +/// so a match inside a styled run keeps its styling even at paragraph start +/// where expand-`After` inheritance has no preceding run character). +#[derive(Clone, Debug)] +pub struct ReplaceMatch { + pub start: TextAnchor, + pub end: TextAnchor, + pub styles: Option, +} + +/// Replace every anchored same-paragraph match with `replacement` (find & +/// replace). Compound intent: ONE gate hold and ONE Loro commit regardless of +/// match count — a replace-all must never become a per-match commit storm +/// (§11 complexity law; the select-all amplification class). The write path +/// applies ranges back-to-front so positions never shift; matches whose +/// anchors resolve across a paragraph boundary, collapse, or overlap a +/// lower match (concurrent edits moved them) are skipped, not rejected — the +/// surviving matches still carry the user's intent. Cross-paragraph matches +/// take the selection + `InsertText` path in the editor instead. +/// `replacement` must not contain structural characters; it may be empty +/// (replace with nothing = delete matches). +#[derive(Clone, Debug)] +pub struct ReplaceMatchesIntent { + pub matches: Vec, + pub replacement: String, +} + +/// Equation source edit (identity + intra-source byte range). +#[derive(Clone, Debug)] +pub struct ReplaceEquationSourceRangeIntent { + pub equation: BlockId, + pub range: Range, + pub text: String, +} + +/// Image metadata/layout intents (identity-based). +#[derive(Clone, Debug)] +pub struct ReplaceImageAltTextIntent { + pub image: BlockId, + pub text: String, +} + +#[derive(Clone, Debug)] +pub struct ReplaceImageCaptionIntent { + pub image: BlockId, + pub caption: Option, +} + +#[derive(Clone, Debug)] +pub struct SetImageLayoutIntent { + pub image: BlockId, + pub sizing: InputImageSizing, + pub alignment: InputBlockAlignment, +} + +/// Table intents — durable row/column/cell identities throughout (§P2b). +/// `InsertTableRow`/`InsertTableColumn` identities are minted by the write +/// path; `after_*` anchors falling back deterministically to the tail when the +/// anchor was concurrently deleted (existing convergent behavior). +#[derive(Clone, Debug)] +pub enum TableIntent { + InsertRow { + table: BlockId, + after_row: Option, + row: InputTableRow, + }, + DeleteRow { + table: BlockId, + row: RowId, + }, + MoveRow { + table: BlockId, + row: RowId, + after_row: Option, + }, + InsertColumn { + table: BlockId, + after_column: Option, + width: InputTableColumnWidth, + }, + DeleteColumn { + table: BlockId, + column: ColumnId, + }, + MoveColumn { + table: BlockId, + column: ColumnId, + after_column: Option, + }, + ReplaceCell { + table: BlockId, + row: RowId, + column: ColumnId, + cell: InputTableCell, + }, + SetCellSpan { + table: BlockId, + row: RowId, + column: ColumnId, + row_span: u16, + column_span: u16, + }, + SetColumnWidth { + table: BlockId, + column: ColumnId, + width: InputTableColumnWidth, + }, +} + +/// The complete intent vocabulary. Single dispatch point for the write path, +/// fuzzing, and counters; the per-intent structs above are the public API +/// surface behind [`LocalWriteAuthority`]. +#[derive(Clone, Debug)] +pub enum LocalIntent { + InsertText(InsertTextIntent), + DeleteRange(DeleteRangeIntent), + SplitParagraph(SplitParagraphIntent), + JoinParagraphs(JoinParagraphsIntent), + SetMarks(SetMarksIntent), + SetParagraphStyle(SetParagraphStyleIntent), + SetParagraphStyles(SetParagraphStylesIntent), + InsertObject(InsertObjectIntent), + ReplaceObject(ReplaceObjectIntent), + DeleteBlocks(DeleteBlocksIntent), + MoveBlock(MoveBlockIntent), + InsertRichFragment(InsertRichFragmentIntent), + ReplaceMatches(ReplaceMatchesIntent), + ReplaceEquationSourceRange(ReplaceEquationSourceRangeIntent), + ReplaceImageAltText(ReplaceImageAltTextIntent), + ReplaceImageCaption(ReplaceImageCaptionIntent), + SetImageLayout(SetImageLayoutIntent), + Table(TableIntent), +} + +impl LocalIntent { + /// Intent class label, stamped onto the Loro commit message for field + /// forensics (spec §16, pre-commit attribution) and used by counters. + #[must_use] + pub fn class(&self) -> &'static str { + match self { + Self::InsertText(_) => "insert-text", + Self::DeleteRange(_) => "delete-range", + Self::SplitParagraph(_) => "split-paragraph", + Self::JoinParagraphs(_) => "join-paragraphs", + Self::SetMarks(_) => "set-marks", + Self::SetParagraphStyle(_) => "set-paragraph-style", + Self::SetParagraphStyles(_) => "set-paragraph-styles", + Self::InsertObject(_) => "insert-object", + Self::ReplaceObject(_) => "replace-object", + Self::DeleteBlocks(_) => "delete-blocks", + Self::MoveBlock(_) => "move-block", + Self::InsertRichFragment(_) => "insert-rich-fragment", + Self::ReplaceMatches(_) => "replace-matches", + Self::ReplaceEquationSourceRange(_) => "replace-equation-source", + Self::ReplaceImageAltText(_) => "replace-image-alt-text", + Self::ReplaceImageCaption(_) => "replace-image-caption", + Self::SetImageLayout(_) => "set-image-layout", + Self::Table(_) => "table-op", + } + } + + /// Compound intents may touch several containers; a mid-apply failure in one + /// triggers the I-10 compensation path. Single-mutation intents cannot fail + /// mid-apply after resolution succeeds. + #[must_use] + pub fn is_compound(&self) -> bool { + matches!(self, Self::InsertRichFragment(_) | Self::ReplaceMatches(_) | Self::SetParagraphStyles(_)) + || matches!(self, Self::Table(TableIntent::InsertColumn { .. })) + } +} + +/// A selection endpoint in authoritative (cursor-backed) form plus its +/// render-space offset at capture time (spec §8, D5). +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CursorEndpoint { + /// Encoded `loro::cursor::Cursor` anchored per the §8 anchor-character rule. + pub cursor: Vec, + /// App-side interpretation delta: caret index = resolved anchor index + + /// `delta` (0 = right-sticky "at anchor", 1 = left-sticky "after anchor"). + pub delta: u8, + pub affinity: SelectionAffinity, + pub gravity: VisualGravity, + /// Render-space offset valid for the projection this endpoint was produced + /// against. Never authority; recomputed on resolution. + pub offset: DocumentOffset, +} + +/// Authoritative selection: a cursor pair (spec I-12). +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SelectionSnapshot { + pub anchor: CursorEndpoint, + pub head: CursorEndpoint, +} + +/// Why a local intent was rejected before mutation (spec I-15). The editor +/// restores selection from the nearest valid cursor, surfaces a recovery +/// notice, and never retries unchanged. +#[derive(Clone, Debug)] +pub enum WriteRejected { + /// The paragraph identity no longer exists in canonical state. + UnresolvedParagraph(ParagraphId), + /// The block identity no longer exists in canonical state. + UnresolvedBlock(BlockId), + /// A table row/column/cell identity no longer exists. + UnresolvedTableEntity { table: BlockId, detail: String }, + /// The supplied cursor could not be decoded or resolved. + UnresolvedCursor, + /// The intent is a no-op (empty text, empty range, empty fragment). + EmptyIntent, + /// The intent would violate document structure (e.g. delete the boundary-0 + /// sentinel, join across a non-adjacent pair). + StructureViolation(&'static str), + /// The write gate is poisoned: a panic occurred while the doc was held. + /// Reload from persisted state (spec I-10d). + GatePoisoned, + /// A mid-apply failure occurred and was compensated via `revert_to` + /// (spec I-10). The document converged back to its pre-intent state; the + /// diagnostic carries the underlying error chain. + CompensatedFailure { class: &'static str, diagnostic: String }, + /// Compensation itself failed. The core is wedged and must be reloaded; the + /// caller must treat the document as read-only until then. + CompensationFailed { class: &'static str, diagnostic: String }, +} + +impl std::fmt::Display for WriteRejected { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UnresolvedParagraph(id) => write!(f, "paragraph identity {} did not resolve against canonical state", id.0), + Self::UnresolvedBlock(id) => write!(f, "block identity {} did not resolve against canonical state", id.0), + Self::UnresolvedTableEntity { table, detail } => { + write!(f, "table {} entity did not resolve against canonical state: {detail}", table.0) + }, + Self::UnresolvedCursor => f.write_str("supplied cursor could not be decoded/resolved against canonical state"), + Self::EmptyIntent => f.write_str("intent is a no-op"), + Self::StructureViolation(reason) => write!(f, "intent violates document structure: {reason}"), + Self::GatePoisoned => f.write_str("write gate poisoned; reload from persisted state"), + Self::CompensatedFailure { class, diagnostic } => { + write!( + f, + "intent '{class}' failed mid-apply and was compensated back to pre-intent state: {diagnostic}" + ) + }, + Self::CompensationFailed { class, diagnostic } => { + write!( + f, + "intent '{class}' failed mid-apply AND compensation failed — core must be reloaded: {diagnostic}" + ) + }, + } + } +} + +impl std::error::Error for WriteRejected {} + +/// Always-on work counters for one committed intent (spec §11 — the +/// complexity-shape contract). Counts, not clocks: these assert O(touched) +/// behavior independent of hardware. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct IntentCounters { + /// Loro ops in the committed transaction (`ChangeMeta.len` via the + /// pre-commit subscription; the source of truth, immune to drift). + pub loro_ops: u64, + /// Distinct Loro containers mutated by the intent (counted at the mutation + /// call sites). + pub containers_touched: u32, + /// Style mark/unmark operations emitted. + pub marks_emitted: u32, + /// Projection patches synthesized. + pub patch_count: u32, + /// True when this intent fell back to a full projection rebuild — must be + /// rare and loud (spec I-14); test failure unless the op class requires it. + pub full_rebuild: bool, + /// Gate hold time (microseconds) for the whole intent. + pub gate_hold_micros: u64, +} + +/// The synchronous result of a committed local intent (spec §4). +#[derive(Clone, Debug)] +pub struct LocalCommit { + /// Exact patches for the UI read model, corresponding to this commit. + pub patches: ProjectionPatchBatch, + /// Encoded doc frontier after the commit. + pub frontier: Vec, + /// Encoded version vector after the commit. + pub version_vector: Vec, + /// Cursor-backed selection after the intent (caret placement authority). + pub selection_after: Option, + pub counters: IntentCounters, +} + +/// A full projection replacement produced when an intent legitimately requires +/// a rebuild (rare, loud) or by undo/redo/revision operations. +#[derive(Clone, Debug)] +pub struct ProjectionReplace { + pub document: DocumentProjection, + pub frontier: Vec, + pub version_vector: Vec, +} + +/// Outcome of an intent that may either patch or (rarely) replace the +/// projection. +#[derive(Clone, Debug)] +pub enum LocalWriteOutcome { + Committed(LocalCommit), + /// Committed, but the projection change could not be expressed as patches; + /// the full document replaces the UI copy. Counted via + /// `IntentCounters::full_rebuild` and the `full-rebuild-after-local-write` + /// metric. + CommittedWithRebuild { + commit: LocalCommit, + replace: Box, + }, +} + +impl LocalWriteOutcome { + #[must_use] + pub fn commit(&self) -> &LocalCommit { + match self { + Self::Committed(commit) => commit, + Self::CommittedWithRebuild { commit, .. } => commit, + } + } +} + +/// Outcome of an undo/redo executed through the collaboration runtime's +/// Loro `UndoManager` (spec §10). +#[derive(Debug)] +pub struct UndoOutcome { + /// Whether a step actually applied (`false` ⇒ empty undo/redo stack). The + /// projection change itself rides the ORDERED stream like every other + /// change; carrying a full projection here cost a wasted O(doc) clone on + /// every undo of a large document. + pub applied: bool, + /// Cursor-restored selection recorded in the undo item's metadata. + pub selection: Option, +} + +/// One item of the ORDERED projection stream (field fix 2026-07-07): every +/// editor-bound projection change — local intent patches, remote import +/// patches, full replaces from undo/rebuild — is queued on the document core +/// IN COMMIT ORDER under the write gate, and the editor drains it as its sole +/// projection input. Splitting delivery across a synchronous local channel and +/// an asynchronous session channel is what produced the base-frontier +/// mismatch cascade in the field logs; a single ordered stream makes it +/// structurally impossible. +#[derive(Clone, Debug)] +pub enum ProjectionStreamItem { + Patches(ProjectionPatchBatch), + Replace(Box), +} + +/// The ONE local write path (Loro-first spec §4, invariant 5), as seen from +/// the editor. The application injects an implementation backed by the +/// gate-protected document core (`LocalDocHandle`); solo and collaborative +/// documents receive the identical authority. Every call is synchronous: the +/// returned commit IS the committed state, and the patches are exact. +pub trait LocalWriteAuthority: Send + Sync { + fn apply(&self, intent: LocalIntent) -> Result; + fn undo(&self) -> Result; + fn redo(&self) -> Result; + /// Begin an undo group at an input-semantic boundary. Fallible by design: + /// a non-disjoint remote import closes the active group underneath the + /// editor, and the editor re-arms at the next boundary. + fn undo_group_start(&self) -> Result; + fn undo_group_end(&self) -> Result<(), WriteRejected>; + /// Drain the ordered projection stream (everything queued since the last + /// drain, in commit order). The editor is the single consumer. + fn drain_projection_stream(&self) -> Result, WriteRejected>; + /// Clone the canonical projection (attach + self-heal fallback). + fn canonical_projection(&self) -> Result; + /// Re-anchor a selection across a remote projection patch using CRDT cursors: + /// `selection` and `before` are the editor's PRE-patch selection and projection, + /// `editor_frontier` its pre-patch frontier. Returns the equivalent selection in + /// the current canonical state (so a concurrent insert/delete before the caret + /// repositions it), or `None` when it can't be resolved — the editor then falls + /// back to clamping. Default `None` keeps non-collab authorities trivial. + fn rebase_selection(&self, selection: &EditorSelection, before: &DocumentProjection, editor_frontier: &[u8]) -> Option { + let _ = (selection, before, editor_frontier); + None + } + /// FAST-path companion to [`Self::rebase_selection`]: encode the selection's + /// endpoints as CRDT cursors against the current (synced) state. The editor + /// stores these at synced moments and, when the caret hasn't moved, resolves + /// them via [`Self::resolve_selection_anchor`] after a remote patch — O(log n), + /// no `fork_at`. `(head_cursor, anchor_cursor)`. + /// `editor_frontier` is the editor's current projection frontier; the + /// authority must return `None` when the core has advanced past it (an + /// undrained remote import), so a re-arm racing that window can't encode the + /// caret against a body position the editor hasn't caught up to yet. + fn encode_selection_anchor(&self, selection: &EditorSelection, editor_frontier: &[u8]) -> Option<(Vec, Vec)> { + let _ = (selection, editor_frontier); + None + } + /// Resolve stored caret cursors to offsets in the current state. + fn resolve_selection_anchor(&self, head_cursor: &[u8], anchor_cursor: &[u8]) -> Option<(DocumentOffset, DocumentOffset)> { + let _ = (head_cursor, anchor_cursor); + None + } +} diff --git a/crates/gpui-flowtext/src/persistence/codec.rs b/crates/gpui-flowtext/src/persistence/codec.rs deleted file mode 100644 index 25fa1b8c..00000000 --- a/crates/gpui-flowtext/src/persistence/codec.rs +++ /dev/null @@ -1,226 +0,0 @@ -#[hotpath::measure] -fn read_u8(cursor: &mut Cursor<&[u8]>) -> io::Result { - let mut bytes = [0; 1]; - cursor.read_exact(&mut bytes)?; - Ok(bytes[0]) -} - -#[hotpath::measure] -fn read_u16(cursor: &mut Cursor<&[u8]>) -> io::Result { - let mut bytes = [0; 2]; - cursor.read_exact(&mut bytes)?; - Ok(u16::from_le_bytes(bytes)) -} - -#[hotpath::measure] -fn read_u32(cursor: &mut Cursor<&[u8]>) -> io::Result { - let mut bytes = [0; 4]; - cursor.read_exact(&mut bytes)?; - Ok(u32::from_le_bytes(bytes)) -} - -#[hotpath::measure] -fn write_u32(bytes: &mut Vec, value: u32) { - bytes.extend_from_slice(&value.to_le_bytes()); -} - -#[hotpath::measure] -fn read_u64(cursor: &mut Cursor<&[u8]>) -> io::Result { - let mut bytes = [0; 8]; - cursor.read_exact(&mut bytes)?; - Ok(u64::from_le_bytes(bytes)) -} - -#[hotpath::measure] -fn read_u128(cursor: &mut Cursor<&[u8]>) -> io::Result { - let mut bytes = [0; 16]; - cursor.read_exact(&mut bytes)?; - Ok(u128::from_le_bytes(bytes)) -} - -#[hotpath::measure] -fn write_u64(bytes: &mut Vec, value: u64) { - bytes.extend_from_slice(&value.to_le_bytes()); -} - -#[hotpath::measure] -fn write_u128(bytes: &mut Vec, value: u128) { - bytes.extend_from_slice(&value.to_le_bytes()); -} - -#[hotpath::measure] -fn read_len(cursor: &mut Cursor<&[u8]>, label: &'static str) -> io::Result { - let raw = read_u64(cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, format!("{label} overflows usize"))) -} - -#[hotpath::measure] -fn read_bytes<'bytes>(cursor: &mut Cursor<&'bytes [u8]>, len: usize, label: &'static str) -> io::Result<&'bytes [u8]> { - let start = usize::try_from(cursor.position()) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, format!("{label} cursor position overflows usize")))?; - let end = start - .checked_add(len) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("{label} length overflows usize")))?; - if end > cursor.get_ref().len() { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, format!("{label} is truncated"))); - } - cursor.set_position(end as u64); - Ok(&cursor.get_ref()[start..end]) -} - -#[hotpath::measure] -fn read_string(cursor: &mut Cursor<&[u8]>) -> io::Result { - let len = read_len(cursor, "native document string length")?; - let bytes = read_bytes(cursor, len, "native document string")?; - std::str::from_utf8(bytes) - .map(std::borrow::ToOwned::to_owned) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document string is not UTF-8")) -} - -#[hotpath::measure] -fn write_string(bytes: &mut Vec, value: &str) { - write_u64(bytes, value.len() as u64); - bytes.extend_from_slice(value.as_bytes()); -} - -const fn encode_block_alignment(alignment: BlockAlignment) -> u8 { - match alignment { - BlockAlignment::Left => 0, - BlockAlignment::Center => 1, - BlockAlignment::Right => 2, - } -} - -#[hotpath::measure] -fn decode_block_alignment(value: u8) -> io::Result { - match value { - 0 => Ok(BlockAlignment::Left), - 1 => Ok(BlockAlignment::Center), - 2 => Ok(BlockAlignment::Right), - _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid block alignment")), - } -} - -const fn encode_paragraph_style(style: ParagraphStyle) -> u8 { - match style { - ParagraphStyle::Custom(0) => 0, - ParagraphStyle::Custom(1) => 1, - ParagraphStyle::Custom(2) => 2, - ParagraphStyle::Custom(3) => 3, - ParagraphStyle::Custom(4) => 4, - ParagraphStyle::Normal => 5, - ParagraphStyle::Custom(6) => 6, - ParagraphStyle::Custom(slot) => 128 + (slot & 0x7f), - } -} - -#[hotpath::measure] -fn decode_paragraph_style(value: u8) -> io::Result { - match value { - 0 => Ok(ParagraphStyle::Custom(0)), - 1 => Ok(ParagraphStyle::Custom(1)), - 2 => Ok(ParagraphStyle::Custom(2)), - 3 => Ok(ParagraphStyle::Custom(3)), - 4 => Ok(ParagraphStyle::Custom(4)), - 5 => Ok(ParagraphStyle::Normal), - 6 => Ok(ParagraphStyle::Custom(6)), - 128..=255 => Ok(ParagraphStyle::Custom(value - 128)), - _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid paragraph style")), - } -} - -const fn encode_section_kind(kind: SectionKind) -> u8 { - match kind { - SectionKind::Custom(slot) => slot & 0x7f, - } -} - -#[hotpath::measure] -fn decode_section_kind(value: u8) -> io::Result { - match value { - 0..=127 => Ok(SectionKind::Custom(value)), - 128..=255 => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid section kind")), - } -} - -#[hotpath::measure] -fn write_run_styles(bytes: &mut Vec, styles: RunStyles) { - bytes.push(encode_run_semantic_style(styles.semantic)); - let mut flags = 0_u8; - if styles.direct_underline { - flags |= 1 << 0; - } - if styles.strikethrough { - flags |= 1 << 1; - } - bytes.push(flags); - bytes.push(encode_highlight_style(styles.highlight)); -} - -#[hotpath::measure] -fn read_run_styles(cursor: &mut Cursor<&[u8]>) -> io::Result { - let semantic = decode_run_semantic_style(read_u8(cursor)?)?; - let flags = read_u8(cursor)?; - if flags & !0b0000_0011 != 0 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid run style flags")); - } - Ok(RunStyles { - semantic, - direct_underline: flags & (1 << 0) != 0, - strikethrough: flags & (1 << 1) != 0, - highlight: decode_highlight_style(read_u8(cursor)?)?, - }) -} - -const fn encode_run_semantic_style(style: RunSemanticStyle) -> u8 { - match style { - RunSemanticStyle::Plain => 0, - RunSemanticStyle::Custom(1) => 1, - RunSemanticStyle::Custom(2) => 2, - RunSemanticStyle::Custom(3) => 3, - RunSemanticStyle::Custom(4) => 4, - RunSemanticStyle::Custom(5) => 5, - RunSemanticStyle::Custom(slot) => 128 + (slot & 0x7f), - } -} - -#[hotpath::measure] -fn decode_run_semantic_style(value: u8) -> io::Result { - match value { - 0 => Ok(RunSemanticStyle::Plain), - 1 => Ok(RunSemanticStyle::Custom(1)), - 2 => Ok(RunSemanticStyle::Custom(2)), - 3 => Ok(RunSemanticStyle::Custom(3)), - 4 => Ok(RunSemanticStyle::Custom(4)), - 5 => Ok(RunSemanticStyle::Custom(5)), - 128..=255 => Ok(RunSemanticStyle::Custom(value - 128)), - _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid run semantic style")), - } -} - -const fn encode_highlight_style(style: Option) -> u8 { - match style { - None => 0, - Some(HighlightStyle::Custom(1)) => 1, - Some(HighlightStyle::Custom(2)) => 2, - Some(HighlightStyle::Custom(3)) => 3, - Some(HighlightStyle::Custom(slot)) => 128 + (slot & 0x7f), - } -} - -#[hotpath::measure] -fn decode_highlight_style(value: u8) -> io::Result> { - Ok(match value { - 0 => None, - 1 => Some(HighlightStyle::Custom(1)), - 2 => Some(HighlightStyle::Custom(2)), - 3 => Some(HighlightStyle::Custom(3)), - 128..=255 => Some(HighlightStyle::Custom(value - 128)), - _ => { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "highlight slot is reserved but has no app style yet", - )); - }, - }) -} diff --git a/crates/gpui-flowtext/src/persistence/io.rs b/crates/gpui-flowtext/src/persistence/io.rs deleted file mode 100644 index 2b07ebe0..00000000 --- a/crates/gpui-flowtext/src/persistence/io.rs +++ /dev/null @@ -1,258 +0,0 @@ -use std::{ - collections::hash_map::DefaultHasher, - fs, - hash::{Hash as _, Hasher as _}, - io::{self, Cursor, Read as _, Write as _}, - ops::Range, - path::{Path, PathBuf}, - sync::Arc, - time::Instant, -}; - -use crop::Rope; -use tempfile::NamedTempFile; - -use super::{ - AssetId, AssetRecord, AssetStore, Block, BlockAlignment, BlockId, Document, DocumentIds, DocumentSection, DocumentTheme, EquationBlock, - EquationDisplay, EquationSyntax, HighlightStyle, ImageBlock, ImageSizing, Paragraph, ParagraphId, ParagraphOffsetIndex, ParagraphStyle, - RunSemanticStyle, RunStyles, SectionId, SectionKind, TableBlock, TableCell, TableCellBlock, TableCellParagraph, TableColumnWidth, TableRow, - TableStyle, TextRun, demo_document, document_text_slice, log_timing_lazy, merge_adjacent_runs, paragraph_byte_range, paragraph_index_for_id, - paragraph_runs_len, paragraph_text_len, rebuild_document_offset_index, rebuild_document_sections, reconcile_document_ids, -}; - -// Native binary document format: a magic header, a version, the raw -// UTF-8 text blob, then per-paragraph run metadata. Keeping the format -// length-prefixed makes the reader resilient against trailing junk. -const DOCUMENT_MAGIC: &[u8; 4] = b"GPTX"; -const LEGACY_DOCUMENT_MAGIC: &[u8; 4] = &[b'D', b'B', b'8', 0]; -const DOCUMENT_LEGACY_VERSION: u32 = 5; -const DOCUMENT_VERSION: u32 = 6; - -const CHUNK_TEXT: u8 = 1; -const CHUNK_ASSETS: u8 = 2; -const CHUNK_BLOCKS: u8 = 3; -const CHUNK_PARAGRAPH_IDS: u8 = 4; -const CHUNK_BLOCK_IDS: u8 = 5; -const CHUNK_SECTIONS: u8 = 6; - -const BLOCK_PARAGRAPH: u8 = 0; -const BLOCK_IMAGE: u8 = 1; -const BLOCK_EQUATION: u8 = 2; -const BLOCK_TABLE: u8 = 3; -const TABLE_CELL_PARAGRAPH: u8 = 0; -const TABLE_CELL_TABLE: u8 = 1; - -pub const DEFAULT_DOCUMENT_EXTENSION: &str = "gptx"; - -#[hotpath::measure] -pub fn load_or_create_document(path: impl AsRef) -> io::Result { - let path = path.as_ref(); - match read_document(path) { - Ok(document) => Ok(document), - Err(error) if error.kind() == io::ErrorKind::NotFound => { - let document = demo_document(); - // Best-effort write: if the path is in a read-only directory (e.g. the - // default demo path when the CWD is not writable) we still open the - // demo in memory rather than crashing. - let _ = write_document(path, &document); - Ok(document) - }, - Err(error) => Err(error), - } -} - -#[hotpath::measure] -pub fn read_document(path: impl AsRef) -> io::Result { - let timing = Instant::now(); - let bytes = fs::read(path)?; - read_document_bytes_with_timing(&bytes, timing) -} - -#[hotpath::measure] -pub fn read_document_bytes(bytes: &[u8]) -> io::Result { - read_document_bytes_with_timing(bytes, Instant::now()) -} - -#[hotpath::measure] -fn read_document_bytes_with_timing(bytes: &[u8], timing: Instant) -> io::Result { - let mut cursor = Cursor::new(bytes); - let mut magic = [0; 4]; - cursor.read_exact(&mut magic)?; - if &magic != DOCUMENT_MAGIC && &magic != LEGACY_DOCUMENT_MAGIC { - return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid document magic")); - } - let version = read_u32(&mut cursor)?; - if version == DOCUMENT_LEGACY_VERSION { - return read_document_current(cursor, timing); - } - if version == DOCUMENT_VERSION { - return read_document_vnext(cursor, timing); - } - Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported document version")) -} - -#[hotpath::measure] -pub fn write_document(path: impl AsRef, document: &Document) -> io::Result<()> { - let path = path.as_ref(); - // Skip directory creation when the parent component is empty (e.g. a bare - // filename like "doc.gptx" with no directory prefix), as create_dir_all("") - // fails on most platforms. write_bytes_atomic handles it identically. - if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { - fs::create_dir_all(parent)?; - } - let document = document_for_serialization(document); - validate_document(&document)?; - let bytes = serialize_document(&document); - write_bytes_atomic(path, &bytes) -} - -#[hotpath::measure] -pub fn document_bytes(document: &Document) -> io::Result> { - let document = document_for_serialization(document); - validate_document(&document)?; - Ok(serialize_document(&document)) -} - -#[hotpath::measure] -fn document_for_serialization(document: &Document) -> Document { - let mut document = document.clone(); - // Recovery/autosave can snapshot while live editing is still settling; make - // sure byte offsets are derived from the paragraph projection we are about - // to serialize instead of trusting cached offsets. - rebuild_document_offset_index(&mut document); - document.blocks = Arc::new(serializable_blocks(&document)); - reconcile_document_ids(&mut document); - rebuild_document_sections(&mut document); - document -} - -#[hotpath::measure] -#[hotpath::measure] -fn serialize_document(document: &Document) -> Vec { - let mut chunks = Vec::<(u8, Vec)>::new(); - let mut text = Vec::with_capacity(document.text.byte_len()); - for chunk in document.text.chunks() { - text.extend_from_slice(chunk.as_bytes()); - } - chunks.push((CHUNK_TEXT, text)); - - let mut assets = Vec::new(); - write_u64(&mut assets, document.assets.assets.len() as u64); - for asset in document.assets.assets.values() { - write_asset_record(&mut assets, asset); - } - chunks.push((CHUNK_ASSETS, assets)); - - let mut blocks = Vec::new(); - write_u64(&mut blocks, document.blocks.len() as u64); - for block in document.blocks.iter() { - write_block_record(&mut blocks, block); - } - chunks.push((CHUNK_BLOCKS, blocks)); - - let mut paragraph_ids = Vec::new(); - write_u64(&mut paragraph_ids, document.ids.paragraph_ids.len() as u64); - for id in &document.ids.paragraph_ids { - write_u128(&mut paragraph_ids, id.0); - } - chunks.push((CHUNK_PARAGRAPH_IDS, paragraph_ids)); - - let mut block_ids = Vec::new(); - write_u64(&mut block_ids, document.ids.block_ids.len() as u64); - for id in &document.ids.block_ids { - write_u128(&mut block_ids, id.0); - } - chunks.push((CHUNK_BLOCK_IDS, block_ids)); - - let mut sections = Vec::new(); - write_u64(&mut sections, document.sections.len() as u64); - for section in document.sections.iter() { - write_section_record(&mut sections, section); - } - chunks.push((CHUNK_SECTIONS, sections)); - - let table_entry_len = 1 + 1 + 2 + 8 + 8; - let header_len = DOCUMENT_MAGIC.len() + std::mem::size_of::() + std::mem::size_of::() + chunks.len() * table_entry_len; - let payload_len = chunks.iter().map(|(_, bytes)| bytes.len()).sum::(); - let mut bytes = Vec::with_capacity(header_len + payload_len); - bytes.extend_from_slice(DOCUMENT_MAGIC); - bytes.extend_from_slice(&DOCUMENT_VERSION.to_le_bytes()); - write_u32( - &mut bytes, - u32::try_from(chunks.len()).expect("native document chunk count is fixed and fits in u32"), - ); - let mut offset = header_len; - for (kind, payload) in &chunks { - bytes.push(*kind); - bytes.push(0); - bytes.extend_from_slice(&0_u16.to_le_bytes()); - write_u64(&mut bytes, offset as u64); - write_u64(&mut bytes, payload.len() as u64); - offset += payload.len(); - } - for (_, payload) in chunks { - bytes.extend_from_slice(&payload); - } - bytes -} - -#[hotpath::measure] -fn serializable_blocks(document: &Document) -> Vec { - let mut paragraph_ix = 0; - let mut blocks = Vec::with_capacity(document.blocks.len().max(document.paragraphs.len())); - - // The current editor mutates the paragraph projection. Rebuild paragraph - // block payloads from that live projection while keeping object/table blocks - // in their structural positions. - for block in document.blocks.iter() { - match block { - Block::Paragraph(_) => { - if let Some(paragraph) = document.paragraphs.get(paragraph_ix) { - let mut paragraph = paragraph.clone(); - paragraph.byte_range = paragraph_byte_range(document, paragraph_ix); - blocks.push(Block::Paragraph(paragraph)); - paragraph_ix += 1; - } - }, - other => blocks.push(other.clone()), - } - } - - while let Some(paragraph) = document.paragraphs.get(paragraph_ix) { - let mut paragraph = paragraph.clone(); - paragraph.byte_range = paragraph_byte_range(document, paragraph_ix); - blocks.push(Block::Paragraph(paragraph)); - paragraph_ix += 1; - } - - blocks -} - -#[hotpath::measure] -fn write_bytes_atomic(path: &Path, bytes: &[u8]) -> io::Result<()> { - // Use "." as fallback when the path has no directory component (e.g. a bare - // filename) so NamedTempFile::new_in doesn't receive an empty path. - let parent = path - .parent() - .filter(|p| !p.as_os_str().is_empty()) - .unwrap_or_else(|| Path::new(".")); - fs::create_dir_all(parent)?; - let mut temp = NamedTempFile::new_in(parent)?; - temp.write_all(bytes)?; - temp.as_file_mut().sync_all()?; - let temp_path = temp.into_temp_path(); - #[cfg(target_os = "windows")] - { - // Windows does not allow the POSIX-style atomic replace that tempfile's - // `persist` relies on for existing files. Remove the old target first, - // then rename the fully written temp file into place. This is slightly - // less atomic on Windows, but avoids false "Access is denied" failures - // when saving a normal existing document. - match fs::remove_file(path) { - Ok(()) => {}, - Err(error) if error.kind() == io::ErrorKind::NotFound => {}, - Err(error) => return Err(error), - } - } - temp_path.persist(path).map_err(|error| error.error) -} diff --git a/crates/gpui-flowtext/src/persistence/mod.rs b/crates/gpui-flowtext/src/persistence/mod.rs index 1447f5a4..a86ba7ac 100644 --- a/crates/gpui-flowtext/src/persistence/mod.rs +++ b/crates/gpui-flowtext/src/persistence/mod.rs @@ -1,4 +1,15 @@ -include!("io.rs"); -include!("records.rs"); -include!("validation.rs"); -include!("codec.rs"); +use std::path::{Path, PathBuf}; + +pub const DEFAULT_DOCUMENT_EXTENSION: &str = "db8"; + +#[hotpath::measure] +#[must_use] +pub fn recovery_path_for_document(path: &Path) -> PathBuf { + let mut recovery_path = path.to_path_buf(); + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .map_or_else(|| "untitled.db8.recovery".to_owned(), |name| format!("{name}.recovery")); + recovery_path.set_file_name(file_name); + recovery_path +} diff --git a/crates/gpui-flowtext/src/persistence/records.rs b/crates/gpui-flowtext/src/persistence/records.rs deleted file mode 100644 index a4c0df79..00000000 --- a/crates/gpui-flowtext/src/persistence/records.rs +++ /dev/null @@ -1,778 +0,0 @@ -#[derive(Clone, Copy)] -struct Db8Chunk { - kind: u8, - offset: usize, - len: usize, -} - -#[hotpath::measure] -fn read_document_vnext(mut cursor: Cursor<&[u8]>, timing: Instant) -> io::Result { - let chunk_count = read_u32(&mut cursor)? as usize; - let mut chunks = Vec::with_capacity(chunk_count.min(32)); - for _ in 0..chunk_count { - let kind = read_u8(&mut cursor)?; - let flags = read_u8(&mut cursor)?; - let _reserved = read_u16(&mut cursor)?; - if flags != 0 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "unsupported native document chunk flags")); - } - let offset = read_len(&mut cursor, "native document chunk offset")?; - let len = read_len(&mut cursor, "native document chunk length")?; - chunks.push(Db8Chunk { kind, offset, len }); - } - - let text_bytes = required_chunk(cursor.get_ref(), &chunks, CHUNK_TEXT, "native document text chunk")?; - let text = std::str::from_utf8(text_bytes) - .map(std::borrow::ToOwned::to_owned) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document text chunk is not UTF-8"))?; - let assets = read_assets_chunk(required_chunk(cursor.get_ref(), &chunks, CHUNK_ASSETS, "native document assets chunk")?)?; - let (blocks, paragraphs) = read_blocks_chunk( - required_chunk(cursor.get_ref(), &chunks, CHUNK_BLOCKS, "native document blocks chunk")?, - &text, - )?; - let paragraph_ids = read_paragraph_ids_chunk(required_chunk( - cursor.get_ref(), - &chunks, - CHUNK_PARAGRAPH_IDS, - "native document paragraph IDs chunk", - )?)?; - let block_ids = read_block_ids_chunk(required_chunk( - cursor.get_ref(), - &chunks, - CHUNK_BLOCK_IDS, - "native document block IDs chunk", - )?)?; - let sections = read_sections_chunk(required_chunk( - cursor.get_ref(), - &chunks, - CHUNK_SECTIONS, - "native document sections chunk", - )?)?; - - let offset_index = ParagraphOffsetIndex::new(¶graphs); - let mut document = Document { - text: Rope::from(text), - paragraphs: Arc::new(paragraphs), - blocks: Arc::new(blocks), - assets, - ids: DocumentIds { - document_id: 0, - paragraph_ids, - block_ids, - }, - sections: Arc::new(sections), - offset_index, - theme: DocumentTheme::default(), - }; - reconcile_document_ids(&mut document); - validate_or_rebuild_sections(&mut document); - validate_document(&document)?; - log_timing_lazy("document vnext read", timing, || { - format!( - "bytes={} blocks={} paragraphs={} sections={}", - document.text.byte_len(), - document.blocks.len(), - document.paragraphs.len(), - document.sections.len() - ) - }); - Ok(document) -} - -#[hotpath::measure] -fn required_chunk<'bytes>(bytes: &'bytes [u8], chunks: &[Db8Chunk], kind: u8, label: &'static str) -> io::Result<&'bytes [u8]> { - let chunk = chunks - .iter() - .find(|chunk| chunk.kind == kind) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("{label} is missing")))?; - let end = chunk - .offset - .checked_add(chunk.len) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, format!("{label} range overflows")))?; - if end > bytes.len() { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, format!("{label} is truncated"))); - } - Ok(&bytes[chunk.offset..end]) -} - -#[hotpath::measure] -fn read_assets_chunk(bytes: &[u8]) -> io::Result { - let mut cursor = Cursor::new(bytes); - let asset_count = read_len(&mut cursor, "native document asset count")?; - let mut assets = AssetStore::default(); - assets.assets.reserve(asset_count); - for _ in 0..asset_count { - let asset = read_asset_record(&mut cursor)?; - assets.assets.insert(asset.id, asset); - } - Ok(assets) -} - -#[hotpath::measure] -fn read_blocks_chunk(bytes: &[u8], text: &str) -> io::Result<(Vec, Vec)> { - let mut cursor = Cursor::new(bytes); - let block_count = read_len(&mut cursor, "native document block count")?; - let mut blocks = Vec::with_capacity(block_count.min(4096)); - let mut paragraphs = Vec::new(); - for _ in 0..block_count { - let mut block = read_block_record(&mut cursor)?; - normalize_block_text_runs(&mut block, text)?; - if let Block::Paragraph(paragraph) = &block { - paragraphs.push(paragraph.clone()); - } - blocks.push(block); - } - if paragraphs.is_empty() { - paragraphs.push(Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }); - blocks.push(Block::Paragraph(paragraphs[0].clone())); - } - Ok((blocks, paragraphs)) -} - -#[hotpath::measure] -fn read_paragraph_ids_chunk(bytes: &[u8]) -> io::Result> { - let mut cursor = Cursor::new(bytes); - let count = read_len(&mut cursor, "native document paragraph ID count")?; - let mut ids = Vec::with_capacity(count); - for _ in 0..count { - ids.push(ParagraphId(read_u128(&mut cursor)?)); - } - Ok(ids) -} - -#[hotpath::measure] -fn read_block_ids_chunk(bytes: &[u8]) -> io::Result> { - let mut cursor = Cursor::new(bytes); - let count = read_len(&mut cursor, "native document block ID count")?; - let mut ids = Vec::with_capacity(count); - for _ in 0..count { - ids.push(BlockId(read_u128(&mut cursor)?)); - } - Ok(ids) -} - -#[hotpath::measure] -fn read_sections_chunk(bytes: &[u8]) -> io::Result> { - let mut cursor = Cursor::new(bytes); - let count = read_len(&mut cursor, "native document section count")?; - let mut sections = Vec::with_capacity(count); - for _ in 0..count { - sections.push(read_section_record(&mut cursor)?); - } - Ok(sections) -} - -#[hotpath::measure] -fn read_document_current(mut cursor: Cursor<&[u8]>, timing: Instant) -> io::Result { - let text_len = { - let raw = read_u64(&mut cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document text length overflows usize"))? - }; - let text_bytes = read_bytes(&mut cursor, text_len, "native document text")?; - let text = std::str::from_utf8(text_bytes) - .map(std::borrow::ToOwned::to_owned) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document text is not UTF-8"))?; - - let asset_count = { - let raw = read_u64(&mut cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document asset count overflows usize"))? - }; - let mut assets = AssetStore::default(); - assets.assets.reserve(asset_count); - for _ in 0..asset_count { - let asset = read_asset_record(&mut cursor)?; - assets.assets.insert(asset.id, asset); - } - - let block_count = { - let raw = read_u64(&mut cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document block count overflows usize"))? - }; - let mut blocks = Vec::with_capacity(block_count.min(4096)); - let mut paragraphs = Vec::new(); - for _ in 0..block_count { - let mut block = read_block_record(&mut cursor)?; - normalize_block_text_runs(&mut block, &text)?; - if let Block::Paragraph(paragraph) = &block { - paragraphs.push(paragraph.clone()); - } - blocks.push(block); - } - if paragraphs.is_empty() { - paragraphs.push(Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }); - blocks.push(Block::Paragraph(paragraphs[0].clone())); - } - - let offset_index = ParagraphOffsetIndex::new(¶graphs); - let mut document = Document { - text: Rope::from(text), - paragraphs: Arc::new(paragraphs), - blocks: Arc::new(blocks), - assets, - ids: DocumentIds::default(), - sections: Arc::new(Vec::new()), - offset_index, - theme: DocumentTheme::default(), - }; - reconcile_document_ids(&mut document); - rebuild_document_sections(&mut document); - validate_document(&document)?; - log_timing_lazy("document read", timing, || { - format!( - "bytes={} blocks={} paragraphs={}", - document.text.byte_len(), - document.blocks.len(), - document.paragraphs.len() - ) - }); - Ok(document) -} - -#[hotpath::measure] -fn validate_or_rebuild_sections(document: &mut Document) { - if document.sections.is_empty() - || document - .sections - .iter() - .any(|section| paragraph_index_for_id(document, section.start_paragraph).is_none()) - { - rebuild_document_sections(document); - } -} - -#[hotpath::measure] -fn normalize_block_text_runs(block: &mut Block, document_text: &str) -> io::Result<()> { - match block { - Block::Paragraph(paragraph) => normalize_paragraph_text_runs(paragraph, document_text), - Block::Image(_) | Block::Equation(_) => Ok(()), - Block::Table(table) => normalize_table_text_runs(table), - } -} - -#[hotpath::measure] -fn normalize_table_text_runs(table: &mut TableBlock) -> io::Result<()> { - for row in &mut table.rows { - for cell in &mut row.cells { - for block in &mut cell.blocks { - match block { - TableCellBlock::Paragraph(paragraph) => normalize_runs_for_text(&mut paragraph.paragraph.runs, ¶graph.text)?, - TableCellBlock::Table(table) => normalize_table_text_runs(table)?, - } - } - } - } - Ok(()) -} - -#[hotpath::measure] -fn normalize_paragraph_text_runs(paragraph: &mut Paragraph, document_text: &str) -> io::Result<()> { - if paragraph.byte_range.start > paragraph.byte_range.end - || paragraph.byte_range.end > document_text.len() - || !document_text.is_char_boundary(paragraph.byte_range.start) - || !document_text.is_char_boundary(paragraph.byte_range.end) - { - return Err(io::Error::new(io::ErrorKind::InvalidData, "paragraph byte range is invalid")); - } - normalize_runs_for_text(&mut paragraph.runs, &document_text[paragraph.byte_range.clone()]) -} - -#[hotpath::measure] -fn normalize_runs_for_text(runs: &mut Vec, text: &str) -> io::Result<()> { - let run_len = runs.iter().map(|run| run.len).sum::(); - if run_len == text.len() && run_boundaries_are_char_boundaries(runs, text) { - return Ok(()); - } - - if run_len == text.chars().count() { - let byte_offsets = char_count_boundaries_to_byte_offsets(runs, text)?; - for (run, range) in runs.iter_mut().zip(byte_offsets.windows(2)) { - run.len = range[1] - range[0]; - } - *runs = merge_adjacent_runs(std::mem::take(runs)); - return Ok(()); - } - - if runs.len() == 1 { - runs[0].len = text.len(); - return Ok(()); - } - - Err(io::Error::new( - io::ErrorKind::InvalidData, - "paragraph run lengths do not match text bytes or characters", - )) -} - -#[hotpath::measure] -fn run_boundaries_are_char_boundaries(runs: &[TextRun], text: &str) -> bool { - let mut byte = 0_usize; - for run in runs { - byte = byte.saturating_add(run.len); - if byte < text.len() && !text.is_char_boundary(byte) { - return false; - } - } - byte == text.len() -} - -#[hotpath::measure] -fn char_count_boundaries_to_byte_offsets(runs: &[TextRun], text: &str) -> io::Result> { - let mut offsets = Vec::with_capacity(runs.len() + 1); - offsets.push(0); - let mut char_count = 0_usize; - for run in runs { - char_count = char_count.saturating_add(run.len); - if char_count == text.chars().count() { - offsets.push(text.len()); - continue; - } - let Some((byte, _)) = text.char_indices().nth(char_count) else { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "run character offset is outside paragraph text", - )); - }; - offsets.push(byte); - } - Ok(offsets) -} - -#[hotpath::measure] -fn read_block_record(cursor: &mut Cursor<&[u8]>) -> io::Result { - let kind = read_u8(cursor)?; - let payload_len = { - let raw = read_u64(cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document block payload length overflows usize"))? - }; - let payload = read_bytes(cursor, payload_len, "native document block payload")?; - let mut payload = Cursor::new(payload); - match kind { - BLOCK_PARAGRAPH => read_paragraph_payload(&mut payload).map(Block::Paragraph), - BLOCK_IMAGE => read_image_payload(&mut payload).map(Block::Image), - BLOCK_EQUATION => read_equation_payload(&mut payload).map(Block::Equation), - BLOCK_TABLE => read_table_payload(&mut payload).map(Block::Table), - _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid native document block kind")), - } -} - -#[hotpath::measure] -fn write_block_record(bytes: &mut Vec, block: &Block) { - let mut payload = Vec::new(); - let kind = match block { - Block::Paragraph(paragraph) => { - write_paragraph_payload(&mut payload, paragraph, paragraph.byte_range.clone()); - BLOCK_PARAGRAPH - }, - Block::Image(image) => { - write_image_payload(&mut payload, image); - BLOCK_IMAGE - }, - Block::Equation(equation) => { - write_equation_payload(&mut payload, equation); - BLOCK_EQUATION - }, - Block::Table(table) => { - write_table_payload(&mut payload, table); - BLOCK_TABLE - }, - }; - bytes.push(kind); - write_u64(bytes, payload.len() as u64); - bytes.extend_from_slice(&payload); -} - -#[hotpath::measure] -fn write_section_record(bytes: &mut Vec, section: &DocumentSection) { - write_u128(bytes, section.id.0); - write_u128(bytes, section.parent_id.map_or(0, |id| id.0)); - bytes.push(encode_section_kind(section.kind)); - bytes.push(u8::from(section.heading_paragraph.is_some())); - bytes.push(u8::from(section.end_paragraph_exclusive.is_some())); - bytes.push(0); - write_u128(bytes, section.heading_paragraph.map_or(0, |id| id.0)); - write_u128(bytes, section.start_paragraph.0); - write_u128(bytes, section.end_paragraph_exclusive.map_or(0, |id| id.0)); -} - -#[hotpath::measure] -fn read_section_record(cursor: &mut Cursor<&[u8]>) -> io::Result { - let id = SectionId(read_u128(cursor)?); - let parent = read_u128(cursor)?; - let kind = decode_section_kind(read_u8(cursor)?)?; - let has_heading = read_u8(cursor)? != 0; - let has_end = read_u8(cursor)? != 0; - let reserved = read_u8(cursor)?; - if reserved != 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "invalid native document section reserved byte", - )); - } - let heading = read_u128(cursor)?; - let start = read_u128(cursor)?; - let end = read_u128(cursor)?; - Ok(DocumentSection { - id, - parent_id: (parent != 0).then_some(SectionId(parent)), - kind, - heading_paragraph: has_heading.then_some(ParagraphId(heading)), - start_paragraph: ParagraphId(start), - end_paragraph_exclusive: has_end.then_some(ParagraphId(end)), - }) -} - -#[hotpath::measure] -fn read_paragraph_payload(cursor: &mut Cursor<&[u8]>) -> io::Result { - let style = decode_paragraph_style(read_u8(cursor)?)?; - let start = { - let raw = read_u64(cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document paragraph start overflows usize"))? - }; - let end = { - let raw = read_u64(cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document paragraph end overflows usize"))? - }; - let run_count = { - let raw = read_u64(cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document run count overflows usize"))? - }; - let mut runs = Vec::with_capacity(run_count.min(4096)); - for _ in 0..run_count { - let len = { - let raw = read_u64(cursor)?; - usize::try_from(raw).map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "native document run length overflows usize"))? - }; - let styles = read_run_styles(cursor)?; - runs.push(TextRun { len, styles }); - } - Ok(Paragraph { - style, - byte_range: start..end, - runs: merge_adjacent_runs(runs), - version: 0, - }) -} - -#[hotpath::measure] -fn write_paragraph_payload(bytes: &mut Vec, paragraph: &Paragraph, range: Range) { - bytes.push(encode_paragraph_style(paragraph.style)); - write_u64(bytes, range.start as u64); - write_u64(bytes, range.end as u64); - write_u64(bytes, paragraph.runs.len() as u64); - for run in ¶graph.runs { - write_u64(bytes, run.len as u64); - write_run_styles(bytes, run.styles); - } -} - -#[hotpath::measure] -fn read_image_payload(cursor: &mut Cursor<&[u8]>) -> io::Result { - let asset_id = AssetId(read_u128(cursor)?); - let alt_text = read_string(cursor)?.into(); - let caption = if read_u8(cursor)? == 1 { - Some(read_paragraph_payload(cursor)?) - } else { - None - }; - let sizing = match read_u8(cursor)? { - 0 => ImageSizing::Intrinsic, - 1 => ImageSizing::FitWidth, - 2 => { - let width_px = read_u32(cursor)?; - let height_px = if read_u8(cursor)? == 1 { Some(read_u32(cursor)?) } else { None }; - ImageSizing::Fixed { width_px, height_px } - }, - _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid image sizing")), - }; - let alignment = decode_block_alignment(read_u8(cursor)?)?; - Ok(ImageBlock { - asset_id, - alt_text, - caption, - sizing, - alignment, - version: 0, - }) -} - -#[hotpath::measure] -fn write_image_payload(bytes: &mut Vec, image: &ImageBlock) { - write_u128(bytes, image.asset_id.0); - write_string(bytes, image.alt_text.as_ref()); - match &image.caption { - Some(caption) => { - bytes.push(1); - write_paragraph_payload(bytes, caption, caption.byte_range.clone()); - }, - None => bytes.push(0), - } - match image.sizing { - ImageSizing::Intrinsic => bytes.push(0), - ImageSizing::FitWidth => bytes.push(1), - ImageSizing::Fixed { width_px, height_px } => { - bytes.push(2); - bytes.extend_from_slice(&width_px.to_le_bytes()); - match height_px { - Some(height_px) => { - bytes.push(1); - bytes.extend_from_slice(&height_px.to_le_bytes()); - }, - None => bytes.push(0), - } - }, - } - bytes.push(encode_block_alignment(image.alignment)); -} - -#[hotpath::measure] -fn read_equation_payload(cursor: &mut Cursor<&[u8]>) -> io::Result { - let syntax = match read_u8(cursor)? { - 0 => EquationSyntax::Latex, - _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid equation syntax")), - }; - let display = match read_u8(cursor)? { - 0 => EquationDisplay::Display, - 1 => EquationDisplay::InlineLikeParagraph, - _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid equation display")), - }; - Ok(EquationBlock { - source: read_string(cursor)?.into(), - syntax, - display, - version: 0, - }) -} - -#[hotpath::measure] -fn write_equation_payload(bytes: &mut Vec, equation: &EquationBlock) { - bytes.push(match equation.syntax { - EquationSyntax::Latex => 0, - }); - bytes.push(match equation.display { - EquationDisplay::Display => 0, - EquationDisplay::InlineLikeParagraph => 1, - }); - write_string(bytes, equation.source.as_ref()); -} - -#[hotpath::measure] -fn read_table_payload(cursor: &mut Cursor<&[u8]>) -> io::Result { - let column_count = read_len(cursor, "native document table column count")?; - let mut column_widths = Vec::with_capacity(column_count.min(64)); - for _ in 0..column_count { - column_widths.push(match read_u8(cursor)? { - 0 => TableColumnWidth::Auto, - 1 => TableColumnWidth::FixedPx(read_u32(cursor)?), - 2 => TableColumnWidth::Fraction(read_u32(cursor)?), - _ => return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid table column width")), - }); - } - let header_row = read_u8(cursor)? != 0; - let row_count = read_len(cursor, "native document table row count")?; - let mut rows = Vec::with_capacity(row_count.min(4096)); - for _ in 0..row_count { - let cell_count = read_len(cursor, "native document table cell count")?; - let mut cells = Vec::with_capacity(cell_count.min(128)); - for _ in 0..cell_count { - let row_span = read_u16(cursor)?; - let col_span = read_u16(cursor)?; - let block_count = read_len(cursor, "native document table cell block count")?; - let mut blocks = Vec::with_capacity(block_count.min(64)); - for _ in 0..block_count { - blocks.push(read_table_cell_block(cursor)?); - } - cells.push(TableCell { blocks, row_span, col_span }); - } - rows.push(TableRow { cells }); - } - Ok(TableBlock { - rows, - column_widths, - style: TableStyle { header_row }, - version: 0, - }) -} - -#[hotpath::measure] -fn write_table_payload(bytes: &mut Vec, table: &TableBlock) { - write_u64(bytes, table.column_widths.len() as u64); - for width in &table.column_widths { - match *width { - TableColumnWidth::Auto => bytes.push(0), - TableColumnWidth::FixedPx(px) => { - bytes.push(1); - bytes.extend_from_slice(&px.to_le_bytes()); - }, - TableColumnWidth::Fraction(fraction) => { - bytes.push(2); - bytes.extend_from_slice(&fraction.to_le_bytes()); - }, - } - } - bytes.push(u8::from(table.style.header_row)); - write_u64(bytes, table.rows.len() as u64); - for row in &table.rows { - write_u64(bytes, row.cells.len() as u64); - for cell in &row.cells { - bytes.extend_from_slice(&cell.row_span.to_le_bytes()); - bytes.extend_from_slice(&cell.col_span.to_le_bytes()); - write_u64(bytes, cell.blocks.len() as u64); - for block in &cell.blocks { - write_table_cell_block(bytes, block); - } - } - } -} - -#[hotpath::measure] -fn read_table_cell_block(cursor: &mut Cursor<&[u8]>) -> io::Result { - match read_u8(cursor)? { - TABLE_CELL_PARAGRAPH => { - let text = read_string(cursor)?; - let paragraph = read_paragraph_payload(cursor)?; - Ok(TableCellBlock::Paragraph(TableCellParagraph { paragraph, text })) - }, - TABLE_CELL_TABLE => read_table_payload(cursor).map(TableCellBlock::Table), - _ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid table cell block kind")), - } -} - -#[hotpath::measure] -fn write_table_cell_block(bytes: &mut Vec, block: &TableCellBlock) { - match block { - TableCellBlock::Paragraph(paragraph) => { - bytes.push(TABLE_CELL_PARAGRAPH); - write_string(bytes, ¶graph.text); - write_paragraph_payload(bytes, ¶graph.paragraph, 0..paragraph.text.len()); - }, - TableCellBlock::Table(table) => { - bytes.push(TABLE_CELL_TABLE); - write_table_payload(bytes, table); - }, - } -} - -#[hotpath::measure] -fn read_asset_record(cursor: &mut Cursor<&[u8]>) -> io::Result { - let id = AssetId(read_u128(cursor)?); - let mime_type = read_string(cursor)?.into(); - let original_name = if read_u8(cursor)? == 1 { - Some(read_string(cursor)?.into()) - } else { - None - }; - let content_hash = read_u64(cursor)?; - let byte_len = read_len(cursor, "native document asset byte length")?; - let bytes = read_bytes(cursor, byte_len, "native document asset bytes")?.to_vec(); - Ok(AssetRecord { - id, - mime_type, - original_name, - content_hash, - bytes: Arc::new(bytes), - }) -} - -#[hotpath::measure] -fn write_asset_record(bytes: &mut Vec, asset: &AssetRecord) { - write_u128(bytes, asset.id.0); - write_string(bytes, asset.mime_type.as_ref()); - match &asset.original_name { - Some(name) => { - bytes.push(1); - write_string(bytes, name.as_ref()); - }, - None => bytes.push(0), - } - write_u64(bytes, asset.content_hash); - write_u64(bytes, asset.bytes.len() as u64); - bytes.extend_from_slice(&asset.bytes); -} - -#[hotpath::measure] -#[must_use] -pub fn recovery_path_for_document(path: &Path) -> PathBuf { - let mut recovery_path = path.to_path_buf(); - let file_name = path - .file_name() - .and_then(|name| name.to_str()) - .map_or_else(|| "untitled.document.recovery".to_owned(), |name| format!("{name}.recovery")); - recovery_path.set_file_name(file_name); - recovery_path -} - -#[cfg(test)] -mod records_tests { - use super::*; - use crate::{InputParagraph, InputRun}; - - #[test] - fn normalizes_legacy_character_count_run_lengths() { - let mut runs = vec![TextRun { - len: "Kepe et al. ‘23".chars().count(), - styles: RunStyles::default(), - }]; - - normalize_runs_for_text(&mut runs, "Kepe et al. ‘23").expect("normalize runs"); - - assert_eq!(runs[0].len, "Kepe et al. ‘23".len()); - } - - #[test] - fn normalizes_legacy_character_count_run_boundaries() { - let cite = RunStyles { - semantic: RunSemanticStyle::Custom(1), - ..RunStyles::default() - }; - let mut runs = vec![ - TextRun { - len: "Kepe et al. ".chars().count(), - styles: RunStyles::default(), - }, - TextRun { - len: "‘23".chars().count(), - styles: cite, - }, - ]; - - normalize_runs_for_text(&mut runs, "Kepe et al. ‘23").expect("normalize runs"); - - assert_eq!(runs[0].len, "Kepe et al. ".len()); - assert_eq!(runs[1].len, "‘23".len()); - } - - #[test] - fn custom_style_slots_round_trip_through_document() { - let document = crate::document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Custom(7), - runs: vec![InputRun { - text: "custom".to_string(), - styles: RunStyles { - semantic: RunSemanticStyle::Custom(9), - highlight: Some(HighlightStyle::Custom(11)), - ..RunStyles::default() - }, - }], - }], - ); - - let bytes = crate::document_bytes(&document).expect("serialize custom styles"); - let loaded = crate::read_document_bytes(&bytes).expect("read custom styles"); - - assert_eq!(loaded.paragraphs[0].style, ParagraphStyle::Custom(7)); - assert_eq!(loaded.paragraphs[0].runs[0].styles.semantic, RunSemanticStyle::Custom(9)); - assert_eq!(loaded.paragraphs[0].runs[0].styles.highlight, Some(HighlightStyle::Custom(11))); - } -} diff --git a/crates/gpui-flowtext/src/persistence/validation.rs b/crates/gpui-flowtext/src/persistence/validation.rs deleted file mode 100644 index 60a91db1..00000000 --- a/crates/gpui-flowtext/src/persistence/validation.rs +++ /dev/null @@ -1,230 +0,0 @@ - -#[allow(dead_code, reason = "Persistence validation is kept available for debug and importer validation paths.")] -#[hotpath::measure] -fn document_fingerprint(document: &Document) -> u64 { - let mut hasher = DefaultHasher::new(); - document_text_slice(document, 0..document.text.byte_len()).hash(&mut hasher); - for (paragraph_ix, paragraph) in document.paragraphs.iter().enumerate() { - let range = paragraph_byte_range(document, paragraph_ix); - paragraph.style.hash(&mut hasher); - range.start.hash(&mut hasher); - range.end.hash(&mut hasher); - paragraph.runs.hash(&mut hasher); - } - hasher.finish() -} - -#[hotpath::measure] -fn validate_document(document: &Document) -> io::Result<()> { - let text_len = document.text.byte_len(); - if document.paragraphs.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidData, "native document has no paragraphs")); - } - if document.ids.paragraph_ids.len() != document.paragraphs.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "paragraph ID count does not match paragraph count", - )); - } - if document.ids.block_ids.len() != document.blocks.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "block ID count does not match block count", - )); - } - for (ix, paragraph) in document.paragraphs.iter().enumerate() { - let range = paragraph_byte_range(document, ix); - if range.start > range.end || range.end > text_len { - return Err(io::Error::new(io::ErrorKind::InvalidData, "paragraph range is outside document text")); - } - if ix == 0 && range.start != 0 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "first paragraph does not start at byte 0")); - } - if paragraph_runs_len(paragraph) != paragraph_text_len(paragraph) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "paragraph run lengths do not match paragraph text", - )); - } - // Verify every run boundary falls on a valid UTF-8 char boundary. A - // corrupt native document could declare correct total run lengths but split a - // multibyte character mid-codepoint, which would panic when layout - // later slices the rope at those offsets. - { - let p_text = document_text_slice(document, range.clone()); - let mut run_end = 0; - for run in ¶graph.runs { - run_end += run.len; - if run_end < p_text.len() && !p_text.is_char_boundary(run_end) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "run boundary splits a UTF-8 character")); - } - } - } - if ix > 0 { - let previous_range = paragraph_byte_range(document, ix - 1); - if previous_range.end + 1 != range.start || document.text.byte(previous_range.end) != b'\n' { - return Err(io::Error::new(io::ErrorKind::InvalidData, "paragraph ranges are not newline separated")); - } - } - } - if document - .paragraphs - .last() - .is_some_and(|_| paragraph_byte_range(document, document.paragraphs.len() - 1).end != text_len) - { - return Err(io::Error::new(io::ErrorKind::InvalidData, "last paragraph does not end at text length")); - } - validate_paragraph_block_projection(document)?; - validate_sections(document)?; - for block in document.blocks.iter() { - validate_block_payload(block, document, 0)?; - } - Ok(()) -} - -#[hotpath::measure] -fn validate_sections(document: &Document) -> io::Result<()> { - for section in document.sections.iter() { - if paragraph_index_for_id(document, section.start_paragraph).is_none() { - return Err(io::Error::new(io::ErrorKind::InvalidData, "section start paragraph ID is invalid")); - } - if let Some(heading) = section.heading_paragraph - && paragraph_index_for_id(document, heading).is_none() - { - return Err(io::Error::new(io::ErrorKind::InvalidData, "section heading paragraph ID is invalid")); - } - if let Some(end) = section.end_paragraph_exclusive - && paragraph_index_for_id(document, end).is_none() - { - return Err(io::Error::new(io::ErrorKind::InvalidData, "section end paragraph ID is invalid")); - } - } - Ok(()) -} - -#[hotpath::measure] -fn validate_paragraph_block_projection(document: &Document) -> io::Result<()> { - let paragraph_blocks = document - .blocks - .iter() - .filter_map(|block| match block { - Block::Paragraph(paragraph) => Some(paragraph), - Block::Image(_) | Block::Equation(_) | Block::Table(_) => None, - }) - .collect::>(); - if paragraph_blocks.len() != document.paragraphs.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "paragraph block count does not match paragraph metadata", - )); - } - for (block_paragraph, paragraph) in paragraph_blocks.iter().zip(document.paragraphs.iter()) { - if *block_paragraph != paragraph { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "paragraph block payload does not match paragraph metadata", - )); - } - } - Ok(()) -} - -#[hotpath::measure] -fn validate_block_payload(block: &Block, document: &Document, table_depth: usize) -> io::Result<()> { - match block { - // Missing assets are allowed so a partially damaged document can still - // open and show a visible missing-image block instead of failing load. - Block::Image(image) => validate_image_payload(image, document)?, - Block::Equation(equation) => validate_equation_payload(equation)?, - Block::Table(table) => validate_table_payload(table, table_depth)?, - Block::Paragraph(paragraph) => { - if paragraph_runs_len(paragraph) != paragraph_text_len(paragraph) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "paragraph block run lengths are invalid")); - } - }, - } - Ok(()) -} - -#[hotpath::measure] -fn validate_image_payload(image: &ImageBlock, document: &Document) -> io::Result<()> { - match image.sizing { - ImageSizing::Fixed { width_px, height_px } => { - if width_px == 0 || height_px == Some(0) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "fixed image dimensions must be nonzero")); - } - }, - ImageSizing::Intrinsic | ImageSizing::FitWidth => {}, - } - if let Some(caption) = &image.caption - && paragraph_runs_len(caption) != paragraph_text_len(caption) - { - return Err(io::Error::new(io::ErrorKind::InvalidData, "image caption run lengths are invalid")); - } - if let Some(asset) = document.assets.assets.get(&image.asset_id) { - let mut hasher = DefaultHasher::new(); - asset.bytes.hash(&mut hasher); - if asset.content_hash != hasher.finish() { - return Err(io::Error::new(io::ErrorKind::InvalidData, "image asset content hash mismatch")); - } - } - Ok(()) -} - -#[hotpath::measure] -fn validate_equation_payload(equation: &EquationBlock) -> io::Result<()> { - if equation.source.len() > 64 * 1024 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "equation source is too large")); - } - Ok(()) -} - -#[hotpath::measure] -fn validate_table_payload(table: &TableBlock, depth: usize) -> io::Result<()> { - if depth > 8 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "nested tables are too deep")); - } - if table.rows.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidData, "table has no rows")); - } - let expected_columns = table.column_widths.len().max(1); - for row in &table.rows { - if row.cells.is_empty() { - return Err(io::Error::new(io::ErrorKind::InvalidData, "table row has no cells")); - } - let mut span_total = 0_usize; - for cell in &row.cells { - if cell.row_span == 0 || cell.col_span == 0 { - return Err(io::Error::new(io::ErrorKind::InvalidData, "table cell span cannot be zero")); - } - span_total = span_total.saturating_add(cell.col_span as usize); - for block in &cell.blocks { - match block { - TableCellBlock::Paragraph(paragraph) => { - if paragraph.paragraph.byte_range != (0..paragraph.text.len()) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "table cell paragraph byte range is invalid")); - } - if paragraph_runs_len(¶graph.paragraph) != paragraph.text.len() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "table cell paragraph run lengths do not match text", - )); - } - let mut run_end = 0; - for run in ¶graph.paragraph.runs { - run_end += run.len; - if run_end < paragraph.text.len() && !paragraph.text.is_char_boundary(run_end) { - return Err(io::Error::new(io::ErrorKind::InvalidData, "table cell run boundary splits UTF-8")); - } - } - }, - TableCellBlock::Table(nested) => validate_table_payload(nested, depth + 1)?, - } - } - } - if span_total != expected_columns { - return Err(io::Error::new(io::ErrorKind::InvalidData, "table row shape does not match column count")); - } - } - Ok(()) -} diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/layout.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/layout.rs index 762613d3..12e8a332 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/layout.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/layout.rs @@ -1,6 +1,6 @@ #[hotpath::measure] fn benchmark_layout_paths( - document: &Document, + document: &DocumentProjection, widths: &[f32], iterations: usize, include_paint: bool, @@ -40,7 +40,23 @@ fn benchmark_layout_paths( let paint_bounds = Bounds::new(point(px(0.0), px(0.0)), size(width_px, layout.size.height)); let paint_plain = include_paint.then(|| { repeated(iterations, || { - paint_layout(&layout, paint_bounds, None, None, false, px(1.0), &[], &[], None, window, cx); + paint_layout( + &layout, + paint_bounds, + None, + None, + false, + px(1.0), + None, + gpui::black(), + &[], + &[], + &[], + &[], + None, + window, + cx, + ); }) }); let selection = top_selection(document); @@ -53,6 +69,10 @@ fn benchmark_layout_paths( None, false, px(1.0), + None, + gpui::black(), + &[], + &[], &[], &[], None, @@ -98,7 +118,7 @@ fn benchmark_layout_paths( #[hotpath::measure] fn benchmark_sample_paragraph_layouts( - document: &Document, + document: &DocumentProjection, stats: &DocumentStats, widths: &[f32], iterations: usize, @@ -179,7 +199,7 @@ fn repeated(iterations: usize, mut run: impl FnMut()) -> DurationStats { } #[hotpath::measure] -fn estimate_error(document: &Document, layout: &LayoutState, width: Pixels) -> (f32, f32) { +fn estimate_error(document: &DocumentProjection, layout: &LayoutState, width: Pixels) -> (f32, f32) { let mut total = 0.0f32; let mut max = 0.0f32; let mut count = 0usize; @@ -199,7 +219,7 @@ fn estimate_error(document: &Document, layout: &LayoutState, width: Pixels) -> ( } #[hotpath::measure] -fn summarize_layout(document: &Document, layout: &LayoutState) -> LayoutSummary { +fn summarize_layout(document: &DocumentProjection, layout: &LayoutState) -> LayoutSummary { let mut summary = LayoutSummary { layout_height: px_to_f32(layout.size.height), ..Default::default() diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/operations.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/operations.rs index 762b150c..05802352 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/operations.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/operations.rs @@ -1,5 +1,5 @@ #[hotpath::measure] -fn benchmark_index_paths(document: &Document, iterations: usize) -> Vec { +fn benchmark_index_paths(document: &DocumentProjection, iterations: usize) -> Vec { let mut rows = Vec::new(); rows.push(operation_row("paragraph_byte_range all", iterations, || { let mut failures = 0; @@ -61,7 +61,7 @@ fn benchmark_index_paths(document: &Document, iterations: usize) -> Vec Vec { +fn benchmark_edit_paths(document: &DocumentProjection, stats: &DocumentStats, iterations: usize) -> Vec { let mut rows = Vec::new(); let largest = stats .largest_paragraph_ix diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/reports.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/reports.rs index 03894409..c8a4d406 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/reports.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/reports.rs @@ -88,62 +88,3 @@ fn write_fidelity_report(out: &mut String, fidelity: &FidelityReport) { } let _ = writeln!(out); } - -#[hotpath::measure] -fn write_roundtrip_report(out: &mut String, loaded: &LoadedDocument, iterations: usize) { - let mut write_timings = Vec::with_capacity(iterations); - let mut read_timings = Vec::with_capacity(iterations); - let mut fingerprint_matches = true; - let mut last_error = None; - let original = fingerprint_document(&loaded.document); - - for _ in 0..iterations { - let result = (|| { - let dir = tempdir().map_err(|error| error.to_string())?; - let path = dir.path().join("roundtrip.gptx"); - let write_start = Instant::now(); - write_document(&path, &loaded.document).map_err(|error| error.to_string())?; - write_timings.push(write_start.elapsed()); - let read_start = Instant::now(); - let roundtrip = read_document(&path).map_err(|error| error.to_string())?; - read_timings.push(read_start.elapsed()); - fingerprint_matches &= fingerprint_document(&roundtrip) == original; - Ok::<_, String>(()) - })(); - if let Err(error) = result { - last_error = Some(error); - break; - } - } - - let _ = writeln!(out, "### Native Document Roundtrip"); - let _ = writeln!(out); - match last_error { - Some(error) => { - let _ = writeln!(out, "- status: `failed`"); - let _ = writeln!(out, "- error: `{}`", md(&error)); - }, - None => { - let write_stats = DurationStats::from_samples(&write_timings); - let read_stats = DurationStats::from_samples(&read_timings); - let _ = writeln!(out, "| phase | min ms | mean ms | max ms | fingerprint match |"); - let _ = writeln!(out, "|---|---:|---:|---:|---|"); - let _ = writeln!( - out, - "| write_document | {:.3} | {:.3} | {:.3} | n/a |", - ms(write_stats.min), - ms(write_stats.mean), - ms(write_stats.max) - ); - let _ = writeln!( - out, - "| read_document roundtrip | {:.3} | {:.3} | {:.3} | {} |", - ms(read_stats.min), - ms(read_stats.mean), - ms(read_stats.max), - fingerprint_matches - ); - }, - } - let _ = writeln!(out); -} diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/runner.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/runner.rs index 925d5602..8db34316 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/runner.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/runner.rs @@ -9,8 +9,6 @@ use std::{ }; use gpui::{Bounds, Context, IntoElement, Pixels, Render, Window, div, point, prelude::*, px, size}; -use tempfile::tempdir; - use super::*; const DEFAULT_WIDTHS: &[f32] = &[720.0, 900.0, 1100.0, 1440.0]; @@ -143,7 +141,7 @@ struct LoadedDocument { label: String, path: Option, file_bytes: Option, - document: Document, + document: DocumentProjection, load: DurationStats, } diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/sources_stats.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/sources_stats.rs index 9509ca70..70de10a6 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/sources_stats.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/sources_stats.rs @@ -9,19 +9,7 @@ fn benchmark_sources(options: &BenchmarkOptions) -> Vec { .collect(); } - let mut paths = fs::read_dir(".") - .ok() - .into_iter() - .flat_map(|entries| entries.filter_map(Result::ok)) - .map(|entry| entry.path()) - .filter(|path| path.extension().is_some_and(|extension| extension == DEFAULT_DOCUMENT_EXTENSION)) - .collect::>(); - paths.sort(); - if paths.is_empty() { - vec![BenchmarkSource::Demo] - } else { - paths.into_iter().map(BenchmarkSource::Path).collect() - } + vec![BenchmarkSource::Demo] } #[hotpath::measure] @@ -33,7 +21,12 @@ fn load_document_source(source: &BenchmarkSource, iterations: usize) -> Result read_document(path).map_err(|error| error.to_string())?, + BenchmarkSource::Path(path) => { + return Err(format!( + "loading {} requires a host Loro package adapter", + path.display() + )); + }, BenchmarkSource::Demo => demo_document(), }; timings.push(started.elapsed()); @@ -66,7 +59,7 @@ fn source_label(source: &BenchmarkSource) -> String { } #[hotpath::measure] -fn document_stats(document: &Document) -> DocumentStats { +fn document_stats(document: &DocumentProjection) -> DocumentStats { let mut stats = DocumentStats { text_bytes: document.text.byte_len(), text_chars: full_document_text(document).chars().count(), @@ -151,7 +144,7 @@ fn accumulate_table_stats(table: &TableBlock, stats: &mut DocumentStats, nested: } #[hotpath::measure] -fn check_document_fidelity(document: &Document) -> FidelityReport { +fn check_document_fidelity(document: &DocumentProjection) -> FidelityReport { let mut report = FidelityReport::default(); let full_text = full_document_text(document); report.check(!document.paragraphs.is_empty(), "document must contain at least one paragraph"); @@ -226,7 +219,7 @@ fn check_table_fidelity(table: &TableBlock, report: &mut FidelityReport, label: .max() .unwrap_or_default(); report.check( - table.column_widths.is_empty() || table.column_widths.len() == widest_row, + table.columns.is_empty() || table.columns.len() == widest_row, format!("{label} column width count should match widest row when explicit widths are present"), ); for (row_ix, row) in table.rows.iter().enumerate() { @@ -239,7 +232,7 @@ fn check_table_fidelity(table: &TableBlock, report: &mut FidelityReport, label: match block { TableCellBlock::Paragraph(paragraph) => { report.check( - paragraph.paragraph.byte_range.len() == paragraph.text.len(), + crate::paragraph_runs_len(¶graph.paragraph) == paragraph.text.len(), format!("{label} cell {row_ix}:{cell_ix} paragraph {block_ix} byte range must match cell text"), ); }, diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/suite.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/suite.rs index 04a1bf4a..99e758d5 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/suite.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/suite.rs @@ -31,7 +31,7 @@ fn run_benchmark_suite(options: &BenchmarkOptions, window: &mut Window, cx: &mut let sources = benchmark_sources(options); if sources.is_empty() { - let _ = writeln!(report, "No `.gptx` files or explicit benchmark documents were found."); + let _ = writeln!(report, "No explicit benchmark documents were found."); return report; } @@ -157,7 +157,6 @@ fn benchmark_document( write_stats_tables(&mut out, stats); write_fidelity_report(&mut out, &fidelity); - write_roundtrip_report(&mut out, loaded, iterations); let index_rows = benchmark_index_paths(document, iterations); write_operation_table(&mut out, "Index And Mapping Benchmarks", &index_rows); diff --git a/crates/gpui-flowtext/src/rich_text/benchmarks/utilities.rs b/crates/gpui-flowtext/src/rich_text/benchmarks/utilities.rs index 0072d69c..6f479955 100644 --- a/crates/gpui-flowtext/src/rich_text/benchmarks/utilities.rs +++ b/crates/gpui-flowtext/src/rich_text/benchmarks/utilities.rs @@ -1,5 +1,5 @@ #[hotpath::measure] -fn first_window_range(document: &Document, paragraph_count: usize) -> Range { +fn first_window_range(document: &DocumentProjection, paragraph_count: usize) -> Range { let end_paragraph = document .paragraphs .len() @@ -12,25 +12,25 @@ fn first_window_range(document: &Document, paragraph_count: usize) -> Range Option { +fn top_selection(document: &DocumentProjection) -> Option { if document.paragraphs.is_empty() { return None; } - Some(EditorSelection { - anchor: DocumentOffset { paragraph: 0, byte: 0 }, - head: first_window_range(document, 3).end, - }) + Some(EditorSelection::range( + DocumentOffset { paragraph: 0, byte: 0 }, + first_window_range(document, 3).end, + )) } #[hotpath::measure] -fn first_char_range(document: &Document, paragraph_ix: usize) -> Option> { +fn first_char_range(document: &DocumentProjection, paragraph_ix: usize) -> Option> { let text = paragraph_text(document, paragraph_ix); let ch = text.chars().next()?; Some(0..ch.len_utf8()) } #[hotpath::measure] -fn safe_mid_byte(document: &Document, paragraph_ix: usize) -> usize { +fn safe_mid_byte(document: &DocumentProjection, paragraph_ix: usize) -> usize { let text = paragraph_text(document, paragraph_ix); if text.is_empty() { return 0; @@ -45,7 +45,7 @@ fn safe_mid_byte(document: &Document, paragraph_ix: usize) -> usize { } #[hotpath::measure] -fn fingerprint_document(document: &Document) -> u64 { +fn fingerprint_document(document: &DocumentProjection) -> u64 { let mut hasher = std::collections::hash_map::DefaultHasher::new(); for chunk in document.text.chunks() { chunk.hash(&mut hasher); @@ -129,8 +129,8 @@ fn hash_paragraph(paragraph: &Paragraph, hasher: &mut impl Hasher) { fn hash_table(table: &TableBlock, hasher: &mut impl Hasher) { table.version.hash(hasher); table.style.header_row.hash(hasher); - table.column_widths.len().hash(hasher); - for width in &table.column_widths { + table.columns.len().hash(hasher); + for width in table.columns.iter().map(|column| &column.width) { match width { TableColumnWidth::Auto => 0u8.hash(hasher), TableColumnWidth::FixedPx(value) => { diff --git a/crates/gpui-flowtext/src/rich_text/editor/action_handlers.rs b/crates/gpui-flowtext/src/rich_text/editor/action_handlers.rs index 267e82ec..fe5bf744 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/action_handlers.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/action_handlers.rs @@ -233,29 +233,39 @@ impl RichTextEditor { key_char.to_string() }; if self.insert_text_into_selected_table_cell(&key_char, cx) { + cx.stop_propagation(); return; } if self.insert_text_into_selected_equation(&key_char, cx) { + cx.stop_propagation(); return; } self.insert_text_command(&key_char, cx); + cx.stop_propagation(); } #[cfg(not(target_os = "windows"))] { let _ = window; if self.insert_text_into_selected_table_cell(key_char, cx) { + cx.stop_propagation(); return; } if self.insert_text_into_selected_equation(key_char, cx) { + cx.stop_propagation(); return; } self.insert_text_command(key_char, cx); + cx.stop_propagation(); } } + /// Loro-first: the edit primitives inside the closure commit through the + /// write authority themselves — there is nothing to capture, diff, replay, + /// or locally record. This wrapper survives only as a call-site-stable + /// pass-through. pub(super) fn apply_document_edit(&mut self, cx: &mut Context, edit: impl FnOnce(&mut Self, &mut Context)) { - self.apply_document_edit_with_capture_range(cx, None, edit); + edit(self, cx); } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/block_helpers.rs b/crates/gpui-flowtext/src/rich_text/editor/block_helpers.rs deleted file mode 100644 index aeaaa87c..00000000 --- a/crates/gpui-flowtext/src/rich_text/editor/block_helpers.rs +++ /dev/null @@ -1,50 +0,0 @@ -#[hotpath::measure] -fn insert_standalone_paragraphs_into_projection( - document: &mut Document, - insert_paragraph_ix: usize, - inserted: &[InputParagraph], -) -> Vec { - if inserted.is_empty() { - return Vec::new(); - } - let mut entries = document - .paragraphs - .iter() - .enumerate() - .map(|(paragraph_ix, paragraph)| (paragraph.clone(), paragraph_text(document, paragraph_ix))) - .collect::>(); - let inserted_entries = inserted - .iter() - .map(|paragraph| { - let text = input_paragraph_text(paragraph); - (paragraph_from_input_paragraph(paragraph), text) - }) - .collect::>(); - let insert_ix = insert_paragraph_ix.min(entries.len()); - entries.splice(insert_ix..insert_ix, inserted_entries.clone()); - for relative_ix in 0..inserted.len() { - insert_paragraph_id(document, insert_ix + relative_ix); - } - - let mut text = String::new(); - let mut byte = 0; - let mut paragraphs = Vec::with_capacity(entries.len()); - for (ix, (mut paragraph, paragraph_text)) in entries.into_iter().enumerate() { - if ix > 0 { - text.push('\n'); - byte += 1; - } - let start = byte; - text.push_str(¶graph_text); - byte += paragraph_text.len(); - paragraph.byte_range = start..byte; - paragraphs.push(paragraph); - } - let inserted_paragraphs = paragraphs[insert_ix..insert_ix + inserted.len()].to_vec(); - document.text = Rope::from(text); - document.paragraphs = Arc::new(paragraphs); - document.offset_index = ParagraphOffsetIndex::new(&document.paragraphs); - rebuild_document_sections(document); - inserted_paragraphs -} - diff --git a/crates/gpui-flowtext/src/rich_text/editor/block_insertion.rs b/crates/gpui-flowtext/src/rich_text/editor/block_insertion.rs index bb01a648..9479b84d 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/block_insertion.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/block_insertion.rs @@ -14,332 +14,98 @@ impl RichTextEditor { if self.insert_rich_fragment_into_selected_table_cell(&fragment, cx) { return; } - if self.insert_rich_fragment_paste_at_caret(&fragment, cx) { - return; - } - self.apply_document_edit(cx, |editor, cx| editor.insert_rich_fragment(fragment, cx)); + let _ = self.write_clipboard_fragment_at_caret(&fragment, cx); } pub fn insert_toolkit_paragraphs_as_blocks(&mut self, paragraphs: Vec, cx: &mut Context) { let blocks = non_empty_input_paragraphs(paragraphs) .into_iter() - .map(InputBlock::Paragraph) + .map(FragmentBlock::Paragraph) .collect::>(); if blocks.is_empty() { return; } - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: Vec::new(), - blocks, - assets: Vec::new(), - }; - self.insert_block_fragment(fragment, cx); + self.write_insert_rich_fragment_at_caret(blocks, cx); } + /// Loro-first (spec §5): a rich fragment commits as ONE `InsertRichFragment` + /// intent through the write authority — paragraphs and objects in order, + /// one gate hold, one Loro commit, one undo unit. The conversion itself + /// lives in `write_clipboard_fragment_at_caret` (paste.rs), shared with the + /// clipboard paste path. fn insert_rich_fragment(&mut self, fragment: RichClipboardFragment, cx: &mut Context) { - if !fragment.blocks.is_empty() { - self.insert_block_fragment(fragment, cx); - return; - } - if fragment.paragraphs.is_empty() { - return; - } - if !self.selection.is_caret() { - self.delete_selection_internal(); - } - let caret = insert_rich_fragment_at(&mut self.document, self.selection.head, &fragment); - self.selection = EditorSelection { anchor: caret, head: caret }; - self.after_text_mutation(cx); - } - - fn insert_block_fragment(&mut self, fragment: RichClipboardFragment, cx: &mut Context) { - if fragment.blocks.is_empty() { - return; - } - let before_document = self.document.clone(); - let before_selection = self.selection.clone(); - for asset in fragment.assets { - self.document.assets.assets.insert( - asset.id, - AssetRecord { - id: asset.id, - mime_type: asset.mime_type.into(), - original_name: asset.original_name.map(Into::into), - content_hash: asset.content_hash, - bytes: Arc::new(asset.bytes), - }, - ); - } - self.insert_ordered_block_fragment_after_caret(&fragment.blocks); - self.push_replace_document_history(before_document, before_selection, cx); - } - - fn insert_ordered_block_fragment_after_caret(&mut self, input_blocks: &[InputBlock]) { - let insert_ix = self.prepare_block_insertion_index(); - let insert_paragraph_ix = self - .document - .blocks - .iter() - .take(insert_ix) - .filter(|block| matches!(block, Block::Paragraph(_))) - .count(); - let inserted_paragraph_inputs = input_blocks - .iter() - .filter_map(|block| match block { - InputBlock::Paragraph(paragraph) => Some(paragraph.clone()), - InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_) => None, - }) - .collect::>(); - let inserted_paragraphs = insert_standalone_paragraphs_into_projection(&mut self.document, insert_paragraph_ix, &inserted_paragraph_inputs); - let mut inserted_paragraph_ix = 0; - let inserted_blocks = input_blocks - .iter() - .map(|block| match block { - InputBlock::Paragraph(_) => { - let paragraph = inserted_paragraphs - .get(inserted_paragraph_ix) - .cloned() - .unwrap_or_else(|| Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }); - inserted_paragraph_ix += 1; - Block::Paragraph(paragraph) - }, - InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_) => block_from_input_block(block), - }) - .collect::>(); - let old_blocks = self.document.blocks.as_ref().clone(); - let old_block_ids = self.document.ids.block_ids.clone(); - let mut paragraph_ix = 0; - let mut output = Vec::with_capacity(old_blocks.len() + inserted_blocks.len()); - let mut output_block_ids = Vec::with_capacity(old_blocks.len() + inserted_blocks.len()); - for (block_ix, block) in old_blocks.iter().enumerate() { - if block_ix == insert_ix { - output.extend(inserted_blocks.iter().cloned()); - output_block_ids.extend((0..inserted_blocks.len()).map(|_| new_block_id())); - } - match block { - Block::Paragraph(_) => { - if let Some(paragraph) = self.document.paragraphs.get(paragraph_ix) { - output.push(Block::Paragraph(paragraph.clone())); - output_block_ids.push(old_block_ids.get(block_ix).copied().unwrap_or_else(new_block_id)); - } - paragraph_ix += 1; - }, - Block::Image(_) | Block::Equation(_) | Block::Table(_) => { - output.push(block.clone()); - output_block_ids.push(old_block_ids.get(block_ix).copied().unwrap_or_else(new_block_id)); - }, - } - } - if insert_ix >= old_blocks.len() { - output.extend(inserted_blocks); - output_block_ids.extend((0..input_blocks.len()).map(|_| new_block_id())); - } - self.document.blocks = Arc::new(output); - self.document.ids.block_ids = output_block_ids; - rebuild_document_sections(&mut self.document); - self.selected_block = None; - self.clear_layout_work_caches(); - self.item_sizes_cache = None; - self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); + let _ = self.write_clipboard_fragment_at_caret(&fragment, cx); } + /// Insert projection blocks at the caret through the write authority (menu + /// insertions: tables.rs, media.rs). Object placement is the runtime's law: + /// the object lands before the caret paragraph at byte 0, after it + /// otherwise (`InsertObject` resolution in the write path). fn insert_blocks_after_caret(&mut self, blocks: Vec, cx: &mut Context) { - if blocks.is_empty() { - return; - } - let before_document = self.document.clone(); - let before_selection = self.selection.clone(); - self.insert_blocks_after_caret_without_history(blocks); - self.push_replace_document_history(before_document, before_selection, cx); + let inputs = blocks.iter().map(input_block_from_block).collect::>(); + let _ = self.write_input_blocks_at_caret(inputs, cx); } - fn insert_blocks_after_caret_without_history(&mut self, blocks: Vec) { + /// Input blocks → intents: a lone object is the precise (non-compound) + /// `InsertObject` intent; anything else is one `InsertRichFragment`. + pub(super) fn write_input_blocks_at_caret(&mut self, mut blocks: Vec, cx: &mut Context) -> bool { if blocks.is_empty() { - return; + return false; } - let insert_ix = self.prepare_block_insertion_index(); - let inserted_count = blocks.len(); - Arc::make_mut(&mut self.document.blocks).splice(insert_ix..insert_ix, blocks); - for relative_ix in 0..inserted_count { - insert_block_id(&mut self.document, insert_ix + relative_ix); + if blocks.len() == 1 && !matches!(blocks[0], InputBlock::Paragraph(_)) { + let block = blocks.remove(0); + return self.write_object_block_replacing_selection(block, cx); } - self.append_missing_paragraph_blocks(); - rebuild_document_sections(&mut self.document); - self.selected_block = None; - self.clear_layout_work_caches(); - self.item_sizes_cache = None; - self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); + let fragment_blocks = blocks + .into_iter() + .map(|block| match block { + InputBlock::Paragraph(paragraph) => FragmentBlock::Paragraph(paragraph), + block => FragmentBlock::Object(block), + }) + .collect::>(); + self.write_insert_rich_fragment_at_caret(fragment_blocks, cx) } - fn prepare_block_insertion_index(&mut self) -> usize { - if let Some( - BlockSelection::Image(block_ix) - | BlockSelection::Equation(block_ix) - | BlockSelection::Table(block_ix) - | BlockSelection::TableCell { block_ix, .. }, - ) = self.selected_block - { - return (block_ix + 1).min(self.document.blocks.len()); - } - - if let Some(insert_ix) = self.remove_empty_caret_paragraph_for_block_insertion() { - return insert_ix; - } - - if !self.selection.is_caret() { - let range = self.selection.normalized(); - let object_indices = self.object_block_indices_in_text_range(range); - if !object_indices.is_empty() { - { - let blocks = Arc::make_mut(&mut self.document.blocks); - for block_ix in object_indices.iter().copied().rev() { - if block_ix < blocks.len() { - blocks.remove(block_ix); - } - } - } - for block_ix in object_indices.into_iter().rev() { - remove_block_ids(&mut self.document, block_ix..block_ix + 1); - } + /// One user edit = one undo group: replacing a selection with an object + /// groups the delete with the `InsertObject` intent (mirrors + /// `write_insert_text_at_caret`'s selection-replacement law). + pub(super) fn write_object_block_replacing_selection(&mut self, block: InputBlock, cx: &mut Context) -> bool { + let grouped = !self.selection.is_caret(); + if grouped { + self.begin_undo_group(); + if !self.write_delete_selection(cx) { + self.end_undo_group(); + return false; } - self.delete_selection_internal(); } - - if let Some(position) = document_position_for_offset(&self.document, self.selection.head) { - debug_assert_eq!(document_offset_for_position(&self.document, &position), Some(self.selection.head)); - if let DocumentPosition::Text { block_ix, .. } = position { - return (block_ix + 1).min(self.document.blocks.len()); - } + let committed = self.write_insert_object_at_caret(block, cx); + if grouped { + self.end_undo_group(); } - self.document.blocks.len() + committed } - fn remove_empty_caret_paragraph_for_block_insertion(&mut self) -> Option { - if !self.selection.is_caret() { - return None; - } - let paragraph_ix = self.selection.head.paragraph; - let paragraph = self.document.paragraphs.get(paragraph_ix)?; - if self.selection.head.byte != 0 || paragraph_text_len(paragraph) != 0 { - return None; - } - let block_ix = self.block_ix_for_paragraph(paragraph_ix)?; - let paragraph_count = self.document.paragraphs.len(); - { - let blocks = Arc::make_mut(&mut self.document.blocks); - if block_ix < blocks.len() { - blocks.remove(block_ix); - } - } - remove_block_ids(&mut self.document, block_ix..block_ix + 1); - - if paragraph_count > 1 { - let range = paragraph_byte_range(&self.document, paragraph_ix); - if paragraph_ix + 1 < paragraph_count { - self.document.text.delete(range.start..range.start + 1); - } else if range.start > 0 { - self.document.text.delete(range.start - 1..range.start); - } - paragraphs_mut(&mut self.document).remove(paragraph_ix); - remove_paragraph_ids(&mut self.document, paragraph_ix..paragraph_ix + 1); - rebuild_document_offset_index(&mut self.document); - rebuild_document_sections(&mut self.document); - let new_paragraph_ix = paragraph_ix.min(self.document.paragraphs.len().saturating_sub(1)); - self.selection = EditorSelection { - anchor: DocumentOffset { - paragraph: new_paragraph_ix, - byte: 0, - }, - head: DocumentOffset { - paragraph: new_paragraph_ix, - byte: 0, - }, + /// Clipboard fragments carry raw asset bytes (images). There is no asset + /// intent (assets are content-addressed sideband bytes, not CRDT content), + /// so pasted bytes seed the projection's asset stores directly — the same + /// pattern as `insert_image_assets` (media.rs) and the remote + /// `AssetArrived` path (`projection_apply.rs`). The object blocks that + /// reference them commit through the write authority like everything else. + fn adopt_clipboard_assets(&mut self, assets: &[InputAsset]) { + for asset in assets { + let record = AssetRecord { + id: asset.id, + mime_type: asset.mime_type.clone().into(), + original_name: asset.original_name.clone().map(Into::into), + content_hash: asset.content_hash, + bytes: Arc::new(asset.bytes.clone()), }; + self.document.assets.assets.insert(asset.id, record); } - Some(block_ix) - } - - fn append_missing_paragraph_blocks(&mut self) { - let existing = self - .document - .blocks - .iter() - .filter(|block| matches!(block, Block::Paragraph(_))) - .count(); - if existing >= self.document.paragraphs.len() { - return; - } - let inserted_count = self.document.paragraphs.len() - existing; - { - let blocks = Arc::make_mut(&mut self.document.blocks); - for paragraph in self.document.paragraphs.iter().skip(existing) { - blocks.push(Block::Paragraph(paragraph.clone())); - } - } - self.document.ids.block_ids.extend((0..inserted_count).map(|_| new_block_id())); - rebuild_document_sections(&mut self.document); - } - - fn push_replace_document_history(&mut self, before_document: Document, before_selection: EditorSelection, cx: &mut Context) { - if before_document.text == self.document.text - && before_document.paragraphs == self.document.paragraphs - && before_document.blocks == self.document.blocks - && before_document.assets == self.document.assets - { - return; - } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceDocument { - before: Box::new(before_document), - after: Box::new(self.document.clone()), - }], - canonical_operations: vec![CanonicalOperation::ReplaceDocument], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); } fn insert_plain_text_fragment(&mut self, text: &str, cx: &mut Context) { - let normalized = text.replace("\r\n", "\n").replace('\r', "\n"); - if normalized.is_empty() { - return; - } - let paragraph_style = self.document.paragraphs[self.selection.head.paragraph].style; - let styles = self.styles_at_caret(); - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: normalized - .split('\n') - .map(|line| InputParagraph { - style: paragraph_style, - runs: if line.is_empty() { - Vec::new() - } else { - vec![InputRun { - text: line.to_string(), - styles, - }] - }, - }) - .collect(), - blocks: Vec::new(), - assets: Vec::new(), - }; - self.insert_rich_fragment(fragment, cx); + let _ = self.write_plain_text_paste(text, cx); } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/caret_movement.rs b/crates/gpui-flowtext/src/rich_text/editor/caret_movement.rs index 09d57050..474fe097 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/caret_movement.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/caret_movement.rs @@ -102,19 +102,29 @@ impl RichTextEditor { } }, }; - let anchor = if extend { self.selection.anchor } else { new_head }; - let selection = EditorSelection { anchor, head: new_head }; - if self.selection == selection { + // §16: a rightward step parks the caret after the preceding glyph, a + // leftward step before the following glyph. Gravity stays neutral — arrow + // motion keeps the historical wrap-seam bias. + let head_affinity = match dir { + HDir::Left => SelectionAffinity::Before, + HDir::Right => SelectionAffinity::After, + }; + let selection = self.selection.moved(new_head, head_affinity, VisualGravity::Neutral, extend); + if self.selection.same_positions(&selection) { self.goal_x = None; return; } + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("move_horizontal", &fid_before); self.goal_x = None; let width = self.current_layout_width(); let _ = self.ensure_paragraph_chunk_containing_byte(new_head.paragraph, new_head.byte, width, window, cx); let _ = self.paragraph_item_sizes(window, cx); self.scroll_head_into_view(); self.reset_caret_blink(cx); + self.emit_selection_changed(cx); cx.notify(); } @@ -164,7 +174,7 @@ impl RichTextEditor { let Some(layout) = self.layout_for_offset(head) else { return; }; - let Some((p_ix, l_ix)) = locate_line(&layout, head) else { + let Some((p_ix, l_ix)) = locate_line(&layout, head, self.selection.head_gravity) else { cx.notify(); return; }; @@ -187,18 +197,23 @@ impl RichTextEditor { }; (new_head, cur_x) }; - let anchor = if extend { self.selection.anchor } else { new_head }; - let selection = EditorSelection { anchor, head: new_head }; - if self.selection == selection { + // Vertical motion lands on a fresh column position; reset to neutral + // affinity/gravity. + let selection = self.selection.moved(new_head, SelectionAffinity::Neutral, VisualGravity::Neutral, extend); + if self.selection.same_positions(&selection) { self.goal_x = Some(used_goal_x); return; } + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("move_vertical", &fid_before); // Preserve the goal x across the move so repeated Up/Down stays on a // straight column. self.goal_x = Some(used_goal_x); self.scroll_head_into_view(); self.reset_caret_blink(cx); + self.emit_selection_changed(cx); cx.notify(); } @@ -251,11 +266,14 @@ impl RichTextEditor { paragraph: target_paragraph, byte: target_byte, }; - let anchor = if extend { self.selection.anchor } else { new_head }; - self.selection = EditorSelection { anchor, head: new_head }; + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); + self.selection = self.selection.moved(new_head, SelectionAffinity::Neutral, VisualGravity::Neutral, extend); + self.fidelity_caret_set("move_to_adjacent_unmounted_paragraph", &fid_before); self.goal_x = Some(goal_x); self.scroll_head_into_view(); self.reset_caret_blink(cx); + self.emit_selection_changed(cx); cx.notify(); } diff --git a/crates/gpui-flowtext/src/rich_text/editor/chunk_layout.rs b/crates/gpui-flowtext/src/rich_text/editor/chunk_layout.rs index 28713862..eba7b1cd 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/chunk_layout.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/chunk_layout.rs @@ -1,7 +1,13 @@ #[hotpath::measure_all] impl RichTextEditor { - fn valid_chunk_cache_entry(&self, paragraph_ix: usize, width: Pixels) -> Option<&ParagraphChunkLayoutCacheEntry> { + // §act-nine A9.3: chunk validity is the content key (style, version) + the + // stable paragraph id + width/invisibility/layout generation — no global + // `edit_generation`, so edits to other paragraphs keep these chunks. The id + // check matters because this cache is positional and (style, version) alone + // can collide across different paragraphs after a row shift. + pub(super) fn valid_chunk_cache_entry(&self, paragraph_ix: usize, width: Pixels) -> Option<&ParagraphChunkLayoutCacheEntry> { let paragraph = self.document.paragraphs.get(paragraph_ix)?; + let paragraph_id = self.paragraph_id_at(paragraph_ix)?; let key = paragraph_cache_key(&self.document, paragraph); self .paragraph_chunk_layout_cache @@ -9,17 +15,35 @@ impl RichTextEditor { .and_then(|entry| entry.as_ref()) .filter(|entry| { entry.key == key + && entry.paragraph_id == paragraph_id && entry.width == width && entry.invisibility_mode == self.invisibility_mode - && entry.edit_generation == self.edit_generation && entry.layout_generation == self.layout_generation }) } + /// Height-cache validity (§act-nine A9.3): content key + stable id + width + + /// invisibility mode — the ONE check both read sites share. No global + /// `edit_generation`; theme/zoom/width changes clear the map wholesale. + pub(super) fn valid_paragraph_height(&self, paragraph_ix: usize, width: Pixels) -> Option { + let paragraph = self.document.paragraphs.get(paragraph_ix)?; + let paragraph_id = self.paragraph_id_at(paragraph_ix)?; + let key = paragraph_cache_key(&self.document, paragraph); + self + .paragraph_height_cache + .get(paragraph_ix) + .and_then(|entry| *entry) + .filter(|entry| entry.key == key && entry.paragraph_id == paragraph_id && entry.width == width && entry.invisibility_mode == self.invisibility_mode) + .map(|entry| entry.height) + } + fn ensure_current_chunk_cache_entry(&mut self, paragraph_ix: usize, width: Pixels) -> bool { let Some(paragraph) = self.document.paragraphs.get(paragraph_ix) else { return false; }; + let Some(paragraph_id) = self.paragraph_id_at(paragraph_ix) else { + return false; + }; self .paragraph_chunk_layout_cache .resize(self.document.paragraphs.len(), None); @@ -30,9 +54,9 @@ impl RichTextEditor { .and_then(|entry| entry.as_ref()) .is_none_or(|entry| { entry.key != key + || entry.paragraph_id != paragraph_id || entry.width != width || entry.invisibility_mode != self.invisibility_mode - || entry.edit_generation != self.edit_generation || entry.layout_generation != self.layout_generation }); if reset { @@ -45,9 +69,9 @@ impl RichTextEditor { } self.paragraph_chunk_layout_cache[paragraph_ix] = Some(ParagraphChunkLayoutCacheEntry { key, + paragraph_id, width, invisibility_mode: self.invisibility_mode, - edit_generation: self.edit_generation, layout_generation: self.layout_generation, prep, chunks: Vec::new(), @@ -155,15 +179,17 @@ impl RichTextEditor { .resize(self.document.paragraphs.len(), None); self.paragraph_height_cache[paragraph_ix] = Some(ParagraphHeightCacheEntry { key: entry.key, + paragraph_id: entry.paragraph_id, width, invisibility_mode: self.invisibility_mode, - edit_generation: self.edit_generation, height: entry.exact_height, }); } if bump_revision { self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); - self.item_sizes_cache = None; + // Patch, don't nuke: only THIS paragraph's items changed (see + // `note_item_sizes_patch_paragraph`). + self.note_item_sizes_patch_paragraph(paragraph_ix); } true } diff --git a/crates/gpui-flowtext/src/rich_text/editor/chunk_materialization.rs b/crates/gpui-flowtext/src/rich_text/editor/chunk_materialization.rs index 33aa5285..4799363a 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/chunk_materialization.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/chunk_materialization.rs @@ -1,3 +1,10 @@ +/// Max consecutive render-path materialization passes allowed at an unchanged +/// (scroll, edit) signature before the loop is broken. Progressive materialization +/// of a viewport converges in a handful of frames; a non-converging item-size loop +/// runs thousands, so this cleanly separates the two while leaving ample headroom +/// for a legitimately complex viewport (it just falls back to estimated heights). +const SCROLL_MATERIALIZE_STALL_CAP: u32 = 30; + #[hotpath::measure_all] impl RichTextEditor { fn materialize_visible_remainders_for_scroll( @@ -7,6 +14,32 @@ impl RichTextEditor { window: &mut Window, cx: &mut Context, ) -> bool { + // §hang-guard: this runs during render and `cx.notify()`s whenever it + // materializes anything, re-entering next frame. On a large object-bearing + // document the item-size cache can't be incrementally patched, so it can + // re-materialize the same visible window every frame without converging and + // freeze the window. Cap consecutive passes at an unchanged scroll+edit + // position; a real scroll or edit changes the signature and resets the count, + // so only the pathological non-converging loop is ever broken (it then renders + // with estimated heights, which the virtualization already tolerates). + let scroll_px: f32 = (-self.scroll_handle.offset().y).max(px(0.0)).into(); + #[allow(clippy::cast_possible_truncation, reason = "rounded pixel scroll offset fits i32 with ample range")] + let scroll_y = scroll_px.round() as i32; + let signature = (scroll_y, self.edit_generation); + if self.scroll_materialize_signature == Some(signature) { + self.scroll_materialize_stall_frames = self.scroll_materialize_stall_frames.saturating_add(1); + } else { + self.scroll_materialize_signature = Some(signature); + self.scroll_materialize_stall_frames = 1; + } + if self.scroll_materialize_stall_frames > SCROLL_MATERIALIZE_STALL_CAP { + if self.scroll_materialize_stall_frames == SCROLL_MATERIALIZE_STALL_CAP + 1 { + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Structure, "scroll-materialize-stall-cap", || { + format!("broke non-converging materialization loop at scroll_y={scroll_y} after {SCROLL_MATERIALIZE_STALL_CAP} passes") + }); + } + return false; + } let scroll_anchor = scroll_anchor.or_else(|| self.capture_scroll_anchor()); let overscan = px(SCROLL_FOREGROUND_OVERSCAN_PX); let Some(visible_range) = self.visible_item_range_for_current_scroll(overscan) else { @@ -50,7 +83,12 @@ impl RichTextEditor { } if changed { - let _ = self.rebuild_item_sizes_cache(width, scroll_anchor, window, cx); + // Patch, don't rebuild: the materialized paragraphs recorded themselves + // in `pending_item_sizes_patch_range`; a full O(blocks) rebuild per + // scroll tick was ~10 ms of every scrolled frame on the reference doc. + if self.try_patch_item_sizes_cache(width, scroll_anchor.clone(), window, cx).is_none() { + let _ = self.rebuild_item_sizes_cache(width, scroll_anchor, window, cx); + } cx.notify(); } changed diff --git a/crates/gpui-flowtext/src/rich_text/editor/chunk_navigation.rs b/crates/gpui-flowtext/src/rich_text/editor/chunk_navigation.rs index f2c2688c..1864cc42 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/chunk_navigation.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/chunk_navigation.rs @@ -93,20 +93,24 @@ impl RichTextEditor { self.resize_layout_aux_caches(); let paragraph = self.document.paragraphs.get(paragraph_ix)?; let key = paragraph_cache_key(&self.document, paragraph); + // §perf-heaven T5: identity-scoped validity — the estimate survives edits to + // OTHER paragraphs (their key/id are unchanged), so a keystroke recomputes + // only the touched paragraph's estimate instead of the whole document. + let paragraph_id = self.document.ids.paragraph_ids.get(paragraph_ix).copied().unwrap_or(ParagraphId(0)); let expected = ParagraphEstimateHeightCacheEntry { key, + paragraph_id, width, invisibility_mode: self.invisibility_mode, - edit_generation: self.edit_generation, layout_generation: self.layout_generation, height: px(0.0), source_len: 0, }; - if let Some(entry) = self.paragraph_estimate_height_cache.get(paragraph_ix).and_then(|entry| *entry) + if let Some(entry) = self.paragraph_estimate_height_cache.get(¶graph_id).copied() && entry.key == expected.key + && entry.paragraph_id == expected.paragraph_id && entry.width == expected.width && entry.invisibility_mode == expected.invisibility_mode - && entry.edit_generation == expected.edit_generation && entry.layout_generation == expected.layout_generation { return Some((entry.height, entry.source_len)); @@ -114,19 +118,22 @@ impl RichTextEditor { let prep = self.valid_paragraph_prep(paragraph_ix); let (height, source_len) = match prep.as_deref() { - Some(prep) => (estimate_paragraph_prep_item_height(&self.document, prep, width), prep.source_len), + Some(prep) => (estimate_paragraph_prep_item_height(&self.document, prep, paragraph_ix, width), prep.source_len), None => ( estimate_paragraph_item_height_with_visibility(&self.document, paragraph_ix, width, self.invisibility_mode), paragraph_text_len(paragraph), ), }; - if let Some(slot) = self.paragraph_estimate_height_cache.get_mut(paragraph_ix) { - *slot = Some(ParagraphEstimateHeightCacheEntry { + // §perf-heaven T7.14: keyed by the stable paragraph id, so a later + // structural shift keeps this estimate instead of invalidating it. + self.paragraph_estimate_height_cache.insert( + paragraph_id, + ParagraphEstimateHeightCacheEntry { height, source_len, ..expected - }); - } + }, + ); Some((height, source_len)) } diff --git a/crates/gpui-flowtext/src/rich_text/editor/chunk_prefetch.rs b/crates/gpui-flowtext/src/rich_text/editor/chunk_prefetch.rs index 97fa085f..c251edc6 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/chunk_prefetch.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/chunk_prefetch.rs @@ -121,9 +121,26 @@ impl RichTextEditor { .valid_chunk_cache_entry(paragraph_ix, width) .map(|entry| entry.chunks.len()) .unwrap_or(before); - changed |= after != before; + let progressed = after != before; + changed |= progressed; + if progressed { + self.note_item_sizes_patch_paragraph(paragraph_ix); + } if self.paragraph_needs_chunk_prefetch(paragraph_ix, width) { - self.chunk_prefetch_queue.push_back(paragraph_ix); + if progressed { + self.chunk_prefetch_queue.push_back(paragraph_ix); + } else { + // The paragraph still reports "needs prefetch" but ensuring the next + // chunk added NOTHING (`after == before`) — the chunker cannot advance + // it to `complete`. Re-queuing it here would keep the queue non-empty, + // which reschedules the `on_next_frame` prefetch below every frame: + // a continuous re-render with no state change that FREEZES the window + // (observed on large documents with object blocks). Drop it instead so + // the queue can drain; the paragraph renders with the chunks it has. + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Structure, "prefetch-no-progress", || { + format!("paragraph {paragraph_ix} needs prefetch but chunking made no progress ({before} chunks); dropped to break the render loop") + }); + } } } if start.elapsed() >= budget { @@ -136,8 +153,13 @@ impl RichTextEditor { } if changed { self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); - self.item_sizes_cache = None; - let _ = self.rebuild_item_sizes_cache_with_prefetch(width, scroll_anchor, false, window, cx); + // Patch, don't nuke: the loop above recorded every refined paragraph in + // `pending_item_sizes_patch_range` (see `note_item_sizes_patch_paragraph`); + // a full O(blocks) rebuild only runs when the patch is inapplicable. + if self.try_patch_item_sizes_cache(width, scroll_anchor.clone(), window, cx).is_none() { + self.item_sizes_cache = None; + let _ = self.rebuild_item_sizes_cache_with_prefetch(width, scroll_anchor, false, window, cx); + } cx.notify(); } if !self.chunk_prefetch_queue.is_empty() && !self.pending_chunk_prefetch { diff --git a/crates/gpui-flowtext/src/rich_text/editor/commands.rs b/crates/gpui-flowtext/src/rich_text/editor/commands.rs index 2e3a3026..30cbdacc 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/commands.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/commands.rs @@ -159,18 +159,18 @@ impl RichTextEditor { if paragraph_ix < self.document.paragraphs.len() { // Outline navigation should place the insertion caret at the start of // the target paragraph, matching what the user just selected in the nav. - self.selection = EditorSelection { - anchor: DocumentOffset { - paragraph: paragraph_ix, - byte: 0, - }, - head: DocumentOffset { - paragraph: paragraph_ix, - byte: 0, - }, - }; + let before_selection = self.selection.clone(); + self.note_explicit_selection_movement(); + self.selection = EditorSelection::collapsed(DocumentOffset { + paragraph: paragraph_ix, + byte: 0, + }); + self.fidelity_caret_set_from("scroll_to_paragraph", &before_selection); self.goal_x = None; self.reset_caret_blink(cx); + if self.selection != before_selection { + self.emit_selection_changed(cx); + } let width = self.current_layout_width(); let start = paragraph_ix.saturating_sub(2); @@ -188,40 +188,25 @@ impl RichTextEditor { } } + /// Loro-first (spec §10): undo executes through the write authority's + /// `UndoManager` — synchronously, cursor-restored, collaboration-safe. The + /// editor holds no content history of its own (invariant 11). pub fn undo(&mut self, cx: &mut Context) { - let Some(record) = self.undo_stack.pop() else { - return; - }; - let restored_generation = record.before_generation; - for operation in record.operations.iter().rev() { - operation.undo(&mut self.document); - } - self.selection = record.before_selection.clone(); - self.edit_generation = restored_generation; - self.redo_stack.push(record); - self.after_history_restore(cx); + self.note_explicit_selection_movement(); + self.undo_via_authority(cx); } pub fn redo(&mut self, cx: &mut Context) { - let Some(record) = self.redo_stack.pop() else { - return; - }; - let restored_generation = record.after_generation; - for operation in &record.operations { - operation.redo(&mut self.document); - } - self.selection = record.after_selection.clone(); - self.edit_generation = restored_generation; - self.undo_stack.push(record); - self.after_history_restore(cx); + self.note_explicit_selection_movement(); + self.redo_via_authority(cx); } pub fn can_undo(&self) -> bool { - !self.undo_stack.is_empty() + self.write_authority.is_some() } pub fn can_redo(&self) -> bool { - !self.redo_stack.is_empty() + self.write_authority.is_some() } pub fn move_left(&mut self, window: &mut Window, cx: &mut Context) { @@ -288,37 +273,41 @@ impl RichTextEditor { } let last = self.document.paragraphs.len() - 1; let last_len = paragraph_text_len(&self.document.paragraphs[last]); - let selection = EditorSelection { - anchor: DocumentOffset { paragraph: 0, byte: 0 }, - head: DocumentOffset { + let selection = EditorSelection::range( + DocumentOffset { paragraph: 0, byte: 0 }, + DocumentOffset { paragraph: last, byte: last_len, }, - }; + ); if self.selection == selection { self.goal_x = None; return; } + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("select_all", &fid_before); self.goal_x = None; self.reset_caret_blink(cx); + self.emit_selection_changed(cx); cx.notify(); } pub fn move_word_left(&mut self, cx: &mut Context) { - self.move_to_offset(self.word_left(self.selection.head), false, cx); + self.move_to_offset(self.word_left(self.selection.head), SelectionAffinity::Before, VisualGravity::Neutral, false, cx); } pub fn move_word_right(&mut self, cx: &mut Context) { - self.move_to_offset(self.word_right(self.selection.head), false, cx); + self.move_to_offset(self.word_right(self.selection.head), SelectionAffinity::After, VisualGravity::Neutral, false, cx); } pub fn select_word_left(&mut self, cx: &mut Context) { - self.move_to_offset(self.word_left(self.selection.head), true, cx); + self.move_to_offset(self.word_left(self.selection.head), SelectionAffinity::Before, VisualGravity::Neutral, true, cx); } pub fn select_word_right(&mut self, cx: &mut Context) { - self.move_to_offset(self.word_right(self.selection.head), true, cx); + self.move_to_offset(self.word_right(self.selection.head), SelectionAffinity::After, VisualGravity::Neutral, true, cx); } pub fn page_up(&mut self, cx: &mut Context) { @@ -338,19 +327,19 @@ impl RichTextEditor { } pub fn move_document_start(&mut self, cx: &mut Context) { - self.move_to_offset(DocumentOffset::default(), false, cx); + self.move_to_offset(DocumentOffset::default(), SelectionAffinity::Before, VisualGravity::Neutral, false, cx); } pub fn move_document_end(&mut self, cx: &mut Context) { - self.move_to_offset(document_end(&self.document), false, cx); + self.move_to_offset(document_end(&self.document), SelectionAffinity::After, VisualGravity::Neutral, false, cx); } pub fn select_document_start(&mut self, cx: &mut Context) { - self.move_to_offset(DocumentOffset::default(), true, cx); + self.move_to_offset(DocumentOffset::default(), SelectionAffinity::Before, VisualGravity::Neutral, true, cx); } pub fn select_document_end(&mut self, cx: &mut Context) { - self.move_to_offset(document_end(&self.document), true, cx); + self.move_to_offset(document_end(&self.document), SelectionAffinity::After, VisualGravity::Neutral, true, cx); } pub fn insert_text_command(&mut self, text: &str, cx: &mut Context) { @@ -358,8 +347,10 @@ impl RichTextEditor { cx.notify(); return; } - if self.insert_single_grapheme_fast_path(text, cx) { - return; + let before = self.selection.clone(); + clamp_selection_to_document(&self.document, &mut self.selection); + if self.selection != before { + self.emit_selection_changed(cx); } self.apply_document_edit(cx, |editor, cx| editor.insert_text(text, cx)); } @@ -369,10 +360,6 @@ impl RichTextEditor { cx.notify(); return; } - if !self.selection.is_caret() && self.selection_crosses_object_blocks(self.selection.normalized()) { - let _ = self.delete_selection_with_document_snapshot(cx); - return; - } self.apply_document_edit(cx, |editor, cx| editor.backspace(cx)); } @@ -381,25 +368,13 @@ impl RichTextEditor { cx.notify(); return; } - if !self.selection.is_caret() && self.selection_crosses_object_blocks(self.selection.normalized()) { - let _ = self.delete_selection_with_document_snapshot(cx); - return; - } self.apply_document_edit(cx, |editor, cx| editor.delete_forward(cx)); } pub fn insert_paragraph_break_command(&mut self, cx: &mut Context) { - if !self.selection.is_caret() { - self.apply_document_edit(cx, |editor, cx| editor.insert_paragraph_break(cx)); - return; - } - - let caret = self.selection.head; - let Some(block_ix) = block_ix_for_paragraph(&self.document, caret.paragraph) else { - self.apply_document_edit(cx, |editor, cx| editor.insert_paragraph_break(cx)); - return; - }; - self.insert_paragraph_break_at_caret(caret, block_ix, cx); + // Loro-first: one primitive covers caret and selection cases — the split + // intent commits through the write authority (spec §5). + self.apply_document_edit(cx, |editor, cx| editor.insert_paragraph_break(cx)); } pub fn delete_word_backward_command(&mut self, cx: &mut Context) { @@ -407,9 +382,11 @@ impl RichTextEditor { if editor.selection.is_caret() { let head = editor.selection.head; let anchor = editor.word_left(head); - editor.selection = EditorSelection { anchor, head }; + let fid_before = editor.fidelity_caret_before(); + editor.selection = EditorSelection::range(anchor, head); + editor.fidelity_caret_set("delete_word_backward_command", &fid_before); } - editor.delete_selection_internal(); + editor.delete_selection_internal_with_cx(cx); editor.after_text_mutation(cx); }); } @@ -419,9 +396,11 @@ impl RichTextEditor { if editor.selection.is_caret() { let anchor = editor.selection.head; let head = editor.word_right(anchor); - editor.selection = EditorSelection { anchor, head }; + let fid_before = editor.fidelity_caret_before(); + editor.selection = EditorSelection::range(anchor, head); + editor.fidelity_caret_set("delete_word_forward_command", &fid_before); } - editor.delete_selection_internal(); + editor.delete_selection_internal_with_cx(cx); editor.after_text_mutation(cx); }); } @@ -470,13 +449,9 @@ impl RichTextEditor { }); return; } - if self.delete_selection_with_document_snapshot(cx) { - return; - } - self.apply_document_edit(cx, |editor, cx| { - editor.delete_selection_internal(); - editor.after_text_mutation(cx); - }); + // Loro-first: one DeleteRange intent handles object-crossing selections + // canonically (the runtime retires objects + records with the text). + self.write_delete_selection(cx); } pub fn paste(&mut self, cx: &mut Context) { @@ -568,6 +543,9 @@ impl RichTextEditor { if text.trim().is_empty() { return; } + if self.insert_text_into_selected_equation(text, cx) { + return; + } if self.insert_plain_text_into_selected_table_cell(text, cx) { return; } diff --git a/crates/gpui-flowtext/src/rich_text/editor/drop_preview.rs b/crates/gpui-flowtext/src/rich_text/editor/drop_preview.rs index 9c32d8a6..f88a56e2 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/drop_preview.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/drop_preview.rs @@ -6,7 +6,7 @@ enum DropPreviewKind { #[derive(Clone)] enum DropPreviewContent { - Document(Box), + DocumentProjection(Box), ExternalPaths { label: SharedString }, } @@ -88,7 +88,7 @@ impl RichTextEditor { && preview.fingerprint == fingerprint && preview.is_first == is_first && preview.is_last == is_last - && matches!(&preview.content, DropPreviewContent::Document(_)) + && matches!(&preview.content, DropPreviewContent::DocumentProjection(_)) { preview.insert_block_ix = placement.insert_block_ix; preview.suppressed_block_ix = placement.suppressed_block_ix; @@ -107,7 +107,7 @@ impl RichTextEditor { is_last, width, height, - content: DropPreviewContent::Document(Box::new(document)), + content: DropPreviewContent::DocumentProjection(Box::new(document)), }); } @@ -182,7 +182,7 @@ impl RichTextEditor { } } - fn drop_preview_document_from_paragraphs(&self, paragraphs: Vec, is_first: bool, is_last: bool) -> Document { + fn drop_preview_document_from_paragraphs(&self, paragraphs: Vec, is_first: bool, is_last: bool) -> DocumentProjection { let mut theme = self.document.theme.clone(); if !is_first { theme.pageless_inset_top = px(0.0); @@ -214,16 +214,16 @@ impl RichTextEditor { cx: &mut Context, ) -> (RenderVirtualItems, Rc>>) { if self.drop_preview.is_none() { - return (RenderVirtualItems::Document(base_items), base_sizes); + return (RenderVirtualItems::DocumentProjection(base_items), base_sizes); } self.refresh_drop_preview_height(width, window, cx); let Some(preview) = &self.drop_preview else { - return (RenderVirtualItems::Document(base_items), base_sizes); + return (RenderVirtualItems::DocumentProjection(base_items), base_sizes); }; let Some(cache) = self.item_sizes_cache.as_ref() else { - return (RenderVirtualItems::Document(base_items), base_sizes); + return (RenderVirtualItems::DocumentProjection(base_items), base_sizes); }; let insert_item_ix = drop_preview_insert_item_ix(cache, preview.insert_block_ix); @@ -244,7 +244,7 @@ impl RichTextEditor { if suppressed_range.contains(&item_ix) { continue; } - items.push(RenderVirtualItem::Document(base_items[item_ix].clone())); + items.push(RenderVirtualItem::DocumentProjection(base_items[item_ix].clone())); sizes.push(base_sizes[item_ix]); } @@ -260,7 +260,7 @@ impl RichTextEditor { } preview.width = width; preview.height = match &preview.content { - DropPreviewContent::Document(document) => drop_preview_document_height(document, width, self.invisibility_mode, window, cx), + DropPreviewContent::DocumentProjection(document) => drop_preview_document_height(document, width, self.invisibility_mode, window, cx), DropPreviewContent::ExternalPaths { .. } => drop_preview_external_paths_height(width, window, cx), }; } @@ -268,13 +268,13 @@ impl RichTextEditor { fn render_drop_preview(preview: DropPreview, invisibility_mode: bool, cx: &mut Context) -> impl IntoElement { let background = match &preview.content { - DropPreviewContent::Document(document) => document.theme.document_background_color.opacity(0.78), + DropPreviewContent::DocumentProjection(document) => document.theme.document_background_color.opacity(0.78), DropPreviewContent::ExternalPaths { .. } => cx.theme().background.opacity(0.78), }; let border = cx.theme().drag_border.opacity(0.72); let text_color = cx.theme().foreground.opacity(0.72); let content = match preview.content { - DropPreviewContent::Document(document) => div() + DropPreviewContent::DocumentProjection(document) => div() .w_full() .child(RichTextDocumentElement::new(*document).with_invisibility_mode(invisibility_mode)) .into_any_element(), @@ -312,7 +312,7 @@ fn drop_preview_insert_item_ix(cache: &ItemSizesCache, insert_block_ix: usize) - } fn drop_preview_document_height( - document: &Document, + document: &DocumentProjection, width: Pixels, invisibility_mode: bool, window: &mut Window, diff --git a/crates/gpui-flowtext/src/rich_text/editor/edit_pipeline.rs b/crates/gpui-flowtext/src/rich_text/editor/edit_pipeline.rs index 7c0bda2a..fa8739b1 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/edit_pipeline.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/edit_pipeline.rs @@ -1,243 +1,17 @@ +// Loro-first (spec §5/§12): the optimistic edit pipeline is gone. There is no +// capture range, no before/after span diffing, no semantic-command synthesis, +// no local history recording — intents commit through the write authority and +// the projection advances by exact returned patches (local_write_path.rs). +// What survives here is the small post-mutation bookkeeping shared by the +// write path and projection installs. + #[hotpath::measure_all] impl RichTextEditor { - fn insert_single_grapheme_fast_path(&mut self, text: &str, cx: &mut Context) -> bool { - if !is_single_grapheme_text_insert(text) || !self.selection.is_caret() || self.selected_block.is_some() { - return false; - } - let caret = self.selection.head; - let Some(paragraph) = self.document.paragraphs.get(caret.paragraph) else { - return false; - }; - if self.invisibility_mode && matches!(paragraph.style, ParagraphStyle::Normal) { - return false; - } - if caret.byte > paragraph_text_len(paragraph) { - return false; - } - let Some(paragraph_id) = self.identity_map.paragraph_id(caret.paragraph) else { - return false; - }; - - let before_selection = self.selection.clone(); - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - let styles = if let Some(styles) = self.pending_styles { - styles - } else { - let (run_ix, _) = run_containing(paragraph, caret.byte); - paragraph - .runs - .get(run_ix) - .map(|run| run.styles) - .unwrap_or_default() - }; - - insert_text_at(&mut self.document, caret.paragraph, caret.byte, text, styles); - let after = DocumentOffset { - paragraph: caret.paragraph, - byte: caret.byte + text.len(), - }; - self.selection = EditorSelection { anchor: after, head: after }; - - let mut merged_into_previous = false; - if let Some(record) = self.undo_stack.last_mut() - && before_selection.anchor == before_selection.head - && record.after_selection == before_selection - && record.operations.len() == 1 - && record.canonical_operations.len() == 1 - && let EditOperation::InsertText { - paragraph, - byte, - text: previous_text, - styles: previous_styles, - } = &mut record.operations[0] - && *paragraph == caret.paragraph - && *previous_styles == styles - && *byte + previous_text.len() == caret.byte - && let CanonicalOperation::InsertText { - paragraph: canonical_paragraph, - byte: canonical_byte, - text: canonical_text, - styles: canonical_styles, - } = &mut record.canonical_operations[0] - && *canonical_paragraph == paragraph_id - && *canonical_styles == styles - && *canonical_byte + canonical_text.len() == caret.byte - { - previous_text.push_str(text); - canonical_text.push_str(text); - record.after_selection = self.selection.clone(); - record.after_generation = after_generation; - merged_into_previous = true; - } - - if !merged_into_previous { - self.undo_stack.push(EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::InsertText { - paragraph: caret.paragraph, - byte: caret.byte, - text: text.to_string(), - styles, - }], - canonical_operations: vec![CanonicalOperation::InsertText { - paragraph: paragraph_id, - byte: caret.byte, - text: text.to_string(), - styles, - }], - }); - } - self.redo_stack.clear(); - self.layout_invalidation_hint = Some(caret.paragraph..caret.paragraph + 1); - self.suppress_mutation_notify += 1; - self.after_text_mutation(cx); - self.suppress_mutation_notify = self.suppress_mutation_notify.saturating_sub(1); - self.mark_document_changed_with_reconcile(after_generation, false, cx); - true - } - - fn apply_document_edit_with_capture_range( - &mut self, - cx: &mut Context, - capture_range: Option>, - edit: impl FnOnce(&mut Self, &mut Context), - ) { - let timing = Instant::now(); - let before_selection = self.selection.clone(); - let before_paragraph_count = self.document.paragraphs.len(); - let before_block_count = self.document.blocks.len(); - let before_range = capture_range.unwrap_or_else(|| self.edit_capture_range()); - let before_span = capture_document_span(&self.document, before_range); - self.layout_invalidation_hint = Some(before_span.start_paragraph..before_span.start_paragraph + before_span.paragraphs.len()); - self.suppress_mutation_notify += 1; - edit(self, cx); - self.suppress_mutation_notify = self.suppress_mutation_notify.saturating_sub(1); - self.layout_invalidation_hint = None; - let paragraph_delta = self.document.paragraphs.len() as isize - before_paragraph_count as isize; - let after_count = before_span - .paragraphs - .len() - .saturating_add_signed(paragraph_delta) - .min( - self - .document - .paragraphs - .len() - .saturating_sub(before_span.start_paragraph), - ); - let after_span = capture_document_span(&self.document, before_span.start_paragraph..before_span.start_paragraph + after_count); - self.finish_document_edit(before_span, before_selection, before_block_count, after_span, cx); - log_timing_lazy("edit command", timing, || { - format!("paragraphs={}", self.document.paragraphs.len()) - }); - } - - fn edit_capture_range(&self) -> Range { - let paragraph_count = self.document.paragraphs.len(); - if paragraph_count == 0 { - return 0..0; - } - let range = self.selection.normalized(); - let start = range.start.paragraph.saturating_sub(1); - let end = (range.end.paragraph + 2) - .min(paragraph_count) - .max(start + 1); - start..end - } - - fn finish_document_edit( - &mut self, - before_span: DocumentSpan, - before_selection: EditorSelection, - before_block_count: usize, - after_span: DocumentSpan, - cx: &mut Context, - ) { - if before_span == after_span && before_selection == self.selection { - return; - } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - let canonical_operations = vec![CanonicalOperation::ReplaceParagraphSpan { - start_paragraph: self.identity_map.paragraph_id(before_span.start_paragraph), - before: before_span.clone(), - after: after_span.clone(), - }]; - let identity_shape_changed = before_span.paragraphs.len() != after_span.paragraphs.len() || before_block_count != self.document.blocks.len(); - let record = EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceParagraphSpan { - before: before_span, - after: after_span, - }], - canonical_operations, - }; - self.undo_stack.push(record); - self.redo_stack.clear(); - self.mark_document_changed_with_reconcile(after_generation, identity_shape_changed, cx); - } - - fn insert_paragraph_break_at_caret(&mut self, caret: DocumentOffset, block_ix: usize, cx: &mut Context) { - let before_selection = self.selection.clone(); - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - let before_span = capture_document_span(&self.document, caret.paragraph..caret.paragraph + 1); - self.layout_invalidation_hint = Some(caret.paragraph..caret.paragraph + 1); - self.suppress_mutation_notify += 1; - self.insert_paragraph_break(cx); - self.suppress_mutation_notify = self.suppress_mutation_notify.saturating_sub(1); - self.layout_invalidation_hint = None; - self - .identity_map - .insert_split_paragraph(caret.paragraph, block_ix); - let after_span = capture_document_span(&self.document, caret.paragraph..caret.paragraph + 2); - - if before_span == after_span && before_selection == self.selection { - return; - } - - let record = EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceParagraphSpan { - before: before_span.clone(), - after: after_span.clone(), - }], - canonical_operations: vec![CanonicalOperation::ReplaceParagraphSpan { - start_paragraph: self.identity_map.paragraph_id(caret.paragraph), - before: before_span, - after: after_span, - }], - }; - self.undo_stack.push(record); - self.redo_stack.clear(); - self.mark_document_changed_with_reconcile(after_generation, false, cx); - } - - fn mark_document_changed(&mut self, generation: u64, cx: &mut Context) { - self.mark_document_changed_with_reconcile(generation, true, cx); - } - - fn mark_document_changed_with_reconcile(&mut self, generation: u64, reconcile_identity: bool, cx: &mut Context) { + pub(super) fn mark_document_changed(&mut self, generation: u64, reconcile_identity: bool, cx: &mut Context) { self.edit_generation = generation; if reconcile_identity { self.identity_map.reconcile(&self.document); } - self.last_collaboration_edit = self.undo_stack.last().map(|record| CollaborationEdit { - operations: record.canonical_operations.clone(), - }); self.refresh_save_status(); self.schedule_recovery_write(cx); cx.notify(); @@ -249,14 +23,4 @@ impl RichTextEditor { } } - fn after_history_restore(&mut self, cx: &mut Context) { - self.goal_x = None; - self.invalidate_document_layout_caches(); - self.refresh_save_status(); - self.scroll_head_into_view(); - self.reset_caret_blink(cx); - self.schedule_recovery_write(cx); - cx.notify(); - } - } diff --git a/crates/gpui-flowtext/src/rich_text/editor/formatting.rs b/crates/gpui-flowtext/src/rich_text/editor/formatting.rs index cbbd348c..0ecbd209 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/formatting.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/formatting.rs @@ -11,7 +11,7 @@ impl RichTextEditor { if self.clear_matching_armed_inline_tool(ArmedInlineTool::Strikethrough, cx) { return; } - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { let Some(selection_range) = self.table_cell_selection_range() else { self.armed_inline_tool = Some(ArmedInlineTool::Strikethrough); cx.notify(); @@ -41,10 +41,10 @@ impl RichTextEditor { } let range = self.selection.normalized(); let all_selected = selection_all_run_styles(&self.document, range.clone(), |styles| styles.strikethrough); - self.apply_document_edit(cx, |editor, cx| { - mutate_runs_in_range(&mut editor.document, range, |styles| styles.strikethrough = !all_selected); - editor.after_formatting_mutation(cx); - }); + let mut styles = run_styles_at_offset(&self.document, range.start); + styles.strikethrough = !all_selected; + self.pending_styles = None; + self.write_set_marks(range, styles, cx); } /// Toggle any semantic inline style for the current selection or caret. @@ -124,17 +124,9 @@ impl RichTextEditor { let Some(range) = self.selection_or_enclosing_section_range(section_slots) else { return; }; - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs, - blocks: Vec::new(), - assets: Vec::new(), - }; - self.selection = EditorSelection { - anchor: range.start, - head: range.end, - }; - self.apply_document_edit(cx, |editor, cx| editor.insert_rich_fragment(fragment, cx)); + self.selection = EditorSelection::range(range.start, range.end); + let blocks = paragraphs.into_iter().map(FragmentBlock::Paragraph).collect(); + self.write_insert_rich_fragment_at_caret(blocks, cx); } fn selection_or_enclosing_section_range(&self, section_slots: &[u8]) -> Option> { @@ -240,25 +232,19 @@ impl RichTextEditor { { return; } - self.apply_document_edit(cx, |editor, cx| { - for paragraph_ix in range_start.paragraph..=range_end.paragraph { - let paragraph_start = if paragraph_ix == range_start.paragraph { range_start.byte } else { 0 }; - let paragraph_end = if paragraph_ix == range_end.paragraph { - range_end.byte - } else { - paragraph_text_len(&editor.document.paragraphs[paragraph_ix]) - }; - if paragraph_start < paragraph_end { - apply_highlight_to_existing_highlights_in_paragraph_range( - &mut editor.document, - paragraph_ix, - paragraph_start..paragraph_end, - highlight, - ); - } - } - editor.after_formatting_mutation(cx); - }); + // Only re-colors runs that already carry a highlight; each maximal + // already-highlighted span becomes one SetMarks intent, grouped as one + // undo unit. + let spans = highlighted_spans_in_range(&self.document, range_start..range_end, highlight); + if spans.is_empty() { + return; + } + self.pending_styles = None; + self.begin_undo_group(); + for (range, styles) in spans { + self.write_set_marks(range, styles, cx); + } + self.end_undo_group(); } pub fn clear_highlight(&mut self, cx: &mut Context) { @@ -266,7 +252,7 @@ impl RichTextEditor { } pub fn clear_formatting(&mut self, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { paragraph.paragraph.style = ParagraphStyle::Normal; for run in &mut paragraph.paragraph.runs { @@ -277,28 +263,60 @@ impl RichTextEditor { }); return; } - self.apply_document_edit(cx, |editor, cx| { - if editor.selection.is_caret() { - let paragraph_ix = editor.selection.head.paragraph; - clear_whole_paragraph_formatting(&mut editor.document, paragraph_ix); + self.pending_styles = None; + if self.selection.is_caret() { + let paragraph_ix = self.selection.head.paragraph; + self.write_clear_whole_paragraph_formatting(paragraph_ix..paragraph_ix + 1, cx); + } else { + let range = self.selection.normalized(); + if selection_contains_whole_paragraph(&self.document, range.clone()) { + self.write_clear_whole_paragraph_formatting(range.start.paragraph..range.end.paragraph + 1, cx); } else { - let range = editor.selection.normalized(); - if selection_contains_whole_paragraph(&editor.document, range.clone()) { - for paragraph_ix in range.start.paragraph..=range.end.paragraph { - clear_whole_paragraph_formatting(&mut editor.document, paragraph_ix); - } - } else { - mutate_runs_in_range(&mut editor.document, range, |styles| *styles = RunStyles::default()); - } + self.write_set_marks(range, RunStyles::default(), cx); } - rebuild_document_sections(&mut editor.document); - editor.pending_styles = None; - editor.after_formatting_mutation(cx); - }); + } + } + + /// Clear-formatting over whole paragraphs: paragraph styles back to Normal + /// (one intent per changed paragraph) plus one run-style reset over the full + /// span, grouped as one undo unit. + fn write_clear_whole_paragraph_formatting(&mut self, paragraphs: Range, cx: &mut Context) { + let end = paragraphs.end.min(self.document.paragraphs.len()); + if paragraphs.start >= end { + return; + } + self.begin_undo_group(); + // ONE batched intent for every changed paragraph style (§11) — the + // per-paragraph loop amplified clear-formatting over a big selection the + // same way select-all restyle did. + let changed: Vec = (paragraphs.start..end) + .filter(|¶graph_ix| { + self + .document + .paragraphs + .get(paragraph_ix) + .is_some_and(|paragraph| paragraph.style != ParagraphStyle::Normal) + }) + .collect(); + if !changed.is_empty() { + self.write_set_paragraph_styles(changed, ParagraphStyle::Normal, cx); + } + let last_paragraph = end - 1; + let range = DocumentOffset { + paragraph: paragraphs.start, + byte: 0, + }..DocumentOffset { + paragraph: last_paragraph, + byte: self.document.paragraphs.get(last_paragraph).map(paragraph_text_len).unwrap_or(0), + }; + if range.start != range.end { + self.write_set_marks(range, RunStyles::default(), cx); + } + self.end_undo_group(); } pub fn apply_run_style_to_selection(&mut self, style: RunStyle, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { let Some(selection_range) = self.table_cell_selection_range() else { return; }; @@ -321,23 +339,14 @@ impl RichTextEditor { if self.selection.is_caret() { return; } - self.apply_document_edit(cx, |editor, cx| { - let range = editor.selection.normalized(); - for paragraph_ix in range.start.paragraph..=range.end.paragraph { - let start = if paragraph_ix == range.start.paragraph { range.start.byte } else { 0 }; - let end = if paragraph_ix == range.end.paragraph { - range.end.byte - } else { - paragraph_text_len(&editor.document.paragraphs[paragraph_ix]) - }; - apply_style_to_paragraph_range(&mut editor.document, paragraph_ix, start..end, style); - } - editor.after_formatting_mutation(cx); - }); + let range = self.selection.normalized(); + let styles = run_styles_at_offset(&self.document, range.start).with(style); + self.pending_styles = None; + self.write_set_marks(range, styles, cx); } pub fn set_paragraph_style_for_selection(&mut self, style: ParagraphStyle, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { if paragraph.paragraph.style != style { paragraph.paragraph.style = style; @@ -346,19 +355,22 @@ impl RichTextEditor { }); return; } - self.apply_document_edit(cx, |editor, cx| { - let range = editor.selection.normalized(); - for paragraph_ix in range.start.paragraph..=range.end.paragraph { - if let Some(paragraph) = paragraphs_mut(&mut editor.document).get_mut(paragraph_ix) - && paragraph.style != style - { - paragraph.style = style; - bump_paragraph_version(paragraph); - } - } - rebuild_document_sections(&mut editor.document); - editor.after_formatting_mutation(cx); - }); + let range = self.selection.normalized(); + // ONE batched intent for the whole selection (one commit, one undo + // member) — the per-paragraph loop amplified a select-all restyle into + // thousands of full write-path round trips. + let changed: Vec = (range.start.paragraph..=range.end.paragraph) + .filter(|¶graph_ix| { + self + .document + .paragraphs + .get(paragraph_ix) + .is_some_and(|paragraph| paragraph.style != style) + }) + .collect(); + if !changed.is_empty() { + self.write_set_paragraph_styles(changed, style, cx); + } } // -------- Action handlers (bound to keystrokes in main.rs) ----------- @@ -367,30 +379,58 @@ impl RichTextEditor { // fn(&mut Self, &Action, &mut Window, &mut Context). } -fn apply_highlight_to_existing_highlights_in_paragraph_range( - document: &mut Document, - paragraph_ix: usize, - range: Range, +/// Pure projection scan: the maximal spans inside `range` whose runs already +/// carry a highlight, with each span's styles recolored to `highlight`. +/// Adjacent run spans that converge on the same styles are merged so the +/// resulting `SetMarks` intents stay minimal. +fn highlighted_spans_in_range( + document: &DocumentProjection, + range: Range, highlight: HighlightStyle, -) { - mutate_runs_in_range( - document, - DocumentOffset { - paragraph: paragraph_ix, - byte: range.start, - }..DocumentOffset { - paragraph: paragraph_ix, - byte: range.end, - }, - |styles| { - if styles.highlight.is_some() { - styles.highlight = Some(highlight); +) -> Vec<(Range, RunStyles)> { + let mut spans: Vec<(Range, RunStyles)> = Vec::new(); + for paragraph_ix in range.start.paragraph..=range.end.paragraph { + let Some(paragraph) = document.paragraphs.get(paragraph_ix) else { + continue; + }; + let paragraph_start = if paragraph_ix == range.start.paragraph { range.start.byte } else { 0 }; + let paragraph_end = if paragraph_ix == range.end.paragraph { + range.end.byte + } else { + paragraph_text_len(paragraph) + }; + let mut offset = 0; + for run in ¶graph.runs { + let run_start = offset; + let run_end = offset + run.len; + offset = run_end; + if run_end <= paragraph_start || run_start >= paragraph_end || run.styles.highlight.is_none() { + continue; + } + let mut styles = run.styles; + styles.highlight = Some(highlight); + let start = DocumentOffset { + paragraph: paragraph_ix, + byte: run_start.max(paragraph_start), + }; + let end = DocumentOffset { + paragraph: paragraph_ix, + byte: run_end.min(paragraph_end), + }; + if let Some((last_range, last_styles)) = spans.last_mut() + && last_range.end == start + && *last_styles == styles + { + last_range.end = end; + } else { + spans.push((start..end, styles)); } - }, - ); + } + } + spans } -fn hierarchical_section_bounds(document: &Document, paragraph_ix: usize, section_slots: &[u8]) -> Option<(usize, usize)> { +fn hierarchical_section_bounds(document: &DocumentProjection, paragraph_ix: usize, section_slots: &[u8]) -> Option<(usize, usize)> { let paragraph = document.paragraphs.get(paragraph_ix)?; let ParagraphStyle::Custom(slot) = paragraph.style else { return None; @@ -422,7 +462,7 @@ fn hierarchical_section_bounds(document: &Document, paragraph_ix: usize, section Some((paragraph_ix, end)) } -fn collapse_heading_kind(document: &Document, paragraph_ix: usize, section_slots: &[u8]) -> Option { +fn collapse_heading_kind(document: &DocumentProjection, paragraph_ix: usize, section_slots: &[u8]) -> Option { let paragraph = document.paragraphs.get(paragraph_ix)?; let ParagraphStyle::Custom(slot) = paragraph.style else { return None; @@ -436,16 +476,16 @@ fn collapse_heading_kind(document: &Document, paragraph_ix: usize, section_slots Some(style.section_kind.unwrap_or(slot)) } -fn section_at_heading(document: &Document, paragraph_ix: usize, heading_kind: u8) -> Option<&DocumentSection> { - document.sections.iter().find(|section| { +fn section_at_heading(document: &DocumentProjection, paragraph_ix: usize, heading_kind: u8) -> Option<&DocumentOutlineNode> { + document.outline.iter().find(|section| { let SectionKind::Custom(section_kind) = section.kind; section_kind == heading_kind && paragraph_index_for_id(document, section.start_paragraph) == Some(paragraph_ix) }) } -fn enclosing_section<'a>(document: &'a Document, paragraph_ix: usize, section_slots: &[u8]) -> Option<&'a DocumentSection> { +fn enclosing_section<'a>(document: &'a DocumentProjection, paragraph_ix: usize, section_slots: &[u8]) -> Option<&'a DocumentOutlineNode> { document - .sections + .outline .iter() .filter(|section| { let SectionKind::Custom(slot) = section.kind; @@ -471,7 +511,7 @@ fn enclosing_section<'a>(document: &'a Document, paragraph_ix: usize, section_sl }) } -fn enclosing_section_bounds(document: &Document, paragraph_ix: usize, section_slots: &[u8]) -> Option<(usize, usize)> { +fn enclosing_section_bounds(document: &DocumentProjection, paragraph_ix: usize, section_slots: &[u8]) -> Option<(usize, usize)> { let section = enclosing_section(document, paragraph_ix, section_slots)?; let start = paragraph_index_for_id(document, section.start_paragraph)?; let end = section diff --git a/crates/gpui-flowtext/src/rich_text/editor/hit_testing.rs b/crates/gpui-flowtext/src/rich_text/editor/hit_testing.rs index f14b9fdf..a09bbfcc 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/hit_testing.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/hit_testing.rs @@ -1,6 +1,8 @@ #[hotpath::measure_all] impl RichTextEditor { - fn hit_test_document_position(&mut self, position: Point, window: &mut Window, cx: &mut Context) -> DocumentOffset { + // §act-eleven C9: `pub(super)` so the in-crate geometry property test can + // drive it directly (it was reachable only through mouse handlers). + pub(super) fn hit_test_document_position(&mut self, position: Point, window: &mut Window, cx: &mut Context) -> DocumentOffset { let paragraph_count = self.document.paragraphs.len(); if paragraph_count == 0 { return DocumentOffset::default(); @@ -90,7 +92,9 @@ impl RichTextEditor { let Some(layout) = self.layout_for_offset(head) else { return; }; - let Some((p_ix, l_ix)) = locate_line(&layout, head) else { + // Resolve the caret's current visual line using its stored gravity so the + // line edges we snap to match where the caret is actually painted. + let Some((p_ix, l_ix)) = locate_line(&layout, head, self.selection.head_gravity) else { return; }; let line = &layout.paragraphs[p_ix].lines[l_ix]; @@ -100,85 +104,65 @@ impl RichTextEditor { paragraph: head.paragraph, byte: new_byte, }; - let anchor = if extend { self.selection.anchor } else { new }; - let selection = EditorSelection { anchor, head: new }; - if self.selection == selection { + // §16: line-start gravitates downstream (start of the lower visual line), + // line-end gravitates upstream (end of the upper visual line). This is the + // canonical wrap-seam case — without Upstream gravity, "End" on a wrapped + // line would otherwise paint at the start of the next visual line. + let (affinity, gravity) = if start { + (SelectionAffinity::Before, VisualGravity::Downstream) + } else { + (SelectionAffinity::After, VisualGravity::Upstream) + }; + let selection = self.selection.moved(new, affinity, gravity, extend); + if self.selection.same_positions(&selection) { self.goal_x = None; return; } + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("move_to_line_edge", &fid_before); self.goal_x = None; self.scroll_head_into_view(); self.reset_caret_blink(cx); + self.emit_selection_changed(cx); cx.notify(); } // -------- Edit primitives -------------------------------------------- + /// Loro-first (spec §5): text insert commits through the write authority. + /// Style inheritance is Loro's expand-`After` marks (spec §9); only an + /// explicit caret style override (`pending_styles`) emits marks, and only + /// over the inserted range. fn insert_text(&mut self, text: &str, cx: &mut Context) { if text.is_empty() { return; } - if !self.selection.is_caret() { - self.delete_selection_internal(); - } - let caret = self.selection.head; - if self.invisibility_mode + // Invisibility mode upgrades an untouched Normal paragraph before typing — + // grouped with the insert as one undo unit. + let invisibility_restyle = self.invisibility_mode && self .document .paragraphs - .get(caret.paragraph) - .is_some_and(|paragraph| matches!(paragraph.style, ParagraphStyle::Normal)) - { - if let Some(paragraph) = paragraphs_mut(&mut self.document).get_mut(caret.paragraph) { - paragraph.style = ParagraphStyle::Custom(4); - bump_paragraph_version(paragraph); - } - update_paragraph_block(&mut self.document, caret.paragraph); + .get(self.selection.head.paragraph) + .is_some_and(|paragraph| matches!(paragraph.style, ParagraphStyle::Normal)); + if invisibility_restyle { + self.begin_undo_group(); + let paragraph_ix = self.selection.head.paragraph; + self.write_set_paragraph_style(paragraph_ix, ParagraphStyle::Custom(4), cx); + } + self.write_insert_text_at_caret(text, cx); + if invisibility_restyle { + self.end_undo_group(); } - // Inherit styles from the run that contains the caret. With left-bias at - // run boundaries this matches Word's "type continues the previous run's - // styling" behavior. - let styles = if let Some(styles) = self.pending_styles { - styles - } else { - let paragraph = &self.document.paragraphs[caret.paragraph]; - let (run_ix, _) = run_containing(paragraph, caret.byte); - paragraph - .runs - .get(run_ix) - .map(|r| r.styles) - .unwrap_or_default() - }; - insert_text_at(&mut self.document, caret.paragraph, caret.byte, text, styles); - let new = DocumentOffset { - paragraph: caret.paragraph, - byte: caret.byte + text.len(), - }; - self.selection = EditorSelection { anchor: new, head: new }; - self.after_text_mutation(cx); } - // Helper for shared selection-deletion logic. Does NOT call `cx.notify()`. - fn delete_selection_internal(&mut self) -> bool { - if self.selection.is_caret() { - return false; - } - let range = self.selection.normalized(); - if range.start.paragraph == range.end.paragraph { - delete_range_in_paragraph(&mut self.document, range.start.paragraph, range.start.byte..range.end.byte); - } else { - // Cross-paragraph selection: delete the tail of the start paragraph, - // the head of the end paragraph, then merge the end paragraph's - // remaining runs onto the end of the start paragraph. Intermediate - // paragraphs are dropped wholesale. - delete_cross_paragraph_range(&mut self.document, range.clone()); - } - self.selection = EditorSelection { - anchor: range.start, - head: range.start, - }; - true + /// Loro-first: selection deletion is ONE `DeleteRange` intent — the runtime's + /// body delete handles cross-paragraph structure (records, orphaned objects) + /// canonically; the editor no longer resolves structure itself. + fn delete_selection_internal_with_cx(&mut self, cx: &mut Context) -> bool { + self.write_delete_selection(cx) } fn backspace(&mut self, cx: &mut Context) { @@ -192,8 +176,7 @@ impl RichTextEditor { return; } if !self.selection.is_caret() { - self.delete_selection_internal(); - self.after_text_mutation(cx); + self.write_delete_selection(cx); return; } let caret = self.selection.head; @@ -202,35 +185,22 @@ impl RichTextEditor { self.select_block(object, cx); return; } - // Joining backwards: merge this paragraph onto the previous one. The - // caret lands at the join seam. + // Joining backwards: merge this paragraph onto the previous one via a + // Join intent. The write path lands the caret at the join seam. if caret.paragraph == 0 { return; } - let prev_ix = caret.paragraph - 1; - let prev_len = paragraph_text_len(&self.document.paragraphs[prev_ix]); - delete_cross_paragraph_range( - &mut self.document, + self.write_join_paragraphs(caret.paragraph - 1, cx); + } else { + let prev = prev_grapheme_boundary_in_paragraph(&self.document, caret.paragraph, caret.byte); + self.write_delete_offset_range( DocumentOffset { - paragraph: prev_ix, - byte: prev_len, + paragraph: caret.paragraph, + byte: prev, }..caret, + cx, ); - let new = DocumentOffset { - paragraph: prev_ix, - byte: prev_len, - }; - self.selection = EditorSelection { anchor: new, head: new }; - } else { - let prev = prev_grapheme_boundary_in_paragraph(&self.document, caret.paragraph, caret.byte); - delete_range_in_paragraph(&mut self.document, caret.paragraph, prev..caret.byte); - let new = DocumentOffset { - paragraph: caret.paragraph, - byte: prev, - }; - self.selection = EditorSelection { anchor: new, head: new }; } - self.after_text_mutation(cx); } fn delete_forward(&mut self, cx: &mut Context) { @@ -241,8 +211,7 @@ impl RichTextEditor { return; } if !self.selection.is_caret() { - self.delete_selection_internal(); - self.after_text_mutation(cx); + self.write_delete_selection(cx); return; } let caret = self.selection.head; @@ -252,42 +221,43 @@ impl RichTextEditor { self.select_block(object, cx); return; } - // Joining forwards: pull the next paragraph's runs onto this one. + // Joining forwards: merge the next paragraph onto this one. if caret.paragraph + 1 >= self.document.paragraphs.len() { return; } - delete_cross_paragraph_range( - &mut self.document, - caret..DocumentOffset { - paragraph: caret.paragraph + 1, - byte: 0, - }, - ); + self.write_join_paragraphs(caret.paragraph, cx); } else { let next = next_grapheme_boundary_in_paragraph(&self.document, caret.paragraph, caret.byte); - delete_range_in_paragraph(&mut self.document, caret.paragraph, caret.byte..next); + self.write_delete_offset_range(caret..DocumentOffset { paragraph: caret.paragraph, byte: next }, cx); } - self.after_text_mutation(cx); } fn insert_paragraph_break(&mut self, cx: &mut Context) { - if !self.selection.is_caret() { - self.delete_selection_internal(); - } - let caret = self.selection.head; - let starts_empty_paragraph = caret.byte == paragraph_text_len(&self.document.paragraphs[caret.paragraph]); - split_paragraph_at(&mut self.document, caret.paragraph, caret.byte); - let new = DocumentOffset { - paragraph: caret.paragraph + 1, - byte: 0, + let caret = self.selection.normalized().start; + // Enter at the end of a paragraph starts a genuinely FRESH paragraph + // (Normal style — heading/inline styling must not leak); mid-paragraph + // splits continue the current style. `inherited_style` marks the NEW + // paragraph's boundary sentinel. + let at_end = self + .document + .paragraphs + .get(caret.paragraph) + .is_some_and(|paragraph| caret.byte >= paragraph_text_len(paragraph)); + let inherited_style = if at_end { + ParagraphStyle::Normal + } else { + self + .document + .paragraphs + .get(caret.paragraph) + .map(|paragraph| paragraph.style) + .unwrap_or(ParagraphStyle::Normal) }; - if starts_empty_paragraph { - // Pressing Enter at the end starts a genuinely fresh paragraph. Reset it - // so header/inline/highlight styling does not leak into the next line. - clear_whole_paragraph_formatting(&mut self.document, new.paragraph); - rebuild_document_sections(&mut self.document); + // Fresh paragraphs also reset the pending caret style so run styling does + // not leak across the break. + if at_end { + self.pending_styles = Some(RunStyles::default()); } - self.selection = EditorSelection { anchor: new, head: new }; - self.after_text_mutation(cx); + self.write_split_at_caret(inherited_style, cx); } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/item_sizes.rs b/crates/gpui-flowtext/src/rich_text/editor/item_sizes.rs index 7a3767ca..98d7b4bc 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/item_sizes.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/item_sizes.rs @@ -1,4 +1,4 @@ -type ParagraphRangeItemSizes = (Vec, Vec>, Vec, Vec>); +type VirtualItemSizeParts = (Vec, Vec>, Vec, Vec>); type FullItemSizes = (Rc>, Vec>, Vec, Rc>>); #[hotpath::measure_all] @@ -104,38 +104,46 @@ impl RichTextEditor { ) -> Option>>> { let range = self.pending_item_sizes_patch_range.clone()?; let paragraph_count = self.document.paragraphs.len(); - if range.start > range.end || range.end > paragraph_count || self.document_has_object_blocks() { + if range.start > range.end || range.end > paragraph_count { return None; } - let cache = self.item_sizes_cache.as_ref()?; - if cache.width != width - || cache.block_count != self.document.blocks.len() - || cache.invisibility_mode != self.invisibility_mode - || cache.paragraph_chunk_item_ranges.len() != paragraph_count - || cache.paragraph_remainder_items.len() != paragraph_count - || cache.block_item_ranges.len() != self.document.blocks.len() - || cache.block_heights.len() != self.document.blocks.len() - || self.height_prefix_index.len() != cache.item_count - { - return None; - } + let block_range = self.block_range_for_paragraph_range(range.clone())?; + let (replace_start, replace_end, item_count) = { + let cache = self.item_sizes_cache.as_ref()?; + if cache.width != width + || cache.block_count != self.document.blocks.len() + || cache.invisibility_mode != self.invisibility_mode + || cache.paragraph_chunk_item_ranges.len() != paragraph_count + || cache.paragraph_remainder_items.len() != paragraph_count + || cache.block_item_ranges.len() != self.document.blocks.len() + || cache.block_heights.len() != self.document.blocks.len() + || self.height_prefix_index.len() != cache.item_count + { + return None; + } - let replace_start = cache - .block_item_ranges - .get(range.start) - .map_or(cache.item_count, |range| range.start); - let replace_end = if range.end == 0 { - 0 - } else { - cache.block_item_ranges.get(range.end - 1)?.end + let replace_start = if block_range.start == block_range.end { + cache + .block_item_ranges + .get(block_range.start) + .map_or(cache.item_count, |range| range.start) + } else { + cache.block_item_ranges.get(block_range.start)?.start + }; + let replace_end = if block_range.start == block_range.end { + replace_start + } else { + cache.block_item_ranges.get(block_range.end - 1)?.end + }; + (replace_start, replace_end, cache.item_count) }; - if replace_start > replace_end || replace_end > cache.item_count { + if replace_start > replace_end || replace_end > item_count { return None; } let (replacement_items, replacement_block_ranges, replacement_block_heights, replacement_sizes) = - self.virtual_item_sizes_for_paragraph_range(range.clone(), width, window, cx)?; + self.virtual_item_sizes_for_block_range(block_range.clone(), range.start, width, window, cx)?; let old_len = replace_end - replace_start; let new_len = replacement_items.len(); let item_delta = new_len as isize - old_len as isize; @@ -147,15 +155,15 @@ impl RichTextEditor { items.splice(replace_start..replace_end, replacement_items); sizes.splice(replace_start..replace_end, replacement_sizes.clone()); - for block_ix in range.clone() { - let relative = &replacement_block_ranges[block_ix - range.start]; + for block_ix in block_range.clone() { + let relative = &replacement_block_ranges[block_ix - block_range.start]; cache.block_item_ranges[block_ix] = replace_start + relative.start..replace_start + relative.end; - cache.block_heights[block_ix] = replacement_block_heights[block_ix - range.start]; + cache.block_heights[block_ix] = replacement_block_heights[block_ix - block_range.start]; } if item_delta != 0 { - for block_range in cache.block_item_ranges.iter_mut().skip(range.end) { - block_range.start = block_range.start.checked_add_signed(item_delta)?; - block_range.end = block_range.end.checked_add_signed(item_delta)?; + for cached_block_range in cache.block_item_ranges.iter_mut().skip(block_range.end) { + cached_block_range.start = cached_block_range.start.checked_add_signed(item_delta)?; + cached_block_range.end = cached_block_range.end.checked_add_signed(item_delta)?; } } @@ -184,61 +192,121 @@ impl RichTextEditor { Some(patched_sizes) } - fn virtual_item_sizes_for_paragraph_range( + /// Record that `paragraph_ix`'s item sizes went stale (a chunk materialized + /// and refined its height) so the next size query PATCHES the cache instead + /// of rebuilding it. The former shape nuked `item_sizes_cache` outright, + /// which made every scroll frame that materialized a chunk pay a full + /// O(blocks) rebuild + prefix-index rebuild (~10 ms per frame on the + /// reference doc — half the frame budget). + pub(super) fn note_item_sizes_patch_paragraph(&mut self, paragraph_ix: usize) { + self.pending_item_sizes_patch_range = Some(match self.pending_item_sizes_patch_range.take() { + Some(range) => range.start.min(paragraph_ix)..range.end.max(paragraph_ix + 1), + None => paragraph_ix..paragraph_ix + 1, + }); + } + + fn block_range_for_paragraph_range(&self, range: Range) -> Option> { + if range.start == range.end { + let block_ix = if range.start == self.document.paragraphs.len() { + self.document.blocks.len() + } else { + self.block_ix_for_paragraph(range.start)? + }; + return Some(block_ix..block_ix); + } + + let start_block = self.block_ix_for_paragraph(range.start)?; + let end_block = if range.end == self.document.paragraphs.len() { + self.document.blocks.len() + } else { + self.block_ix_for_paragraph(range.end)? + }; + Some(start_block..end_block.max(start_block)) + } + + // §perf: the sole caller already knows the paragraph index of `block_range.start` + // (it derives the block range from a paragraph range whose `start` is exactly that + // index), so it threads it in as `first_paragraph_ix`. This removes a former + // O(block_range.start) scan of the whole block vector that ran on every keystroke + // editing low in a large document. + fn virtual_item_sizes_for_block_range( &mut self, - range: Range, + block_range: Range, + first_paragraph_ix: usize, width: Pixels, - _window: &mut Window, - _cx: &mut Context, - ) -> Option { - let mut items = Vec::with_capacity(range.len()); - let mut sizes = Vec::with_capacity(range.len()); - let mut block_item_ranges = Vec::with_capacity(range.len()); - let mut block_heights = Vec::with_capacity(range.len()); - - for paragraph_ix in range { - let paragraph = self.document.paragraphs.get(paragraph_ix)?; - if !matches!(self.document.blocks.get(paragraph_ix), Some(Block::Paragraph(_))) { - return None; - } + window: &mut Window, + cx: &mut Context, + ) -> Option { + let mut items = Vec::with_capacity(block_range.len()); + let mut sizes = Vec::with_capacity(block_range.len()); + let mut block_item_ranges = Vec::with_capacity(block_range.len()); + let mut block_heights = Vec::with_capacity(block_range.len()); + let mut paragraph_ix = first_paragraph_ix; + for block_ix in block_range { let block_start = items.len(); let mut block_height = px(0.0); - if self.paragraph_hidden_by_collapsed_section(paragraph_ix) { - block_item_ranges.push(block_start..items.len()); - block_heights.push(px(0.0)); - continue; - } - if self.invisibility_mode && !paragraph_is_visible(&self.document, paragraph) { - block_item_ranges.push(block_start..items.len()); - block_heights.push(px(0.0)); - continue; - } - let complete = self - .valid_chunk_cache_entry(paragraph_ix, width) - .map(|entry| { - for (chunk_ix, chunk) in entry.chunks.iter().enumerate() { - items.push(VirtualItem::ParagraphChunk { - block_ix: paragraph_ix, - paragraph_ix, - chunk_ix, + match self.document.blocks.get(block_ix) { + Some(Block::Paragraph(paragraph)) => { + let current_paragraph_ix = paragraph_ix; + paragraph_ix += 1; + if self.paragraph_hidden_by_collapsed_section(current_paragraph_ix) { + block_item_ranges.push(block_start..items.len()); + block_heights.push(px(0.0)); + continue; + } + if self.invisibility_mode && !paragraph_is_visible(&self.document, paragraph) { + block_item_ranges.push(block_start..items.len()); + block_heights.push(px(0.0)); + continue; + } + + let complete = self + .valid_chunk_cache_entry(current_paragraph_ix, width) + .map(|entry| { + for (chunk_ix, chunk) in entry.chunks.iter().enumerate() { + items.push(VirtualItem::ParagraphChunk { + block_ix, + paragraph_ix: current_paragraph_ix, + chunk_ix, + }); + sizes.push(size(width, chunk.height)); + block_height += chunk.height; + } + entry.complete + }) + .unwrap_or(false); + + if !complete { + let estimate = self.paragraph_remainder_estimate(current_paragraph_ix, width); + items.push(VirtualItem::ParagraphRemainder { + block_ix, + paragraph_ix: current_paragraph_ix, }); - sizes.push(size(width, chunk.height)); - block_height += chunk.height; + sizes.push(size(width, estimate)); + block_height += estimate; } - entry.complete - }) - .unwrap_or(false); - - if !complete { - let estimate = self.paragraph_remainder_estimate(paragraph_ix, width); - items.push(VirtualItem::ParagraphRemainder { - block_ix: paragraph_ix, - paragraph_ix, - }); - sizes.push(size(width, estimate)); - block_height += estimate; + }, + Some(Block::Image(_) | Block::Equation(_) | Block::Table(_)) => { + if self.invisibility_mode { + block_item_ranges.push(block_start..items.len()); + block_heights.push(px(0.0)); + continue; + } + let height = layout_structural_block_at(&self.document, block_ix, width, px(0.0), window, cx) + .as_ref() + .map(structural_block_height) + .unwrap_or_else(|| estimate_structural_block_item_height(&self.document, block_ix, width)) + + self.document.theme.paragraph_after; + items.push(VirtualItem::StructuralBlock { block_ix }); + sizes.push(size(width, height)); + block_height += height; + }, + None => { + items.push(VirtualItem::HiddenBlock { block_ix }); + sizes.push(size(width, px(0.0))); + }, } block_item_ranges.push(block_start..items.len()); @@ -248,7 +316,7 @@ impl RichTextEditor { Some((items, block_item_ranges, block_heights, sizes)) } - pub(crate) fn benchmark_paragraph_item_sizes( + pub fn benchmark_paragraph_item_sizes( &mut self, width: Pixels, window: &mut Window, @@ -308,7 +376,40 @@ impl RichTextEditor { } } - pub(crate) fn benchmark_invalidate_document_layout_caches(&mut self) { + /// §perf-heaven T5 net: force an exact layout pass, then for every paragraph + /// with a COMPLETE exact layout compare its estimate to its exact height. + /// Returns the worst over/under ratios — the layout-fidelity oracle for the + /// estimate heuristic and the guard any persisted-estimate work must pass. + pub fn benchmark_estimate_accuracy(&mut self, width: Pixels, window: &mut Window, cx: &mut Context) -> EstimateAccuracy { + self.measured_item_width = Some(width); + let _ = self.paragraph_item_sizes(window, cx); + let mut acc = EstimateAccuracy::default(); + for paragraph_ix in 0..self.document.paragraphs.len() { + let Some(entry) = self.valid_chunk_cache_entry(paragraph_ix, width) else { + continue; + }; + if !entry.complete { + continue; + } + let exact: f32 = entry.exact_height.into(); + if exact <= 1.0 { + continue; + } + let estimate: f32 = self + .paragraph_estimated_total_height(paragraph_ix, width) + .map_or(0.0, |(height, _)| height.into()); + let ratio = estimate / exact; + acc.compared += 1; + if ratio >= 1.0 { + acc.max_over_ratio = acc.max_over_ratio.max(ratio - 1.0); + } else { + acc.max_under_ratio = acc.max_under_ratio.max(1.0 - ratio); + } + } + acc + } + + pub fn benchmark_invalidate_document_layout_caches(&mut self) { self.invalidate_document_layout_caches(); } @@ -398,7 +499,7 @@ impl RichTextEditor { if self.collapsed_section_ids.is_empty() { return false; } - self.document.sections.iter().any(|section| { + self.document.outline.iter().any(|section| { if !self.collapsed_section_ids.contains(§ion.id) { return false; } @@ -422,7 +523,7 @@ impl RichTextEditor { fn reusable_virtual_item_buffers( old_cache: Option, block_count: usize, -) -> ParagraphRangeItemSizes { +) -> VirtualItemSizeParts { let Some(cache) = old_cache else { return ( Vec::with_capacity(block_count), diff --git a/crates/gpui-flowtext/src/rich_text/editor/layout_access.rs b/crates/gpui-flowtext/src/rich_text/editor/layout_access.rs index bc5d7880..427d327b 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/layout_access.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/layout_access.rs @@ -190,31 +190,18 @@ impl RichTextEditor { } fn paragraph_before_block(&self, target_block_ix: usize) -> Option { - let mut paragraph_ix = 0; - let mut last = None; - for (block_ix, block) in self.document.blocks.iter().enumerate() { - if block_ix >= target_block_ix { - return last; - } - if matches!(block, Block::Paragraph(_)) { - last = Some(paragraph_ix); - paragraph_ix += 1; - } - } - last + // §act-ten A10.8: O(log N) tree rank instead of the per-call block walk. + self + .document + .blocks + .paragraphs_before_row(target_block_ix) + .checked_sub(1) } fn paragraph_after_block(&self, target_block_ix: usize) -> Option { - let mut paragraph_ix = 0; - for (block_ix, block) in self.document.blocks.iter().enumerate() { - if matches!(block, Block::Paragraph(_)) { - if block_ix > target_block_ix { - return Some(paragraph_ix); - } - paragraph_ix += 1; - } - } - None + // §act-ten A10.8: rank of the first paragraph past `target_block_ix`. + let rank = self.document.blocks.paragraphs_before_row(target_block_ix.saturating_add(1)); + (rank < self.document.paragraphs.len()).then_some(rank) } fn collapse_object_selection(&mut self, dir: HDir, cx: &mut Context) -> bool { @@ -239,10 +226,15 @@ impl RichTextEditor { .map(|paragraph| DocumentOffset { paragraph, byte: 0 }), }; if let Some(offset) = offset { - self.selection = EditorSelection { - anchor: offset, - head: offset, + // §16 object boundary: collapsing leftward lands after the preceding + // paragraph's content (After); rightward lands before the following + // paragraph's content (Before). + let affinity = match dir { + HDir::Left => SelectionAffinity::After, + HDir::Right => SelectionAffinity::Before, }; + self.note_explicit_selection_movement(); + self.selection = EditorSelection::collapsed_with(offset, affinity, VisualGravity::Neutral); self.scroll_head_into_view(); self.reset_caret_blink(cx); cx.notify(); @@ -253,26 +245,11 @@ impl RichTextEditor { } fn paragraph_ix_for_block(&self, target_block_ix: usize) -> Option { - if self.document.blocks.len() == self.document.paragraphs.len() - && self - .document - .blocks - .get(target_block_ix) - .is_some_and(|block| matches!(block, Block::Paragraph(_))) - { - return Some(target_block_ix); - } - - let mut paragraph_ix = 0; - for (block_ix, block) in self.document.blocks.iter().enumerate() { - if matches!(block, Block::Paragraph(_)) { - if block_ix == target_block_ix { - return Some(paragraph_ix); - } - paragraph_ix += 1; - } - } - None + // §act-ten A10.8: O(log N) tree rank (was a per-call full block walk — + // O(visible items x blocks) per layout pass on object docs via + // `paragraph_range_for_item_range`). Same semantics: `Some` iff the row is + // a paragraph block. + self.document.blocks.paragraph_ix_for_block_row(target_block_ix) } fn document_has_object_blocks(&self) -> bool { @@ -333,26 +310,16 @@ impl RichTextEditor { let mut found_start = false; for paragraph_ix in 0..paragraph_count { - let Some(paragraph) = self.document.paragraphs.get(paragraph_ix) else { + if self.document.paragraphs.get(paragraph_ix).is_none() { break; - }; - let key = paragraph_cache_key(&self.document, paragraph); + } let height = self - .paragraph_height_cache - .get(paragraph_ix) - .and_then(|entry| *entry) - .filter(|entry| { - entry.key == key - && entry.width == width - && entry.invisibility_mode == self.invisibility_mode - && entry.edit_generation == self.edit_generation - }) - .map(|entry| entry.height) + .valid_paragraph_height(paragraph_ix, width) .unwrap_or_else(|| { self .valid_paragraph_prep(paragraph_ix) .as_deref() - .map(|prep| estimate_paragraph_prep_item_height(&self.document, prep, width)) + .map(|prep| estimate_paragraph_prep_item_height(&self.document, prep, paragraph_ix, width)) .unwrap_or_else(|| estimate_paragraph_item_height_with_visibility(&self.document, paragraph_ix, width, self.invisibility_mode)) }); let next_y = y + height; @@ -420,6 +387,33 @@ impl RichTextEditor { } fn prepare_render_layout(&mut self, window: &mut Window, cx: &mut Context) -> RenderLayoutSnapshot { + // §hang-watchdog: time the whole render. The freeze shows as multi-second + // gaps between render passes; timing the render tells us whether the stall is + // INSIDE the render (this logs a `slow-render`) or between renders (collab / + // projection apply — then this stays quiet and the collab path is the cause). + // Logged always (not gated on `enabled()`), at WARN, so it survives even with + // fidelity tracing off and can't be lost to a dropped-under-load JSONL buffer. + let render_started = std::time::Instant::now(); + // §hang-probe: one line per render-layout pass capturing the virtualization + // state that drives the scroll-materialization loop. In a freeze this fires + // continuously; diffing consecutive lines shows WHICH field changes each + // iteration (the invalidation source): a drifting `scroll_y` means the stall + // guard's signature never settles (scroll-anchor restoration), a bumping + // `layout_gen`/`height_rev` means a per-frame cache clear, etc. + if flowstate_fidelity::enabled() { + let scroll_px: f32 = (-self.scroll_handle.offset().y).max(px(0.0)).into(); + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Structure, "render-layout-pass", || { + format!( + "scroll_y={scroll_px:.1} edit_gen={} layout_gen={} height_rev={} measured_w={:?} item_cache={} stall={}", + self.edit_generation, + self.layout_generation, + self.paragraph_height_cache_revision, + self.measured_item_width, + self.item_sizes_cache.is_some(), + self.scroll_materialize_stall_frames, + ) + }); + } let hide_until_viewport_measured = self.scroll_handle.bounds().size.width <= px(1.0); let mut item_sizes = self.paragraph_item_sizes(window, cx); let has_startup_layout_width = self.measured_item_width.is_some() || self.document.paragraphs.is_empty(); @@ -447,6 +441,14 @@ impl RichTextEditor { .map(|cache| cache.items.clone()) .unwrap_or_else(|| Rc::new(Vec::new())); let (items, item_sizes) = self.render_items_with_drop_preview(base_items, item_sizes, width, window, cx); + let render_ms = render_started.elapsed().as_millis(); + if render_ms > 150 { + let paragraphs = self.document.paragraphs.len(); + tracing::warn!("slow editor render pass (hang watchdog): {render_ms}ms, paragraphs={paragraphs}"); + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Structure, "slow-render", || { + format!("render took {render_ms}ms (paragraphs={paragraphs})") + }); + } RenderLayoutSnapshot { width, item_sizes, diff --git a/crates/gpui-flowtext/src/rich_text/editor/layout_prep.rs b/crates/gpui-flowtext/src/rich_text/editor/layout_prep.rs index 698fbbfc..e9484527 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/layout_prep.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/layout_prep.rs @@ -3,29 +3,53 @@ const LAYOUT_PREP_MAX_TEXT_BYTES_PER_BATCH: usize = 512 * 1024; #[hotpath::measure_all] impl RichTextEditor { + /// Resolve a paragraph index to its stable id, the key for the id-keyed + /// prep/shaping caches (§perf-heaven T8.12). + pub(super) fn paragraph_id_at(&self, paragraph_ix: usize) -> Option { + self.document.ids.paragraph_ids.get(paragraph_ix).copied() + } + fn resize_layout_aux_caches(&mut self) { let paragraph_count = self.document.paragraphs.len(); - self.paragraph_prep_cache.resize_with(paragraph_count, ParagraphPrepSlot::default); - self - .paragraph_shaping_cache - .resize_with(paragraph_count, || None); - self - .paragraph_estimate_height_cache - .resize(paragraph_count, None); + // §perf-heaven T8.12/T7.14: the prep, shaping, and estimate caches are all + // id-keyed maps — no positional resize. Deleted paragraphs leave stale id + // entries; bound each map so a long editing session cannot leak them without + // limit. §act-ten A10.12: when the bound trips, RETAIN the live ids instead + // of clearing wholesale — after a mass delete every SURVIVING paragraph + // used to re-prep/re-shape/re-estimate for no reason. + if self.paragraph_prep_cache.len() > paragraph_count * 2 + 64 + || self.paragraph_shaping_cache.len() > paragraph_count * 2 + 64 + || self.paragraph_estimate_height_cache.len() > paragraph_count * 2 + 64 + { + let live: std::collections::HashSet = self.document.ids.paragraph_ids.iter().copied().collect(); + self.paragraph_prep_cache.retain(|id, _| live.contains(id)); + self.paragraph_shaping_cache.retain(|id, _| live.contains(id)); + self.paragraph_estimate_height_cache.retain(|id, _| live.contains(id)); + } } - fn valid_paragraph_prep(&self, paragraph_ix: usize) -> Option> { + // §act-nine A9.3: validity is CONTENT-keyed — the (style, version) cache key + // plus the invisibility mode plus the stable paragraph id — with NO global + // `edit_generation`, so edits to OTHER paragraphs keep this prep. Soundness + // rests on version discipline (every content change bumps the version; + // structural rebuilds and canonical installs carry versions forward instead + // of resetting to 0). §act-eleven A11.7: NO positional check either — a + // structural shift (Enter/delete-row above) moves a paragraph without + // changing its content, and (id, style, version) already pins exactly that + // content; the prep's stored `paragraph_ix` is build-time context only + // (position-dependent consumers take the CURRENT index as a parameter). + pub(super) fn valid_paragraph_prep(&self, paragraph_ix: usize) -> Option> { let paragraph = self.document.paragraphs.get(paragraph_ix)?; + let paragraph_id = self.paragraph_id_at(paragraph_ix)?; let expected_key = ParagraphPrepKey { paragraph_key: paragraph_cache_key(&self.document, paragraph), invisibility_mode: self.invisibility_mode, - edit_generation: self.edit_generation, }; self .paragraph_prep_cache - .get(paragraph_ix) + .get(¶graph_id) .and_then(|slot| slot.get(self.invisibility_mode)) - .filter(|prep| prep.paragraph_ix == paragraph_ix && prep.key == expected_key) + .filter(|prep| prep.paragraph_id == paragraph_id && prep.key == expected_key) .cloned() } @@ -36,20 +60,13 @@ impl RichTextEditor { self.valid_paragraph_prep(paragraph_ix).is_none() } - fn ensure_paragraph_prep_sync(&mut self, paragraph_ix: usize) -> Option> { + pub(super) fn ensure_paragraph_prep_sync(&mut self, paragraph_ix: usize) -> Option> { if let Some(prep) = self.valid_paragraph_prep(paragraph_ix) { return Some(prep); } - let prep = Arc::new(build_paragraph_prep( - &self.document, - paragraph_ix, - self.edit_generation, - self.invisibility_mode, - )?); + let prep = Arc::new(build_paragraph_prep(&self.document, paragraph_ix, self.invisibility_mode)?); self.resize_layout_aux_caches(); - if let Some(slot) = self.paragraph_prep_cache.get_mut(paragraph_ix) { - slot.set(prep.clone()); - } + self.paragraph_prep_cache.entry(prep.paragraph_id).or_default().set(prep.clone()); Some(prep) } @@ -67,7 +84,6 @@ impl RichTextEditor { } let request = LayoutPrepRequest { width, - edit_generation: self.edit_generation, invisibility_mode: self.invisibility_mode, paragraphs, }; @@ -83,7 +99,7 @@ impl RichTextEditor { self.pending_layout_prep_request = Some(request); return; }; - if pending.edit_generation != request.edit_generation || pending.invisibility_mode != request.invisibility_mode { + if pending.invisibility_mode != request.invisibility_mode { *pending = request; return; } @@ -103,7 +119,6 @@ impl RichTextEditor { if !overflow.is_empty() { self.merge_pending_layout_prep_request(LayoutPrepRequest { width: request.width, - edit_generation: request.edit_generation, invisibility_mode: request.invisibility_mode, paragraphs: overflow, }); @@ -111,7 +126,6 @@ impl RichTextEditor { let width = request.width; let batch = paragraph_prep_batch_request( &self.document, - request.edit_generation, request.invisibility_mode, request.paragraphs, LAYOUT_PREP_MAX_PARAGRAPHS_PER_BATCH, @@ -141,14 +155,14 @@ impl RichTextEditor { ); } - fn install_layout_prep_batch(&mut self, width: Pixels, result: ParagraphPrepBatchResult, cx: &mut Context) { + pub(super) fn install_layout_prep_batch(&mut self, width: Pixels, result: ParagraphPrepBatchResult, cx: &mut Context) { self.resize_layout_aux_caches(); self.layout_prep_metrics.batches = self.layout_prep_metrics.batches.saturating_add(1); self.layout_prep_metrics.requested = self.layout_prep_metrics.requested.saturating_add(result.requested); self.layout_prep_metrics.completed = self.layout_prep_metrics.completed.saturating_add(result.completed); self.layout_prep_metrics.text_bytes = self.layout_prep_metrics.text_bytes.saturating_add(result.text_bytes); - if result.edit_generation == self.edit_generation && result.invisibility_mode == self.invisibility_mode { + if result.invisibility_mode == self.invisibility_mode { let deferred = result .deferred_paragraphs .iter() @@ -158,7 +172,6 @@ impl RichTextEditor { if !deferred.is_empty() { self.merge_pending_layout_prep_request(LayoutPrepRequest { width, - edit_generation: result.edit_generation, invisibility_mode: result.invisibility_mode, paragraphs: deferred, }); @@ -166,25 +179,40 @@ impl RichTextEditor { } let mut installed = 0usize; + // §act-eleven A11.7: resolve each prep's CURRENT index by id (one map for + // the whole batch) — the prep's build-time position may have shifted + // structurally during the background build, and a shifted-but-unchanged + // paragraph must still install (the id-keyed cache doesn't care where the + // paragraph lives). + let index_by_id: FxHashMap = self + .document + .ids + .paragraph_ids + .iter() + .enumerate() + .map(|(current_ix, id)| (*id, current_ix)) + .collect(); for prep in result.preps { - let paragraph_ix = prep.paragraph_ix; - let valid = result.edit_generation == self.edit_generation - && result.invisibility_mode == self.invisibility_mode - && prep.key.edit_generation == self.edit_generation - && prep.key.invisibility_mode == self.invisibility_mode - && self - .document - .paragraphs - .get(paragraph_ix) - .is_some_and(|paragraph| paragraph_cache_key(&self.document, paragraph) == prep.key.paragraph_key); + // §act-nine A9.3 install gate: a completed background prep installs iff + // its content is STILL CURRENT — same paragraph id (wherever it now + // lives) with the same (style, version) content key and invisibility + // mode. No `edit_generation` conjunct: an unrelated edit landing during + // the batch must not discard prep for untouched paragraphs (the + // background-prep thrash during typing bursts). + let valid = prep.key.invisibility_mode == self.invisibility_mode + && index_by_id.get(&prep.paragraph_id).is_some_and(|¤t_ix| { + self + .document + .paragraphs + .get(current_ix) + .is_some_and(|paragraph| paragraph_cache_key(&self.document, paragraph) == prep.key.paragraph_key) + }); if !valid { self.layout_prep_metrics.stale = self.layout_prep_metrics.stale.saturating_add(1); continue; } - if let Some(slot) = self.paragraph_prep_cache.get_mut(paragraph_ix) { - slot.set(Arc::new(prep)); - installed += 1; - } + self.paragraph_prep_cache.entry(prep.paragraph_id).or_default().set(Arc::new(prep)); + installed += 1; } if installed == 0 { return; @@ -197,45 +225,19 @@ impl RichTextEditor { } fn clear_all_layout_prep(&mut self) { - for slot in &mut self.paragraph_prep_cache { - slot.clear(); - } + self.paragraph_prep_cache.clear(); self.pending_layout_prep_task = None; self.pending_layout_prep_request = None; } - - fn clear_layout_prep_range(&mut self, range: Range) { - self.resize_layout_aux_caches(); - for paragraph_ix in range { - if let Some(slot) = self.paragraph_prep_cache.get_mut(paragraph_ix) { - slot.clear(); - } - } - self.pending_layout_prep_request = None; - } - fn clear_layout_work_caches(&mut self) { self.layout_generation = self.layout_generation.wrapping_add(1); self.paragraph_shaping_cache.clear(); - self.paragraph_shaping_cache.resize_with(self.document.paragraphs.len(), || None); self.layout_cache_retain_ranges = ParagraphCacheRetainRanges::default(); self.prep_cache_retain_ranges = ParagraphCacheRetainRanges::default(); self.pending_chunk_prefetch = false; self.chunk_prefetch_queue.clear(); } - - fn clear_layout_work_cache_range(&mut self, range: Range) { - self.resize_layout_aux_caches(); - for paragraph_ix in range { - if let Some(cache) = self.paragraph_shaping_cache.get_mut(paragraph_ix) { - *cache = None; - } - } - self.pending_chunk_prefetch = false; - self.chunk_prefetch_queue.clear(); - } - - fn paragraph_work_key(&self, prep: &ParagraphPrep, width: Pixels) -> ParagraphLayoutWorkKey { + pub(super) fn paragraph_work_key(&self, prep: &ParagraphPrep, width: Pixels) -> ParagraphLayoutWorkKey { ParagraphLayoutWorkKey { prep_key: prep.key, width, @@ -243,18 +245,18 @@ impl RichTextEditor { } } - fn take_paragraph_shape_cache(&mut self, paragraph_ix: usize, key: ParagraphLayoutWorkKey) -> FragmentShapeCache { + pub(super) fn take_paragraph_shape_cache(&mut self, paragraph_ix: usize, key: ParagraphLayoutWorkKey) -> FragmentShapeCache { self.resize_layout_aux_caches(); - match self.paragraph_shaping_cache.get_mut(paragraph_ix).and_then(Option::take) { + match self.paragraph_id_at(paragraph_ix).and_then(|id| self.paragraph_shaping_cache.remove(&id)) { Some(entry) if entry.key == key => entry.fragment_shapes, _ => FragmentShapeCache::default(), } } - fn store_paragraph_shape_cache(&mut self, paragraph_ix: usize, key: ParagraphLayoutWorkKey, fragment_shapes: FragmentShapeCache) { + pub(super) fn store_paragraph_shape_cache(&mut self, paragraph_ix: usize, key: ParagraphLayoutWorkKey, fragment_shapes: FragmentShapeCache) { self.resize_layout_aux_caches(); - if let Some(slot) = self.paragraph_shaping_cache.get_mut(paragraph_ix) { - *slot = Some(ParagraphShapingCacheEntry { key, fragment_shapes }); + if let Some(paragraph_id) = self.paragraph_id_at(paragraph_ix) { + self.paragraph_shaping_cache.insert(paragraph_id, ParagraphShapingCacheEntry { key, fragment_shapes }); } } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/lifecycle.rs b/crates/gpui-flowtext/src/rich_text/editor/lifecycle.rs index 315f3d34..191cfcff 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/lifecycle.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/lifecycle.rs @@ -1,3 +1,44 @@ +// Projection snapshots carry semantic style slots; their appearance catalog is editor-local. +fn projection_with_local_theme(mut document: DocumentProjection, theme: &DocumentTheme) -> DocumentProjection { + document.theme = theme.clone(); + document +} + +// §act-nine A9.3 version discipline (prerequisite for content-keyed layout +// caches): a canonical install replaces THE projection with materializer +// output whose paragraph versions all restart at 0. The layout caches key on +// (style, version), so a surviving paragraph id whose content changed would +// collide with its cached entries and serve stale layout. Carry every +// surviving id's version FORWARD — always advanced by one, so a +// (style, version) pair is never reused for possibly-different content. +// Conservative by design: surviving-but-unchanged paragraphs re-prep once per +// canonical install (a rare whole-document event). Genuinely new ids keep 0. +fn carry_forward_paragraph_versions(previous: &DocumentProjection, incoming: &mut DocumentProjection) { + if previous.paragraphs.is_empty() || incoming.paragraphs.is_empty() { + return; + } + let mut previous_versions: FxHashMap = FxHashMap::default(); + for (ix, id) in previous.ids.paragraph_ids.iter().enumerate() { + if let Some(paragraph) = previous.paragraphs.get(ix) { + previous_versions.insert(*id, paragraph.version); + } + } + let ids = incoming.ids.paragraph_ids.clone(); + for (ix, id) in ids.iter().enumerate() { + let Some(previous_version) = previous_versions.get(id) else { + continue; + }; + if let Some(paragraph) = paragraphs_mut(incoming).get_mut(ix) { + paragraph.version = previous_version.wrapping_add(1); + } + // Mirror into the block copy — the projection invariant every other + // paragraph mutation site maintains. + if let Some(row) = incoming.blocks.block_row_for_paragraph_ix(ix) { + update_paragraph_block_at(incoming, ix, row); + } + } +} + #[hotpath::measure_all] impl RichTextEditor { pub fn clear_document_equation_caches(&self) { @@ -8,7 +49,7 @@ impl RichTextEditor { EquationRenderer::clear_entries(keys); } - pub fn new_with_path(mut document: Document, document_path: Option, cx: &mut Context) -> Self { + pub fn new_with_path(mut document: DocumentProjection, document_path: Option, cx: &mut Context) -> Self { rebuild_document_sections(&mut document); let paragraph_count = document.paragraphs.len(); let saved_generation = if document_path.is_some() { 0 } else { u64::MAX }; @@ -24,8 +65,11 @@ impl RichTextEditor { .map(|name| SharedString::from(name.to_string_lossy().to_string())), recovery_path: document_path.as_deref().map(recovery_path_for_document), document_path, + write_authority: None, document, selection: EditorSelection::caret(), + caret_anchor: None, + selection_movement_epoch: 0, config: RichTextEditorConfig::default(), edit_generation: 0, saved_generation, @@ -34,11 +78,13 @@ impl RichTextEditor { last_format_export_generation: None, zoom_percent: 100.0, save_status: SaveStatus::Saved, - undo_stack: Vec::new(), - redo_stack: Vec::new(), identity_map, - last_collaboration_edit: None, + reconciliation_recoveries: 0, + native_save_hook: None, + native_export_hook: None, + native_recovery_hook: None, collaboration_role: None, + own_collaboration_caret_color_rgb: None, recovery_write_in_progress: false, recovery_write_pending: false, last_recovery_generation: 0, @@ -68,15 +114,18 @@ impl RichTextEditor { caret_visible: true, caret_blink_active: false, external_carets: Vec::new(), + external_selections: Vec::new(), + annotation_selections: Vec::new(), search_highlights: Vec::new(), active_search_highlight: None, last_text_input_at: None, + ime_marked_range: None, pending_typing_prefetch_resume: false, resume_chunk_prefetch_after_typing: false, paragraph_chunk_layout_cache: vec![None; paragraph_count], - paragraph_prep_cache: vec![ParagraphPrepSlot::default(); paragraph_count], - paragraph_shaping_cache: (0..paragraph_count).map(|_| None).collect(), - paragraph_estimate_height_cache: vec![None; paragraph_count], + paragraph_prep_cache: FxHashMap::default(), + paragraph_shaping_cache: FxHashMap::default(), + paragraph_estimate_height_cache: FxHashMap::default(), pending_layout_prep_task: None, pending_layout_prep_request: None, layout_generation: 0, @@ -86,9 +135,10 @@ impl RichTextEditor { chunk_prefetch_queue: VecDeque::new(), paragraph_height_cache: vec![None; paragraph_count], paragraph_height_cache_revision: 0, + scroll_materialize_signature: None, + scroll_materialize_stall_frames: 0, item_sizes_cache: None, pending_item_sizes_patch_range: None, - layout_invalidation_hint: None, suppress_mutation_notify: 0, last_scroll_anchor: None, scroll_anchor_lock: None, @@ -110,10 +160,24 @@ impl RichTextEditor { } } - pub fn document(&self) -> &Document { + pub fn document(&self) -> &DocumentProjection { &self.document } + fn emit_selection_changed(&mut self, cx: &mut Context) { + // Re-arm the caret's CRDT anchor for the NEW selection while we are in sync + // with the core (the fast O(log n) path for repositioning the caret across a + // remote edit). Without this the anchor only refreshed after a write/sync, + // so the first edit — or an incoming remote edit — after any caret move fell + // to the O(doc) `fork_at` rebase (~350ms on a large doc). Fires once per + // selection change; `encode_selection_anchor` is a no-op when out of sync or + // the caret is non-body. + self.capture_caret_anchor(); + cx.emit(EditorEvent::SelectionChanged { + selection: self.selection.clone(), + }); + } + pub fn dispose_for_close(&mut self) { if self.disposed { return; @@ -128,7 +192,10 @@ impl RichTextEditor { self.recovery_path = None; self.document = blank_document(); self.identity_map = DocumentIdentityMap::new(&self.document); + let fid_sel_before = self.fidelity_caret_before(); self.selection = EditorSelection::caret(); + self.fidelity_caret_set("dispose_for_close", &fid_sel_before); + self.selection_movement_epoch = 0; self.edit_generation = 0; self.saved_generation = 0; self.next_edit_generation = 1; @@ -143,10 +210,11 @@ impl RichTextEditor { } fn release_transient_memory(&mut self) { - self.undo_stack = Vec::new(); - self.redo_stack = Vec::new(); - self.last_collaboration_edit = None; + self.native_save_hook = None; + self.native_export_hook = None; + self.native_recovery_hook = None; self.collaboration_role = None; + self.own_collaboration_caret_color_rgb = None; self.recovery_write_in_progress = false; self.recovery_write_pending = false; self.paste_cache = None; @@ -175,13 +243,16 @@ impl RichTextEditor { self.caret_visible = false; self.caret_blink_active = false; self.external_carets.clear(); + self.external_selections.clear(); + self.annotation_selections.clear(); self.last_text_input_at = None; + self.ime_marked_range = None; self.pending_typing_prefetch_resume = false; self.resume_chunk_prefetch_after_typing = false; self.paragraph_chunk_layout_cache = Vec::new(); - self.paragraph_prep_cache = Vec::new(); - self.paragraph_shaping_cache = Vec::new(); - self.paragraph_estimate_height_cache = Vec::new(); + self.paragraph_prep_cache = FxHashMap::default(); + self.paragraph_shaping_cache = FxHashMap::default(); + self.paragraph_estimate_height_cache = FxHashMap::default(); self.pending_layout_prep_task = None; self.pending_layout_prep_request = None; self.layout_generation = self.layout_generation.wrapping_add(1); @@ -193,7 +264,6 @@ impl RichTextEditor { self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); self.item_sizes_cache = None; self.pending_item_sizes_patch_range = None; - self.layout_invalidation_hint = None; self.suppress_mutation_notify = 0; self.last_scroll_anchor = None; self.scroll_anchor_lock = None; @@ -211,27 +281,51 @@ impl RichTextEditor { self.goal_x = None; } - pub fn last_collaboration_edit(&self) -> Option<&CollaborationEdit> { - self.last_collaboration_edit.as_ref() + /// Count of reconciliation recoveries since this editor was created. A + /// nonzero value means optimistic state diverged from the canonical + /// projection and had to be repaired — always a bug worth investigating. + pub fn reconciliation_recoveries(&self) -> u64 { + self.reconciliation_recoveries } - pub fn last_collaboration_operations(&self) -> Option<&[CanonicalOperation]> { - self - .last_collaboration_edit - .as_ref() - .map(|edit| edit.operations.as_slice()) + pub fn restore_runtime_selection(&mut self, selection: EditorSelection, cx: &mut Context) { + let fid_sel_before = self.fidelity_caret_before(); + self.selection = selection; + clamp_selection_to_document(&self.document, &mut self.selection); + self.fidelity_caret_set("restore_runtime_selection", &fid_sel_before); + self.emit_selection_changed(cx); + self.scroll_head_into_view(); + cx.notify(); } - pub fn last_collaboration_operation_bytes(&self) -> Option> { - self - .last_collaboration_edit - .as_ref() - .and_then(|edit| crate::encode_canonical_operations(&edit.operations)) + pub fn mark_as_unsaved_branch(&mut self, cx: &mut Context) { + self.edit_generation = self.next_edit_generation; + self.next_edit_generation = self.next_edit_generation.wrapping_add(1); + self.refresh_save_status(); + cx.notify(); + } + + pub(super) fn note_explicit_selection_movement(&mut self) { + self.selection_movement_epoch = self.selection_movement_epoch.wrapping_add(1); + } + + pub fn set_native_save_hook(&mut self, hook: Option) { + self.native_save_hook = hook; + } + + pub fn set_native_export_hook(&mut self, hook: Option) { + self.native_export_hook = hook; + } + + pub fn set_native_recovery_hook(&mut self, hook: Option) { + self.native_recovery_hook = hook; } - pub fn clear_collaboration_edit(&mut self) { - self.last_collaboration_edit = None; + pub fn set_recovery_path(&mut self, path: Option, cx: &mut Context) { + self.recovery_path = path; + self.schedule_recovery_write(cx); } + pub fn collaboration_role(&self) -> Option { self.collaboration_role } @@ -258,180 +352,45 @@ impl RichTextEditor { self.identity_map.block_id(block_ix) } - pub fn table_cell_id(&self, block_ix: usize, row_ix: usize, cell_ix: usize) -> Option { - self.identity_map.table_cell_id(block_ix, row_ix, cell_ix) + fn semantic_block_id(&self, block_ix: usize) -> Option { + self + .identity_map + .block_id(block_ix) + .or_else(|| self.document.ids.block_ids.get(block_ix).copied()) } - pub fn apply_remote_operations(&mut self, operations: &[CanonicalOperation], cx: &mut Context) { - for operation in operations { - self.apply_canonical_operation(operation); - } - self.identity_map.reconcile(&self.document); - self.last_collaboration_edit = None; + pub fn replace_document_projection(&mut self, document: DocumentProjection, cx: &mut Context) { + self.install_canonical_projection(document, cx); + // Whole-document replacement: the stale-scan (content-key revalidation of + // every slot) is warranted HERE — the shared mutation hook no longer + // invalidates anything (§act-ten A10.12). + self.invalidate_stale_paragraph_layout_caches(); self.after_text_mutation(cx); } - pub fn replace_document_from_collaboration(&mut self, document: Document, cx: &mut Context) { - self.document = document; - self.identity_map.reconcile(&self.document); - self.last_collaboration_edit = None; - self.after_text_mutation(cx); + pub fn document_path(&self) -> Option<&PathBuf> { + self.document_path.as_ref() } - fn apply_canonical_operation(&mut self, operation: &CanonicalOperation) { - match operation { - CanonicalOperation::InsertText { - paragraph, - byte, - text, - styles, - } => { - if let Some(paragraph_ix) = self.identity_map.paragraph_index(*paragraph) - && paragraph_offset_in_bounds( - &self.document, - DocumentOffset { - paragraph: paragraph_ix, - byte: *byte, - }, - ) - { - insert_text_at(&mut self.document, paragraph_ix, *byte, text, *styles); - } - }, - CanonicalOperation::DeleteRange { - start_paragraph, - start_byte, - end_paragraph, - end_byte, - } => { - let Some(start_paragraph) = self.identity_map.paragraph_index(*start_paragraph) else { - return; - }; - let Some(end_paragraph) = self.identity_map.paragraph_index(*end_paragraph) else { - return; - }; - if paragraph_offset_in_bounds( - &self.document, - DocumentOffset { - paragraph: start_paragraph, - byte: *start_byte, - }, - ) && paragraph_offset_in_bounds( - &self.document, - DocumentOffset { - paragraph: end_paragraph, - byte: *end_byte, - }, - ) { - delete_cross_paragraph_range( - &mut self.document, - DocumentOffset { - paragraph: start_paragraph, - byte: *start_byte, - }..DocumentOffset { - paragraph: end_paragraph, - byte: *end_byte, - }, - ); - } - }, - CanonicalOperation::SplitParagraph { paragraph, byte, .. } => { - if let Some(paragraph_ix) = self.identity_map.paragraph_index(*paragraph) - && paragraph_offset_in_bounds( - &self.document, - DocumentOffset { - paragraph: paragraph_ix, - byte: *byte, - }, - ) - { - split_paragraph_at(&mut self.document, paragraph_ix, *byte); - } - }, - CanonicalOperation::JoinParagraphs { first, second } => { - let Some(first_ix) = self.identity_map.paragraph_index(*first) else { - return; - }; - let Some(second_ix) = self.identity_map.paragraph_index(*second) else { - return; - }; - if second_ix == first_ix + 1 { - let byte = paragraph_text_len(&self.document.paragraphs[first_ix]); - delete_cross_paragraph_range( - &mut self.document, - DocumentOffset { paragraph: first_ix, byte }..DocumentOffset { - paragraph: second_ix, - byte: 0, - }, - ); - } - }, - CanonicalOperation::SetParagraphStyle { paragraph, style } => { - if let Some(paragraph_ix) = self.identity_map.paragraph_index(*paragraph) - && let Some(paragraph) = paragraphs_mut(&mut self.document).get_mut(paragraph_ix) - { - paragraph.style = *style; - bump_paragraph_version(paragraph); - update_paragraph_block(&mut self.document, paragraph_ix); - rebuild_document_sections(&mut self.document); - } - }, - CanonicalOperation::SetRunStyles { paragraph, range, styles } => { - if let Some(paragraph_ix) = self.identity_map.paragraph_index(*paragraph) - && paragraph_offset_in_bounds( - &self.document, - DocumentOffset { - paragraph: paragraph_ix, - byte: range.start, - }, - ) - && paragraph_offset_in_bounds( - &self.document, - DocumentOffset { - paragraph: paragraph_ix, - byte: range.end, - }, - ) - { - mutate_runs_in_range( - &mut self.document, - DocumentOffset { - paragraph: paragraph_ix, - byte: range.start, - }..DocumentOffset { - paragraph: paragraph_ix, - byte: range.end, - }, - |run_styles| *run_styles = *styles, - ); - } - }, - CanonicalOperation::ReplaceParagraphSpan { - start_paragraph, - before, - after, - } => { - let start = start_paragraph - .and_then(|id| self.identity_map.paragraph_index(id)) - .unwrap_or(before.start_paragraph); - let current = capture_document_span(&self.document, start..start + before.paragraphs.len()); - let replacement = DocumentSpan { - start_paragraph: start, - paragraphs: after.paragraphs.clone(), - text: after.text.clone(), - }; - apply_document_span_replacement(&mut self.document, ¤t, &replacement); - }, - CanonicalOperation::InsertBlock { .. } - | CanonicalOperation::DeleteBlock { .. } - | CanonicalOperation::MoveBlock { .. } - | CanonicalOperation::ReplaceBlock { .. } - | CanonicalOperation::ReplaceDocument => {}, - } + pub fn set_document_path_for_runtime(&mut self, path: PathBuf, cx: &mut Context) { + self.document_path = Some(path.clone()); + self.recovery_path = Some(recovery_path_for_document(&path)); + cx.notify(); } - pub fn document_path(&self) -> Option<&PathBuf> { - self.document_path.as_ref() + /// §data-loss fix (2026-07-08): reconcile the editor's WRITE path to the + /// authority's canonical open path when the runtime attaches — including + /// resetting it to `None`. Imported formats (docx/pdf) load with `None`: the + /// source file is NOT a save target, so save/autosave must stay disabled + /// until the user picks a `.db8` via Save As. The phase-V pending panel seeds + /// the editor with the *source* path for display/recents; without this reset + /// that source path becomes the autosave target and gets overwritten with a + /// `.db8` journal (the docx-clobber bug). Mirrors the one-shot open path, + /// which builds the editor with `loaded.path` directly. + pub fn set_runtime_document_path(&mut self, path: Option, cx: &mut Context) { + self.recovery_path = path.as_ref().map(|path| recovery_path_for_document(path)); + self.document_path = path; + cx.notify(); } pub fn set_document_display_name(&mut self, name: SharedString, cx: &mut Context) { @@ -460,10 +419,42 @@ impl RichTextEditor { cx.notify(); } + pub fn set_show_own_collaboration_caret_color(&mut self, enabled: bool, cx: &mut Context) { + if self.config.show_own_collaboration_caret_color != enabled { + self.config.show_own_collaboration_caret_color = enabled; + cx.notify(); + } + } + + pub fn set_own_collaboration_caret_color(&mut self, color_rgb: Option, cx: &mut Context) { + if self.own_collaboration_caret_color_rgb != color_rgb { + self.own_collaboration_caret_color_rgb = color_rgb; + cx.notify(); + } + } + + pub(super) fn local_caret_color_rgb(&self) -> Option { + self + .config + .show_own_collaboration_caret_color + .then_some(self.own_collaboration_caret_color_rgb) + .flatten() + } + pub fn save_status(&self) -> &SaveStatus { &self.save_status } + /// Install a selection (host/test surface). Clamped to the document; emits + /// the selection-changed event. + pub fn set_selection(&mut self, selection: EditorSelection, cx: &mut Context) { + let anchor = self.clamp_offset_to_document(selection.anchor); + let head = self.clamp_offset_to_document(selection.head); + self.selection = EditorSelection { anchor, head, ..selection }; + self.emit_selection_changed(cx); + cx.notify(); + } + pub fn selection(&self) -> &EditorSelection { &self.selection } @@ -483,11 +474,134 @@ impl RichTextEditor { .cloned() .collect() } + + pub fn set_external_selections(&mut self, external_selections: Vec, cx: &mut Context) { + if self.external_selections != external_selections { + self.external_selections = external_selections; + cx.notify(); + } + } + + /// Install non-presence annotation ranges, such as unresolved comment + /// anchors. These are an overlay only and never mutate document run styles. + pub fn set_annotation_selections(&mut self, selections: Vec, cx: &mut Context) { + if self.annotation_selections != selections { + self.annotation_selections = selections; + cx.notify(); + } + } + + /// Peer selection spans that intersect `paragraph_ix`. A multi-paragraph + /// selection is returned for every paragraph it covers; the shared paint path + /// slices the correct byte span per paragraph, mirroring the local selection. + pub(super) fn external_selections_for_paragraph(&self, paragraph_ix: usize) -> Vec { + self + .external_selections + .iter() + .filter(|external| { + let range = external.selection.normalized(); + range.start.paragraph <= paragraph_ix && range.end.paragraph >= paragraph_ix + }) + .cloned() + .collect() + } + + pub(super) fn annotation_selections_for_paragraph(&self, paragraph_ix: usize) -> Vec { + self + .annotation_selections + .iter() + .filter(|annotation| { + let range = annotation.selection.normalized(); + range.start.paragraph <= paragraph_ix && range.end.paragraph >= paragraph_ix + }) + .cloned() + .collect() + } } -fn paragraph_offset_in_bounds(document: &Document, offset: DocumentOffset) -> bool { - document - .paragraphs - .get(offset.paragraph) - .is_some_and(|paragraph| offset.byte <= paragraph_text_len(paragraph)) +// -- Fidelity instrumentation helpers ------------------------------------- +// +// Strictly additive diagnostics for caret/reconciliation/text fidelity. Kept in +// a separate impl block WITHOUT `#[hotpath::measure_all]` so the caret hot path +// is not wrapped in profiling timers, and so every entry point stays gated by a +// single relaxed atomic load (`flowstate_fidelity::enabled`) when tracing is +// off. None of these read/write anything that alters editor behavior. +impl RichTextEditor { + /// Snapshot the live selection immediately before a `self.selection =` write, + /// but only when fidelity tracing is on. Returns `None` (and clones nothing) + /// when off, so the common caret hot path pays one atomic load. + #[inline] + pub(super) fn fidelity_caret_before(&self) -> Option { + flowstate_fidelity::enabled().then(|| self.selection.clone()) + } + + /// Emit the `Caret`/`set` firehose event for a completed selection write whose + /// pre-value was captured via [`Self::fidelity_caret_before`]. No-op when the + /// snapshot is `None` (tracing off). + #[inline] + #[allow( + clippy::ref_option, + reason = "gated diagnostic helper borrows the caller's Option-typed caret snapshot; Option<&T> would push .as_ref() onto every call site" + )] + pub(super) fn fidelity_caret_set(&self, site: &'static str, before: &Option) { + if let Some(old) = before { + self.fidelity_caret_set_from(site, old); + } + } + + /// Emit the `Caret`/`set` firehose event for a completed selection write whose + /// pre-value is already held in an existing local (no extra clone). The detail + /// closure is lazy, so this costs one atomic load when tracing is off. + #[inline] + pub(super) fn fidelity_caret_set_from(&self, site: &'static str, old: &EditorSelection) { + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Caret, "set", || { + format!( + "site={site} old={old:?} new={:?} frontier_len={} gen={}", + self.selection, + self.document.frontier.len(), + self.edit_generation, + ) + }); + } +} + +fn structural_block_for_input(block_id: BlockId, paragraph_id: Option, block: InputBlock) -> ProjectionStructuralBlock { + ProjectionStructuralBlock { + block_id, + paragraph_id: match &block { + InputBlock::Paragraph(_) => Some(paragraph_id.unwrap_or_else(new_paragraph_id)), + InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_) => None, + }, + block, + } +} + +fn table_column_width_from_input_width(width: &InputTableColumnWidth) -> TableColumnWidth { + match *width { + InputTableColumnWidth::Auto => TableColumnWidth::Auto, + InputTableColumnWidth::FixedPx(px) => TableColumnWidth::FixedPx(px), + InputTableColumnWidth::Fraction(fraction) => TableColumnWidth::Fraction(fraction), + } +} + +#[cfg(test)] +mod projection_theme_tests { + use super::*; + + #[test] + fn replacement_projection_preserves_local_theme_catalog() { + let mut current = blank_document(); + current.theme.body_font_size = px(18.0); + current.theme.custom_highlight_styles.insert( + 77, + CustomHighlightStyle { + color: rgb(0x0012_3456).into(), + }, + ); + + let replacement = projection_with_local_theme(blank_document(), ¤t.theme); + + assert_eq!(replacement.theme.body_font_size, px(18.0)); + assert!(replacement.theme.custom_highlight_styles.contains_key(&77)); + } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/local_write_path.rs b/crates/gpui-flowtext/src/rich_text/editor/local_write_path.rs new file mode 100644 index 00000000..395d15c1 --- /dev/null +++ b/crates/gpui-flowtext/src/rich_text/editor/local_write_path.rs @@ -0,0 +1,556 @@ +// The editor's Loro-first write path (spec §5, D2/D3). +// +// Every local mutation flows through here: build a typed intent from the +// user's action + selection, call the injected [`LocalWriteAuthority`] +// synchronously (the commit happens inside the call, under the write gate), +// and advance THE projection with the returned exact patches. There is no +// optimistic state, no command capture, no pending queue, no reconcile — the +// "optimistic" apply IS the committed apply. +// +// Rejections (spec I-15): restore the caret to the nearest valid position, +// surface a recovery notice, never retry unchanged, never synthesize a +// replacement mutation from the visible projection. + +// NOTE: spliced into editor/mod.rs via include! like every editor submodule — +// names resolve in that scope; local-intents types are imported there. + +impl RichTextEditor { + /// Install the document's write authority (the ONE local write path — + /// identical object for solo and collaborative documents, invariant 5) and + /// adopt its canonical projection as THE document. + pub fn set_write_authority(&mut self, authority: std::sync::Arc, projection: DocumentProjection, cx: &mut Context) { + self.write_authority = Some(authority); + self.install_canonical_projection(projection, cx); + } + + #[must_use] + pub fn has_write_authority(&self) -> bool { + self.write_authority.is_some() + } + + /// Replace THE projection wholesale (authority attach, undo/redo, rare + /// audited full rebuilds, remote `ProjectionUpdated`). + pub fn install_canonical_projection(&mut self, projection: DocumentProjection, cx: &mut Context) { + let theme = self.document.theme.clone(); + let mut incoming = projection_with_local_theme(projection, &theme); + // §act-nine A9.3: canonical output restarts every paragraph version at 0; + // carry surviving ids' versions forward so the content-keyed layout caches + // can never serve a stale (style, version) collision. + carry_forward_paragraph_versions(&self.document, &mut incoming); + self.document = incoming; + self.identity_map.reconcile(&self.document); + self.clamp_selection_to_document(cx); + let generation = self.next_edit_generation; + self.next_edit_generation = self.next_edit_generation.wrapping_add(1); + self.mark_document_changed(generation, false, cx); + } + + /// Build a stable text anchor for a render-space offset (identity + hint; + /// raw offsets never leave the editor as authority — spec I-2). + #[must_use] + pub(super) fn text_anchor_at(&self, offset: DocumentOffset) -> Option { + let paragraph = self.identity_map.paragraph_id(offset.paragraph)?; + Some(TextAnchor::new(paragraph, offset.byte)) + } + + /// Central dispatch: run one intent through the authority and integrate the + /// outcome. Returns the commit on success. All `write_*` helpers route here. + pub(super) fn write_intent(&mut self, intent: LocalIntent, cx: &mut Context) -> Option { + let Some(authority) = self.write_authority.clone() else { + // No authority ⇒ the document is read-only (display surface). Nothing + // mutates locally; there is no fallback editing mode (invariant 5). + return None; + }; + let class = intent.class(); + match authority.apply(intent) { + Ok(outcome) => Some(self.integrate_outcome(outcome, cx)), + Err(rejection) => { + self.handle_write_rejection(class, &rejection, cx); + None + }, + } + } + + fn integrate_outcome(&mut self, outcome: LocalWriteOutcome, cx: &mut Context) -> LocalCommit { + // Field fix 2026-07-07: the projection advances ONLY by draining the + // core's ordered stream — the intent's own batch arrives as the tail of + // the drain, AFTER any remote batches that committed before it. Applying + // the returned batch directly (the old shape) raced async remote delivery + // and produced the base-frontier mismatch cascade in the field logs. + // Local variant: the caret is set from `commit.selection_after` below, so + // the O(doc) caret rebase is skipped (it froze typing after every caret + // move on large docs). + self.sync_projection_local(cx); + let commit = match outcome { + LocalWriteOutcome::Committed(commit) => commit, + LocalWriteOutcome::CommittedWithRebuild { commit, .. } => commit, + }; + if let Some(selection) = &commit.selection_after { + let caret = self.clamp_offset_to_document(selection.head.offset); + self.set_caret_after_local_write(caret, cx); + } + // Re-arm the fast path for the NEW post-write caret (the capture inside the + // sync above was for the pre-write selection). Keeps the typing peer on the + // O(log n) path when a remote edit lands between its own keystrokes. + self.capture_caret_anchor(); + commit + } + + /// Drain the core's ordered projection stream and apply every item in + /// commit order. The ONE way this editor's projection advances (local + /// intents call it synchronously; the collaboration session calls it as a + /// pump when remote batches land; solo documents need no pump). + /// + /// The remote pump passes `rebase_caret = true` so a remote insert/delete + /// before the caret repositions it (the interleave fix). LOCAL callers pass + /// `false`: they set the caret explicitly from their own commit right after + /// this returns, so the caret-rebase here is not only wasted — its O(doc) + /// `fork_at` fallback (taken whenever the caret moved since the last synced + /// capture, i.e. the first edit after every click/arrow) froze typing on + /// large docs for the whole fork (~350ms on a 2.3M-char doc). + pub fn sync_projection_from_authority(&mut self, cx: &mut Context) { + self.sync_projection_stream(cx, true); + } + + /// Local-write variant: drain + apply patches WITHOUT the remote caret + /// rebase (the caller overrides the caret from its own commit). + pub(super) fn sync_projection_local(&mut self, cx: &mut Context) { + self.sync_projection_stream(cx, false); + } + + fn sync_projection_stream(&mut self, cx: &mut Context, rebase_caret: bool) { + let Some(authority) = self.write_authority.clone() else { + return; + }; + let items = match authority.drain_projection_stream() { + Ok(items) => items, + Err(rejection) => { + self.handle_write_rejection("projection-sync", &rejection, cx); + return; + }, + }; + if items.is_empty() { + return; + } + // §caret-anchor: reposition the caret across the remote edit. FAST path — when + // the caret hasn't moved since its last synced capture, resolve the stored CRDT + // cursors (O(log n)); SLOW fallback — reconstruct the pre-patch body and rebase + // (O(doc) `fork_at`). Either way a remote insert/delete before the caret shifts + // it instead of stranding it at a stale offset (the interleave bug). Captured + // BEFORE the patches mutate `self.document`. `None` ⇒ clamp fallback. + let reanchored_selection = rebase_caret + .then(|| { + self + .caret_anchor + .as_ref() + .filter(|anchor| anchor.selection == self.selection) + .and_then(|anchor| authority.resolve_selection_anchor(&anchor.head_cursor, &anchor.anchor_cursor)) + .map(|(head, anchor)| EditorSelection { anchor, head, ..self.selection.clone() }) + .or_else(|| authority.rebase_selection(&self.selection, &self.document, &self.document.frontier)) + }) + .flatten(); + let mut needs_self_heal = false; + for item in items { + match item { + ProjectionStreamItem::Patches(batch) => { + if batch.new_frontier == self.document.frontier { + continue; // already applied (idempotent redelivery) + } + if batch.base_frontier != self.document.frontier { + // Exactly-once ordered delivery makes this unreachable unless a + // bookkeeping bug slipped in — heal from canonical, loudly. + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Frontier, "stream-order-heal", || { + format!("txn={} base_len={} doc_len={}", batch.transaction_id, batch.base_frontier.len(), self.document.frontier.len()) + }); + needs_self_heal = true; + continue; + } + // In-place apply: `apply_projection_patch_batch` is internally + // transactional (content batches journal + roll back; structural + // batches build a candidate), so the outer whole-projection clone + // this used to make per batch was pure churn. Patches never touch + // the local theme, so the former re-theme pass is dropped too. On + // failure the canonical self-heal below replaces the document + // wholesale anyway. + if apply_projection_patch_batch(&mut self.document, &batch).is_err() { + needs_self_heal = true; + continue; + } + }, + ProjectionStreamItem::Replace(document) => { + let mut document = *document; + // Asset BYTES are UI-cached sideband state; a canonical replace must + // not drop bytes the cache already holds (the canonical store may + // only carry metadata for assets this replica hasn't pulled). + for (id, record) in &self.document.assets.assets { + document.assets.assets.entry(*id).or_insert_with(|| record.clone()); + } + let theme = self.document.theme.clone(); + let mut incoming = projection_with_local_theme(document, &theme); + // §act-nine A9.3: canonical replace = all versions 0; carry + // surviving ids' versions forward (see install_canonical_projection). + carry_forward_paragraph_versions(&self.document, &mut incoming); + self.document = incoming; + }, + } + } + if needs_self_heal + && let Ok(document) = authority.canonical_projection() + { + tracing::error!("projection stream self-heal: installing canonical projection"); + let theme = self.document.theme.clone(); + let mut incoming = projection_with_local_theme(document, &theme); + // §act-nine A9.3: same version carry-forward as every canonical install. + carry_forward_paragraph_versions(&self.document, &mut incoming); + self.document = incoming; + } + self.identity_map.reconcile(&self.document); + // §caret-anchor: prefer the CRDT-reanchored selection (repositioned across the + // remote edit); clamp only as a fallback when it couldn't be resolved. Both are + // then clamped to the freshly-applied document as a final safety net. + let next = reanchored_selection.unwrap_or_else(|| self.selection.clone()); + let head = self.clamp_offset_to_document(next.head); + let anchor = self.clamp_offset_to_document(next.anchor); + if head != self.selection.head || anchor != self.selection.anchor { + self.selection = EditorSelection { anchor, head, ..next }; + self.emit_selection_changed(cx); + } + // Re-arm the fast path against the now-current core for the next remote patch. + self.capture_caret_anchor(); + let generation = self.next_edit_generation; + self.next_edit_generation = self.next_edit_generation.wrapping_add(1); + self.mark_document_changed(generation, false, cx); + } + + /// §caret-anchor: snapshot the current selection's CRDT cursors while the editor + /// and canonical core are in sync, so a subsequent remote patch can reposition + /// the caret via [`crate::LocalWriteAuthority::resolve_selection_anchor`] without + /// the O(doc) `fork_at` of the rebase fallback. Cheap no-op without an authority + /// or when the caret can't be encoded (e.g. a non-body caret) — the fork rebase + /// then covers it. Call at synced moments only (after a sync or a local write). + pub(super) fn capture_caret_anchor(&mut self) { + self.caret_anchor = self + .write_authority + .as_ref() + .and_then(|authority| authority.encode_selection_anchor(&self.selection, &self.document.frontier)) + .map(|(head_cursor, anchor_cursor)| CaretAnchor { + selection: self.selection.clone(), + head_cursor, + anchor_cursor, + }); + } + + /// Spec I-15 rejection handling: nearest-valid caret + loud notice; never an + /// automatic retry. + fn handle_write_rejection(&mut self, class: &'static str, rejection: &WriteRejected, cx: &mut Context) { + tracing::warn!(class, %rejection, "local write intent rejected"); + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Structure, "write-intent-rejected", || { + format!("class={class} rejection={rejection}") + }); + match rejection { + WriteRejected::EmptyIntent => {}, + WriteRejected::CompensatedFailure { .. } | WriteRejected::CompensationFailed { .. } | WriteRejected::GatePoisoned => { + self.note_write_recovery(class, cx); + }, + _ => { + self.clamp_selection_to_document(cx); + self.note_write_recovery(class, cx); + }, + } + } + + fn note_write_recovery(&mut self, class: &'static str, cx: &mut Context) { + self.reconciliation_recoveries = self.reconciliation_recoveries.saturating_add(1); + let _ = class; + cx.notify(); + } + + pub(super) fn clamp_offset_to_document(&self, offset: DocumentOffset) -> DocumentOffset { + let paragraph = offset.paragraph.min(self.document.paragraphs.len().saturating_sub(1)); + let byte = self + .document + .paragraphs + .get(paragraph) + .map(paragraph_text_len) + .unwrap_or_default() + .min(offset.byte); + DocumentOffset { paragraph, byte } + } + + fn clamp_selection_to_document(&mut self, cx: &mut Context) { + let head = self.clamp_offset_to_document(self.selection.head); + let anchor = self.clamp_offset_to_document(self.selection.anchor); + self.selection = EditorSelection::range(anchor, head); + self.emit_selection_changed(cx); + } + + fn set_caret_after_local_write(&mut self, caret: DocumentOffset, cx: &mut Context) { + self.selection = EditorSelection::collapsed(caret); + self.pending_scroll_head_after_layout = true; + self.emit_selection_changed(cx); + } + + // ---- Command-level write helpers ----------------------------------------- + // These are the surface the command/action layer calls. Each builds an + // intent from selection state, delegating selection-replacement rules here + // so every entry point (keystroke, IME, paste, menus) shares one law. + + /// Insert plain text at the caret, replacing the active selection first. + /// One user edit = one undo group: a selection replacement groups its + /// delete+insert. + pub(super) fn write_insert_text_at_caret(&mut self, text: &str, cx: &mut Context) -> bool { + if text.is_empty() { + return false; + } + let grouped = !self.selection.is_caret(); + if grouped { + self.begin_undo_group(); + if !self.write_delete_selection(cx) { + self.end_undo_group(); + return false; + } + } + let caret = self.selection.head; + let Some(at) = self.text_anchor_at(caret) else { + if grouped { + self.end_undo_group(); + } + return false; + }; + let style_override = self.pending_styles.take(); + let committed = self + .write_intent( + LocalIntent::InsertText(InsertTextIntent { + at, + text: text.to_string(), + style_override, + }), + cx, + ) + .is_some(); + if grouped { + self.end_undo_group(); + } + committed + } + + /// Delete the active selection (no-op on a bare caret). + pub(super) fn write_delete_selection(&mut self, cx: &mut Context) -> bool { + if self.selection.is_caret() { + return false; + } + let range = self.selection.normalized(); + self.write_delete_offset_range(range, cx) + } + + pub(super) fn write_delete_offset_range(&mut self, range: std::ops::Range, cx: &mut Context) -> bool { + let (Some(start), Some(end)) = (self.text_anchor_at(range.start), self.text_anchor_at(range.end)) else { + return false; + }; + self + .write_intent(LocalIntent::DeleteRange(DeleteRangeIntent { start, end }), cx) + .is_some() + } + + /// Split the paragraph at the caret (Enter), replacing any selection first. + pub(super) fn write_split_at_caret(&mut self, inherited_style: ParagraphStyle, cx: &mut Context) -> bool { + let grouped = !self.selection.is_caret(); + if grouped { + self.begin_undo_group(); + if !self.write_delete_selection(cx) { + self.end_undo_group(); + return false; + } + } + let caret = self.selection.head; + let Some(at) = self.text_anchor_at(caret) else { + if grouped { + self.end_undo_group(); + } + return false; + }; + let committed = self + .write_intent(LocalIntent::SplitParagraph(SplitParagraphIntent { at, inherited_style }), cx) + .is_some(); + if grouped { + self.end_undo_group(); + } + committed + } + + /// Join the caret's paragraph with its predecessor (Backspace at paragraph + /// start) or successor (Delete at paragraph end). + pub(super) fn write_join_paragraphs(&mut self, first_ix: usize, cx: &mut Context) -> bool { + let (Some(first), Some(second)) = (self.identity_map.paragraph_id(first_ix), self.identity_map.paragraph_id(first_ix + 1)) else { + return false; + }; + // Caret lands at the junction point. + let junction = DocumentOffset { + paragraph: first_ix, + byte: self.document.paragraphs.get(first_ix).map(paragraph_text_len).unwrap_or(0), + }; + let committed = self + .write_intent(LocalIntent::JoinParagraphs(JoinParagraphsIntent { first, second }), cx) + .is_some(); + if committed { + let caret = self.clamp_offset_to_document(junction); + self.set_caret_after_local_write(caret, cx); + } + committed + } + + /// Apply run styles over an offset range (minimal marks over the changed + /// range only — spec §9). + pub(super) fn write_set_marks(&mut self, range: std::ops::Range, styles: RunStyles, cx: &mut Context) -> bool { + let (Some(start), Some(end)) = (self.text_anchor_at(range.start), self.text_anchor_at(range.end)) else { + return false; + }; + self + .write_intent(LocalIntent::SetMarks(SetMarksIntent { start, end, styles }), cx) + .is_some() + } + + pub(super) fn write_set_paragraph_style(&mut self, paragraph_ix: usize, style: ParagraphStyle, cx: &mut Context) -> bool { + let Some(paragraph) = self.identity_map.paragraph_id(paragraph_ix) else { + return false; + }; + self + .write_intent(LocalIntent::SetParagraphStyle(SetParagraphStyleIntent { paragraph, style }), cx) + .is_some() + } + + /// Batched selection-wide restyle: ONE intent → one gate hold, one Loro + /// commit, one undo member, one patch batch (§11 anti-amplification). The + /// per-paragraph loop this replaces cost N full write-path round trips — + /// 64.7s for a select-all restyle on the reference doc — and made undo + /// replay N members. + pub(super) fn write_set_paragraph_styles(&mut self, paragraph_ixs: impl IntoIterator, style: ParagraphStyle, cx: &mut Context) -> bool { + let paragraphs: Vec<_> = paragraph_ixs + .into_iter() + .filter_map(|paragraph_ix| self.identity_map.paragraph_id(paragraph_ix)) + .collect(); + if paragraphs.is_empty() { + return false; + } + self + .write_intent(LocalIntent::SetParagraphStyles(SetParagraphStylesIntent { paragraphs, style }), cx) + .is_some() + } + + pub(super) fn write_insert_object_at_caret(&mut self, block: InputBlock, cx: &mut Context) -> bool { + let caret = self.selection.head; + let Some(at) = self.text_anchor_at(caret) else { + return false; + }; + self + .write_intent(LocalIntent::InsertObject(InsertObjectIntent { at, block }), cx) + .is_some() + } + + pub(super) fn write_insert_rich_fragment_at_caret(&mut self, blocks: Vec, cx: &mut Context) -> bool { + let grouped = !self.selection.is_caret(); + if grouped { + self.begin_undo_group(); + if !self.write_delete_selection(cx) { + self.end_undo_group(); + return false; + } + } + let caret = self.selection.head; + let Some(at) = self.text_anchor_at(caret) else { + if grouped { + self.end_undo_group(); + } + return false; + }; + let committed = self + .write_intent(LocalIntent::InsertRichFragment(InsertRichFragmentIntent { at, blocks }), cx) + .is_some(); + if grouped { + self.end_undo_group(); + } + committed + } + + /// Replace-all (find & replace): every same-paragraph match rides ONE + /// compound intent — one gate hold, one commit, one undo member. + pub(super) fn write_replace_matches(&mut self, matches: Vec, replacement: &str, cx: &mut Context) -> bool { + if matches.is_empty() { + return false; + } + self + .write_intent( + LocalIntent::ReplaceMatches(ReplaceMatchesIntent { + matches, + replacement: replacement.to_string(), + }), + cx, + ) + .is_some() + } + + // ---- Undo grouping (spec §10) --------------------------------------------- + + /// Open an undo group at an input-semantic boundary. Fallible by design + /// (remote imports close groups); the editor simply re-arms next time. + pub(super) fn begin_undo_group(&mut self) { + if let Some(authority) = &self.write_authority + && !matches!(authority.undo_group_start(), Ok(true)) + { + tracing::debug!("undo group did not open (closed by import or already open); re-arming at next boundary"); + } + } + + pub(super) fn end_undo_group(&mut self) { + if let Some(authority) = &self.write_authority { + let _ = authority.undo_group_end(); + } + } + + // ---- Undo / redo (spec §10) ----------------------------------------------- + + pub(super) fn undo_via_authority(&mut self, cx: &mut Context) -> bool { + let Some(authority) = self.write_authority.clone() else { + return false; + }; + match authority.undo() { + Ok(outcome) => self.apply_undo_outcome(outcome, cx), + Err(rejection) => { + self.handle_write_rejection("undo", &rejection, cx); + false + }, + } + } + + pub(super) fn redo_via_authority(&mut self, cx: &mut Context) -> bool { + let Some(authority) = self.write_authority.clone() else { + return false; + }; + match authority.redo() { + Ok(outcome) => self.apply_undo_outcome(outcome, cx), + Err(rejection) => { + self.handle_write_rejection("redo", &rejection, cx); + false + }, + } + } + + fn apply_undo_outcome(&mut self, outcome: LocalUndoOutcome, cx: &mut Context) -> bool { + if !outcome.applied { + return false; // empty undo/redo stack + } + // Field fix: the undo's projection change rides the ordered stream like + // every other change; the outcome's payload is only the applied signal. + // Local variant: the caret is set from `outcome.selection` below. + self.sync_projection_local(cx); + if let Some(selection) = outcome.selection { + let head = self.clamp_offset_to_document(selection.head); + let anchor = self.clamp_offset_to_document(selection.anchor); + self.selection = EditorSelection::range(anchor, head); + self.emit_selection_changed(cx); + } else { + self.clamp_selection_to_document(cx); + } + true + } +} diff --git a/crates/gpui-flowtext/src/rich_text/editor/media.rs b/crates/gpui-flowtext/src/rich_text/editor/media.rs index b7dbd8ae..f1f40f55 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/media.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/media.rs @@ -4,9 +4,9 @@ impl RichTextEditor { let Some(BlockSelection::Image(block_ix)) = self.selected_block else { return; }; - self.edit_selected_image(block_ix, cx, |image| { + let alignment = input_alignment_from_alignment(alignment); + self.write_selected_image_layout(block_ix, cx, |image| { image.alignment = alignment; - image.version = image.version.wrapping_add(1); }); } @@ -14,9 +14,8 @@ impl RichTextEditor { let Some(BlockSelection::Image(block_ix)) = self.selected_block else { return; }; - self.edit_selected_image(block_ix, cx, |image| { - image.sizing = ImageSizing::FitWidth; - image.version = image.version.wrapping_add(1); + self.write_selected_image_layout(block_ix, cx, |image| { + image.sizing = InputImageSizing::FitWidth; }); } @@ -24,9 +23,8 @@ impl RichTextEditor { let Some(BlockSelection::Image(block_ix)) = self.selected_block else { return; }; - self.edit_selected_image(block_ix, cx, |image| { - image.sizing = ImageSizing::Intrinsic; - image.version = image.version.wrapping_add(1); + self.write_selected_image_layout(block_ix, cx, |image| { + image.sizing = InputImageSizing::Intrinsic; }); } @@ -69,12 +67,11 @@ impl RichTextEditor { _ => None, }) .unwrap_or(320); - self.edit_selected_image(block_ix, cx, |image| { - image.sizing = ImageSizing::Fixed { + self.write_selected_image_layout(block_ix, cx, |image| { + image.sizing = InputImageSizing::Fixed { width_px: (current_width + delta_px).clamp(32, 2400) as u32, height_px: None, }; - image.version = image.version.wrapping_add(1); }); } @@ -109,65 +106,49 @@ impl RichTextEditor { cx.notify(); } + /// Track a live image-resize drag. Loro-first (spec §5): the drag never + /// touches THE projection — it only accumulates the target width (rebased + /// into `start_width`/`start_position` each move), and the commit at drag + /// end is one typed `SetImageLayout` intent. fn update_image_resize_drag(&mut self, position: Point, cx: &mut Context) -> bool { - let Some(drag) = self.image_resize_drag.clone() else { + if self.image_resize_drag.is_none() { return false; - }; - let delta: f32 = (position.x - drag.start_position.x).into(); - let delta = delta * drag.handle.horizontal_sign(); - let start_width: f32 = drag.start_width.into(); + } let max_width: f32 = (self.current_layout_width() - self.document.theme.pageless_inset_x * 2.0) .max(px(32.0)) .into(); - let width_px = (start_width + delta) - .clamp(32.0, max_width.max(32.0)) - .round() as u32; - let Some(Block::Image(image)) = Arc::make_mut(&mut self.document.blocks).get_mut(drag.block_ix) else { - self.image_resize_drag = None; - return true; + let Some(drag) = self.image_resize_drag.as_mut() else { + return false; }; - if image.sizing == (ImageSizing::Fixed { width_px, height_px: None }) { - return true; - } - image.sizing = ImageSizing::Fixed { width_px, height_px: None }; - image.version = drag.before.version.wrapping_add(1); - self.invalidate_document_layout_caches(); + let delta: f32 = (position.x - drag.start_position.x).into(); + let delta = delta * drag.handle.horizontal_sign(); + let start_width: f32 = drag.start_width.into(); + let width = (start_width + delta).clamp(32.0, max_width.max(32.0)).round(); + drag.start_width = px(width); + drag.start_position = position; cx.notify(); true } + /// Commit the image resize at drag end through the ONE write path: one + /// typed `SetImageLayout` intent addressed by the image's durable identity. + /// The runtime's patches advance THE projection — no direct write, no + /// history snapshot. A drag that ends at its starting width is a no-op. fn finish_image_resize_drag(&mut self, cx: &mut Context) -> bool { let Some(drag) = self.image_resize_drag.take() else { return false; }; - let Some(Block::Image(after)) = self.document.blocks.get(drag.block_ix).cloned() else { - cx.notify(); - return true; - }; - if after == drag.before { + let final_width: f32 = drag.start_width.into(); + let width_px = final_width.round() as u32; + let initial_width: f32 = self.image_rendered_width(&drag.before).into(); + if width_px == initial_width.round() as u32 { cx.notify(); return true; } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { - block_ix: drag.block_ix, - before: Block::Image(drag.before), - after: Block::Image(after), - }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(drag.block_ix), - }], + self.write_selected_image_layout(drag.block_ix, cx, |image| { + image.sizing = InputImageSizing::Fixed { width_px, height_px: None }; }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); + cx.notify(); true } @@ -192,52 +173,64 @@ impl RichTextEditor { return; }; let alt_text = alt_text.into(); - self.edit_selected_image(block_ix, cx, |image| { - image.alt_text = alt_text; - image.version = image.version.wrapping_add(1); - }); + let Some(Block::Image(image)) = self.document.blocks.get(block_ix) else { + return; + }; + if image.alt_text == alt_text { + return; + } + let Some(image_id) = self.identity_map.block_id(block_ix) else { + tracing::warn!(block_ix, "refusing image alt-text edit: projection block has no durable id, so no intent can address it"); + return; + }; + self.write_intent( + LocalIntent::ReplaceImageAltText(crate::local_intents::ReplaceImageAltTextIntent { + image: image_id, + text: alt_text.to_string(), + }), + cx, + ); } - fn edit_selected_image(&mut self, block_ix: usize, cx: &mut Context, update: impl FnOnce(&mut ImageBlock)) { - let Some(Block::Image(image)) = self.document.blocks.get(block_ix).cloned() else { + /// Route an image layout change through the ONE write path: convert the + /// current projection image to its input shape, apply the change, and + /// commit one typed `SetImageLayout` intent addressed by the image's + /// durable identity. Unchanged layouts never reach the authority. + fn write_selected_image_layout(&mut self, block_ix: usize, cx: &mut Context, update: impl FnOnce(&mut InputImageBlock)) { + let Some(block @ Block::Image(_)) = self.document.blocks.get(block_ix) else { + return; + }; + let InputBlock::Image(current) = input_block_from_block(block) else { return; }; - let mut updated = image.clone(); + let mut updated = current.clone(); update(&mut updated); - if updated == image { + if updated.sizing == current.sizing && updated.alignment == current.alignment { return; } - let before = Block::Image(image); - let after = Block::Image(updated); - if let Some(block) = Arc::make_mut(&mut self.document.blocks).get_mut(block_ix) { - *block = after.clone(); - } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { block_ix, before, after }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(block_ix), - }], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); + let Some(image_id) = self.identity_map.block_id(block_ix) else { + tracing::warn!(block_ix, "refusing image layout edit: projection block has no durable id, so no intent can address it"); + return; + }; + self.write_intent( + LocalIntent::SetImageLayout(crate::local_intents::SetImageLayoutIntent { + image: image_id, + sizing: updated.sizing, + alignment: updated.alignment, + }), + cx, + ); } pub fn insert_equation(&mut self, source: impl Into, cx: &mut Context) { - self.insert_blocks_after_caret( - vec![Block::Equation(EquationBlock { - source: source.into(), - syntax: EquationSyntax::Latex, - display: EquationDisplay::Display, - version: 0, - })], + // Loro-first: the equation enters through an InsertObject intent (identity + // minted by the write path), matching the converted table insertion. + self.write_insert_object_at_caret( + InputBlock::Equation(InputEquationBlock { + source: source.into().to_string(), + syntax: InputEquationSyntax::Latex, + display: InputEquationDisplay::Display, + }), cx, ); } @@ -246,12 +239,13 @@ impl RichTextEditor { self.insert_image_assets(vec![(asset, alt_text.into())], cx); } + /// Loro-first: asset bytes are content-addressed sideband state (same + /// pattern as clipboard adoption and the remote `AssetArrived` path); the + /// image BLOCKS commit as intents through the write authority. fn insert_image_assets(&mut self, assets: Vec<(AssetRecord, SharedString)>, cx: &mut Context) { if assets.is_empty() { return; } - let before_document = self.document.clone(); - let before_selection = self.selection.clone(); let mut blocks = Vec::with_capacity(assets.len()); for (asset, alt_text) in assets { let asset_id = asset.id; @@ -262,11 +256,11 @@ impl RichTextEditor { caption: None, sizing: ImageSizing::FitWidth, alignment: BlockAlignment::Center, + external_url: None, version: 0, })); } - self.insert_blocks_after_caret_without_history(blocks); - self.push_replace_document_history(before_document, before_selection, cx); + self.insert_blocks_after_caret(blocks, cx); } pub fn prompt_insert_image(&mut self, cx: &mut Context) { @@ -360,10 +354,7 @@ impl RichTextEditor { } } let offset = self.hit_test_document_position(position, window, cx); - self.selection = EditorSelection { - anchor: offset, - head: offset, - }; + self.selection = EditorSelection::collapsed(offset); self.clear_block_selection(); self.goal_x = None; self.reset_caret_blink(cx); diff --git a/crates/gpui-flowtext/src/rich_text/editor/mod.rs b/crates/gpui-flowtext/src/rich_text/editor/mod.rs index 646f429d..0312a06a 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/mod.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/mod.rs @@ -1,10 +1,12 @@ use std::{ collections::{VecDeque, hash_map::DefaultHasher}, fs, + future::Future, hash::{Hash, Hasher}, io, ops::Range, path::{Path, PathBuf}, + pin::Pin, rc::Rc, sync::{Arc, Mutex, OnceLock}, time::{Duration, Instant}, @@ -12,15 +14,15 @@ use std::{ use crop::Rope; use gpui::{ - App, Bounds, ClipboardEntry, ClipboardItem, Context, CursorStyle, DragMoveEvent, Entity, ExternalPaths, FocusHandle, Focusable, Image, - ImageFormat, InteractiveElement, IntoElement, KeyDownEvent, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, PathPromptOptions, - Pixels, Point, Render, SharedString, Size, Subscription, Task, Timer, Window, actions, div, img, point, prelude::*, px, relative, rgb, size, + App, Bounds, ClipboardEntry, ClipboardItem, Context, CursorStyle, DragMoveEvent, Entity, EntityInputHandler, ExternalPaths, FocusHandle, + Focusable, Image, ImageFormat, InteractiveElement, IntoElement, KeyDownEvent, MouseButton, MouseDownEvent, MouseMoveEvent, MouseUpEvent, + PathPromptOptions, Pixels, Point, Render, SharedString, Size, Subscription, Task, Timer, UTF16Selection, Window, actions, div, img, point, + prelude::*, px, relative, rgb, size, }; use gpui_component::ActiveTheme as _; use gpui_component::scroll::{Scrollbar, ScrollbarHandle, ScrollbarShow}; use gpui_component::{VirtualListScrollHandle, v_virtual_list}; use rustc_hash::{FxHashMap, FxHashSet}; -use unicode_segmentation::UnicodeSegmentation; use super::*; @@ -28,7 +30,9 @@ const DISABLE_SCROLL_LIMITING_FUNCTIONS: bool = true; // cfg!(target_os = "linux const SCROLL_FOREGROUND_OVERSCAN_PX: f32 = 384.0; const SCROLL_FOREGROUND_MATERIALIZE_BUDGET_MS: u64 = 8; const SCROLL_FOREGROUND_MAX_CHUNK_LINES: usize = 96; -const TYPING_PREFETCH_SUPPRESSION_WINDOW: Duration = Duration::from_millis(150); +const TYPING_PREFETCH_SUPPRESSION_WINDOW: Duration = Duration::from_millis(500); +const RECOVERY_WRITE_DEBOUNCE: Duration = Duration::from_millis(750); +const RECOVERY_TYPING_IDLE_WINDOW: Duration = Duration::from_secs(2); const OFFSCREEN_LAYOUT_CACHE_OVERSCAN_PARAGRAPHS: usize = 24; const OFFSCREEN_PREP_CACHE_OVERSCAN_PARAGRAPHS: usize = 160; @@ -156,17 +160,96 @@ impl Render for ToolkitTextDrag { } } -#[derive(Clone, Debug, Eq, PartialEq)] +/// Editor selection (§16). +/// +/// `anchor`/`head` remain plain [`DocumentOffset`]s so all existing +/// position/range math keeps working unchanged. Each endpoint additionally +/// carries an explicit [`SelectionAffinity`] and [`VisualGravity`] describing +/// the user's intent. These are a genuine, stored part of the selection — the +/// collaboration runtime reads them directly instead of re-deriving a side from +/// selection direction. +/// +/// Construct selections through the helpers below ([`EditorSelection::collapsed`], +/// [`EditorSelection::range`], [`EditorSelection::moved`]) rather than struct +/// literals so the affinity/gravity fields are always populated; all default to +/// neutral. +#[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct EditorSelection { pub anchor: DocumentOffset, pub head: DocumentOffset, + /// Affinity of the fixed (anchor) endpoint. + pub anchor_affinity: SelectionAffinity, + /// Affinity of the moving (head) endpoint. This is the side the caret sits on + /// for a collapsed selection. + pub head_affinity: SelectionAffinity, + /// Visual gravity of the fixed (anchor) endpoint at a soft-wrap seam. + pub anchor_gravity: VisualGravity, + /// Visual gravity of the moving (head) endpoint at a soft-wrap seam. This is + /// the gravity consumed when painting a collapsed caret. + pub head_gravity: VisualGravity, } #[hotpath::measure_all] impl EditorSelection { fn caret() -> Self { - let zero = DocumentOffset::default(); - Self { anchor: zero, head: zero } + Self::default() + } + + /// Collapsed caret at `offset` with neutral affinity/gravity. + #[must_use] + pub fn collapsed(offset: DocumentOffset) -> Self { + Self { + anchor: offset, + head: offset, + ..Self::default() + } + } + + /// Collapsed caret at `offset` carrying an explicit affinity/gravity. Used by + /// boundary navigation (objects, line edges) where the side/gravity matters + /// even though the selection is collapsed. + #[must_use] + pub fn collapsed_with(offset: DocumentOffset, affinity: SelectionAffinity, gravity: VisualGravity) -> Self { + Self { + anchor: offset, + head: offset, + anchor_affinity: affinity, + head_affinity: affinity, + anchor_gravity: gravity, + head_gravity: gravity, + } + } + + /// Range selection from `anchor` to `head` with neutral affinity/gravity on + /// both endpoints. + #[must_use] + pub fn range(anchor: DocumentOffset, head: DocumentOffset) -> Self { + Self { + anchor, + head, + ..Self::default() + } + } + + /// Apply a caret motion: move `head` to `new_head` carrying the supplied + /// affinity/gravity. When `extend` is set the anchor (and its intent) is + /// preserved, producing/continuing a range; otherwise the selection collapses + /// onto the new head and the anchor mirrors the head's intent. This preserves + /// the moving endpoint's affinity across selection extension (§16). + #[must_use] + pub(super) fn moved(&self, new_head: DocumentOffset, head_affinity: SelectionAffinity, head_gravity: VisualGravity, extend: bool) -> Self { + if extend { + Self { + anchor: self.anchor, + head: new_head, + anchor_affinity: self.anchor_affinity, + head_affinity, + anchor_gravity: self.anchor_gravity, + head_gravity, + } + } else { + Self::collapsed_with(new_head, head_affinity, head_gravity) + } } pub(super) fn normalized(&self) -> Range { @@ -176,6 +259,13 @@ impl EditorSelection { pub fn is_caret(&self) -> bool { self.anchor == self.head } + + /// Whether two selections occupy the same anchor/head offsets, ignoring + /// affinity/gravity. Used by directional movement to detect a true positional + /// no-op (preserving the historical offset-based early-return behavior). + pub(super) fn same_positions(&self, other: &Self) -> bool { + self.anchor == other.anchor && self.head == other.head + } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -192,138 +282,8 @@ impl CollaborationRole { } } -struct EditRecord { - before_selection: EditorSelection, - before_generation: u64, - after_selection: EditorSelection, - after_generation: u64, - operations: Vec, - canonical_operations: Vec, -} - -#[derive(Clone, Debug)] -pub(super) enum EditOperation { - InsertText { - paragraph: usize, - byte: usize, - text: String, - styles: RunStyles, - }, - ReplaceParagraphSpan { - before: DocumentSpan, - after: DocumentSpan, - }, - InsertRichFragment { - offset: DocumentOffset, - inserted_end: DocumentOffset, - fragment: RichClipboardFragment, - }, - DeleteBlock { - block_ix: usize, - block: Block, - }, - #[allow(dead_code, reason = "IME state accessor is retained for platform input diagnostics.")] - InsertBlocks { - block_ix: usize, - blocks: Vec, - }, - ReplaceBlock { - block_ix: usize, - before: Block, - after: Block, - }, - ReplaceDocument { - before: Box, - after: Box, - }, - MoveRichText { - source_range: Range, - adjusted_drop: DocumentOffset, - inserted_range: Range, - fragment: RichClipboardFragment, - }, -} - -#[hotpath::measure_all] -impl EditOperation { - pub(super) fn undo(&self, document: &mut Document) { - match self { - Self::InsertText { paragraph, byte, text, .. } => { - delete_range_in_paragraph(document, *paragraph, *byte..*byte + text.len()); - }, - Self::ReplaceParagraphSpan { before, after } => apply_document_span_replacement(document, after, before), - Self::InsertRichFragment { offset, inserted_end, .. } => delete_cross_paragraph_range(document, *offset..*inserted_end), - Self::DeleteBlock { block_ix, block } => { - let insert_ix = (*block_ix).min(document.blocks.len()); - Arc::make_mut(&mut document.blocks).insert(insert_ix, block.clone()); - }, - Self::InsertBlocks { block_ix, blocks } => { - let end = (*block_ix + blocks.len()).min(document.blocks.len()); - Arc::make_mut(&mut document.blocks).drain(*block_ix..end); - }, - Self::ReplaceBlock { block_ix, before, .. } => { - if let Some(block) = Arc::make_mut(&mut document.blocks).get_mut(*block_ix) { - *block = before.clone(); - } - }, - Self::ReplaceDocument { before, .. } => { - *document = before.as_ref().clone(); - }, - Self::MoveRichText { - source_range, - inserted_range, - fragment, - .. - } => { - delete_cross_paragraph_range(document, inserted_range.clone()); - insert_rich_fragment_at(document, source_range.start, fragment); - }, - } - } - - pub(super) fn redo(&self, document: &mut Document) { - match self { - Self::InsertText { - paragraph, - byte, - text, - styles, - } => { - insert_text_at(document, *paragraph, *byte, text, *styles); - }, - Self::ReplaceParagraphSpan { before, after } => apply_document_span_replacement(document, before, after), - Self::InsertRichFragment { offset, fragment, .. } => { - insert_rich_fragment_at(document, *offset, fragment); - }, - Self::DeleteBlock { block_ix, .. } => { - if !matches!(document.blocks.get(*block_ix), Some(Block::Paragraph(_))) { - Arc::make_mut(&mut document.blocks).remove(*block_ix); - } - }, - Self::InsertBlocks { block_ix, blocks } => { - let insert_ix = (*block_ix).min(document.blocks.len()); - Arc::make_mut(&mut document.blocks).splice(insert_ix..insert_ix, blocks.clone()); - }, - Self::ReplaceBlock { block_ix, after, .. } => { - if let Some(block) = Arc::make_mut(&mut document.blocks).get_mut(*block_ix) { - *block = after.clone(); - } - }, - Self::ReplaceDocument { after, .. } => { - *document = after.as_ref().clone(); - }, - Self::MoveRichText { - source_range, - adjusted_drop, - fragment, - .. - } => { - delete_cross_paragraph_range(document, source_range.clone()); - insert_rich_fragment_at(document, *adjusted_drop, fragment); - }, - } - } -} +// Loro-first (spec §10, invariant 11): the editor holds NO content history. +// Undo/redo executes through the write authority's Loro UndoManager. #[derive(Clone, Debug)] pub enum SaveStatus { @@ -397,11 +357,6 @@ fn point_distance_squared(a: Point, b: Point) -> f32 { dx * dx + dy * dy } -#[hotpath::measure] -fn is_single_grapheme_text_insert(text: &str) -> bool { - !text.is_empty() && !text.contains('\n') && !text.contains(SOFT_LINE_BREAK) && text.graphemes(true).take(2).count() == 1 -} - #[hotpath::measure] pub(super) fn adjust_drop_after_source_delete(drop: DocumentOffset, source: Range) -> DocumentOffset { if drop <= source.start { @@ -448,26 +403,53 @@ pub struct RichTextEditorStyleState { #[derive(Clone, Debug, Eq, PartialEq)] pub struct RichTextEditorConfig { pub smart_word_selection: bool, + pub show_own_collaboration_caret_color: bool, } #[hotpath::measure_all] impl Default for RichTextEditorConfig { fn default() -> Self { - Self { smart_word_selection: true } + Self { + smart_word_selection: true, + show_own_collaboration_caret_color: true, + } } } #[derive(Clone, Debug, Eq, PartialEq)] pub struct ExternalCaret { pub offset: DocumentOffset, + pub visual_gravity: VisualGravity, pub color_rgb: u32, } + +/// A remote peer's (non-collapsed) selection range, painted behind the text in +/// that peer's presence color. The head caret is carried separately as an +/// [`ExternalCaret`]; this is only the highlighted span. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExternalSelection { + pub selection: EditorSelection, + pub color_rgb: u32, +} + +// Loro-first: hooks carry NO pending edit batches (nothing is ever pending — +// intents commit synchronously) and return no replacement projection (the +// canonical doc is always current; there is nothing to replay). +pub type NativeSaveHook = Rc) -> Pin>>>>; +pub type NativeExportHook = Rc) -> Pin>>>>; +pub type NativeRecoveryHook = Rc Pin>>>>; + #[derive(Clone)] -struct ParagraphChunkLayoutCacheEntry { +pub(super) struct ParagraphChunkLayoutCacheEntry { key: ParagraphCacheKey, + // §act-nine A9.3: the chunk cache is POSITIONAL (indexed by paragraph_ix), + // and the (style, version) `key` alone can collide across DIFFERENT + // paragraphs after a row shift. The stable id pins the slot to the paragraph + // the chunks were built for — replacing the old global `edit_generation` + // check, which nuked every paragraph's chunks on any edit. + paragraph_id: ParagraphId, width: Pixels, invisibility_mode: bool, - edit_generation: u64, layout_generation: u64, prep: Arc, chunks: Vec, @@ -506,11 +488,6 @@ impl ParagraphPrepSlot { self.normal = Some(prep); } } - - fn clear(&mut self) { - self.normal = None; - self.invisible = None; - } } struct ParagraphShapingCacheEntry { @@ -551,9 +528,17 @@ fn range_within(outer: &Range, inner: &Range) -> bool { #[derive(Clone, Copy, PartialEq)] struct ParagraphEstimateHeightCacheEntry { key: ParagraphCacheKey, + // §perf-heaven T5: the STABLE paragraph identity for this slot. The estimate + // cache is indexed by paragraph_ix, and `key` hashes only (style, version) — + // so after an insert/delete a slot can be reused by a different paragraph + // that coincidentally shares (style, version). Storing the paragraph id makes + // slot reuse a cache MISS instead of a stale-height hit, which lets validity + // drop the global `edit_generation` (that over-invalidated EVERY paragraph's + // estimate on any edit — the O(document) cold estimate cost). `layout_generation` + // still catches theme/zoom changes the per-paragraph key cannot see. + paragraph_id: ParagraphId, width: Pixels, invisibility_mode: bool, - edit_generation: u64, layout_generation: u64, height: Pixels, source_len: usize, @@ -562,7 +547,6 @@ struct ParagraphEstimateHeightCacheEntry { #[derive(Clone)] struct LayoutPrepRequest { width: Pixels, - edit_generation: u64, invisibility_mode: bool, paragraphs: Vec, } @@ -586,22 +570,41 @@ struct LayoutRuntimeMetrics { } #[derive(Clone, Debug)] -pub(crate) struct ItemSizeBenchmarkResult { - pub(crate) elapsed: Duration, - pub(crate) cache_hit: bool, - pub(crate) item_count: usize, - pub(crate) exact_height_count: usize, - pub(crate) total_height: f32, - pub(crate) prep_requested: usize, - pub(crate) prep_completed: usize, - pub(crate) prep_installed: usize, - pub(crate) prep_stale: usize, - pub(crate) prep_batches: usize, - pub(crate) prep_text_bytes: usize, - pub(crate) ui_chunk_builds: usize, - pub(crate) ui_chunk_build_time: Duration, - pub(crate) prefetch_budget_overruns: usize, - pub(crate) scroll_budget_overruns: usize, +pub struct ItemSizeBenchmarkResult { + pub elapsed: Duration, + pub cache_hit: bool, + pub item_count: usize, + pub exact_height_count: usize, + pub total_height: f32, + pub prep_requested: usize, + pub prep_completed: usize, + pub prep_installed: usize, + pub prep_stale: usize, + pub prep_batches: usize, + pub prep_text_bytes: usize, + pub ui_chunk_builds: usize, + pub ui_chunk_build_time: Duration, + pub prefetch_budget_overruns: usize, + pub scroll_budget_overruns: usize, +} + +/// §perf-heaven T5 net: the layout-fidelity oracle for the per-paragraph height +/// ESTIMATE — the heuristic scroll uses for not-yet-laid-out paragraphs, and the +/// value a persisted read model would serialize/reload. Layout heights are NOT +/// covered by the CRDT convergence fuzz or the corpus sweep, so this IS the net: +/// for every paragraph that has a COMPLETE exact layout, it compares the estimate +/// to the exact height. A test asserts the estimate never wildly UNDER-shoots +/// (the dangerous case — scroll jumps up as the real height lands) and stays in a +/// bounded band, so any change to the estimate (or a persisted estimate that +/// diverges from a fresh one) trips loudly. +#[derive(Clone, Debug, Default)] +pub struct EstimateAccuracy { + /// Paragraphs with a complete exact layout that were compared. + pub compared: usize, + /// Worst estimate/exact - 1 over the compared set (estimate too TALL). + pub max_over_ratio: f32, + /// Worst 1 - estimate/exact over the compared set (estimate too SHORT). + pub max_under_ratio: f32, } #[derive(Clone)] @@ -665,7 +668,17 @@ pub(super) enum BlockSelection { Image(usize), Equation(usize), Table(usize), - TableCell { block_ix: usize, row_ix: usize, cell_ix: usize }, + /// A selected table cell (§P2b). `row_ix`/`cell_ix` stay for positional access + /// by the layout/paint/caret readers, while `row_id`/`column_id` carry the + /// durable identity resolved from the id-bearing model at selection time so + /// structural emission and replay address the cell by id, not by a stale index. + TableCell { + block_ix: usize, + row_ix: usize, + cell_ix: usize, + row_id: RowId, + column_id: ColumnId, + }, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -673,6 +686,8 @@ pub(super) struct TableCellCaret { pub(super) block_ix: usize, pub(super) row_ix: usize, pub(super) cell_ix: usize, + pub(super) row_id: RowId, + pub(super) column_id: ColumnId, pub(super) paragraph_block_ix: usize, pub(super) anchor: usize, pub(super) byte: usize, @@ -722,27 +737,31 @@ struct RenderLayoutSnapshot { #[derive(Clone)] enum RenderVirtualItems { - Document(Rc>), + DocumentProjection(Rc>), WithDropPreview(Rc>), } #[derive(Clone)] enum RenderVirtualItem { - Document(VirtualItem), + DocumentProjection(VirtualItem), DropPreview, } impl RenderVirtualItems { fn get(&self, item_ix: usize) -> Option { match self { - Self::Document(items) => items.get(item_ix).cloned().map(RenderVirtualItem::Document), + Self::DocumentProjection(items) => items + .get(item_ix) + .cloned() + .map(RenderVirtualItem::DocumentProjection), Self::WithDropPreview(items) => items.get(item_ix).cloned(), } } } enum RecoveryWriteDecision { - Write { generation: u64, document: Box }, + Write { generation: u64, document: Box }, + WriteRuntime { generation: u64, hook: NativeRecoveryHook }, Rescheduled, Idle, } @@ -846,6 +865,21 @@ impl HeightPrefixIndex { } } +use crate::local_intents::{ + DeleteRangeIntent, FragmentBlock, InsertObjectIntent, InsertRichFragmentIntent, InsertTextIntent, JoinParagraphsIntent, LocalCommit, + LocalIntent, LocalWriteAuthority, LocalWriteOutcome, ProjectionStreamItem, ReplaceMatch, ReplaceMatchesIntent, SetMarksIntent, + SetParagraphStyleIntent, SetParagraphStylesIntent, SplitParagraphIntent, TextAnchor, UndoOutcome as LocalUndoOutcome, WriteRejected, +}; + +/// §caret-anchor: the local caret encoded as CRDT cursors, captured while the +/// editor and canonical core are in sync. `selection` is the caret it was +/// captured for — the fast path is used only while the live caret still equals it. +struct CaretAnchor { + selection: EditorSelection, + head_cursor: Vec, + anchor_cursor: Vec, +} + pub struct RichTextEditor { pub(super) focus_handle: FocusHandle, focus_subscriptions: Vec, @@ -854,8 +888,17 @@ pub struct RichTextEditor { document_path: Option, document_display_name: Option, recovery_path: Option, - pub(super) document: Document, + pub(super) document: DocumentProjection, + /// Loro-first: the injected write authority — the ONE local write path + /// (spec invariant 5). `None` = read-only display surface. + write_authority: Option>, pub(super) selection: EditorSelection, + /// §caret-anchor FAST path: the selection's CRDT cursors, captured at synced + /// moments. When a remote patch arrives while `selection` still equals + /// `anchor.selection`, the caret is repositioned by resolving these cursors + /// (O(log n)) instead of the O(doc) `fork_at` rebase. `None`/stale ⇒ fallback. + caret_anchor: Option, + selection_movement_epoch: u64, config: RichTextEditorConfig, edit_generation: u64, saved_generation: u64, @@ -864,11 +907,13 @@ pub struct RichTextEditor { last_format_export_generation: Option, zoom_percent: f32, save_status: SaveStatus, - undo_stack: Vec, - redo_stack: Vec, identity_map: DocumentIdentityMap, - last_collaboration_edit: Option, + reconciliation_recoveries: u64, + native_save_hook: Option, + native_export_hook: Option, + native_recovery_hook: Option, collaboration_role: Option, + own_collaboration_caret_color_rgb: Option, recovery_write_in_progress: bool, recovery_write_pending: bool, last_recovery_generation: u64, @@ -898,15 +943,36 @@ pub struct RichTextEditor { pub(super) caret_visible: bool, caret_blink_active: bool, last_text_input_at: Option, + ime_marked_range: Option>, external_carets: Vec, + external_selections: Vec, + /// Durable document annotations (currently unresolved comment anchors). + /// Kept separate from peer presence so either layer can refresh without + /// erasing the other. + annotation_selections: Vec, pub(super) search_highlights: Vec>, pub(super) active_search_highlight: Option, pending_typing_prefetch_resume: bool, resume_chunk_prefetch_after_typing: bool, paragraph_chunk_layout_cache: Vec>, - paragraph_prep_cache: Vec, - paragraph_shaping_cache: Vec>, - paragraph_estimate_height_cache: Vec>, + // §perf-heaven T8.12: keyed by STABLE `ParagraphId`, not by `paragraph_ix`. + // A positional `Vec` had to `resize_with` on every structural edit and left + // trailing slots misaligned with the shifted paragraphs; the id-keyed map lets + // an unchanged paragraph keep its slot regardless of position. Validity is + // gated by the content key (style, version) + id + position inside the slot + // (§act-nine A9.3 — no global `edit_generation`), so this is + // correctness-neutral. Bounded in `resize_layout_aux_caches` against leaking + // entries for deleted paragraphs (same policy as the estimate cache below). + paragraph_prep_cache: FxHashMap, + paragraph_shaping_cache: FxHashMap, + // §perf-heaven T7.14: keyed by STABLE `ParagraphId`, not by `paragraph_ix`. + // A positional `Vec` shifted on any mid-document insert/delete, turning every + // trailing paragraph's estimate into a stale-slot MISS → an O(document) tail + // recompute on the total-height pass. An id-keyed map lets a shifted paragraph + // keep its cached estimate, so a structural edit recomputes only the touched + // paragraphs. Bounded against unbounded growth from deletions in + // `resize_layout_aux_caches`. + paragraph_estimate_height_cache: FxHashMap, pending_layout_prep_task: Option>, pending_layout_prep_request: Option, layout_generation: u64, @@ -916,9 +982,18 @@ pub struct RichTextEditor { chunk_prefetch_queue: VecDeque, paragraph_height_cache: Vec>, paragraph_height_cache_revision: u64, + // Convergence backstop for the scroll-materialization render loop. Records the + // (rounded scroll-y, edit_generation) of the last render-path materialization + // pass and how many consecutive passes ran at that unchanged signature. A large + // document whose item-size cache cannot be incrementally patched re-runs a full + // O(doc) rebuild + `cx.notify()` every render; if materialization never + // "sticks" it loops forever and freezes the window. Any real scroll or edit + // changes the signature and resets the count, so this caps ONLY the + // pathological non-converging loop. + scroll_materialize_signature: Option<(i32, u64)>, + scroll_materialize_stall_frames: u32, item_sizes_cache: Option, pending_item_sizes_patch_range: Option>, - layout_invalidation_hint: Option>, suppress_mutation_notify: usize, last_scroll_anchor: Option, scroll_anchor_lock: Option, @@ -944,7 +1019,11 @@ pub struct RichTextEditor { goal_x: Option, } +impl gpui::EventEmitter for RichTextEditor {} + include!("lifecycle.rs"); +include!("local_write_path.rs"); +include!("projection_apply.rs"); include!("object_selection.rs"); include!("style_state.rs"); include!("search_highlights.rs"); @@ -978,7 +1057,6 @@ include!("traits.rs"); include!("platform.rs"); include!("virtual_helpers.rs"); include!("table_helpers.rs"); -include!("block_helpers.rs"); include!("render_blocks.rs"); include!("equation_renderer.rs"); include!("object_assets.rs"); diff --git a/crates/gpui-flowtext/src/rich_text/editor/mouse.rs b/crates/gpui-flowtext/src/rich_text/editor/mouse.rs index d5444fa7..456bb48d 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/mouse.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/mouse.rs @@ -35,20 +35,22 @@ impl RichTextEditor { 2 => SelectionGranularity::Word, _ => SelectionGranularity::Paragraph, }; + let before_selection = self.selection.clone(); + // Mouse placement carries no explicit affinity/gravity (§16): clicks/drags + // produce neutral endpoints. self.selection = match self.drag_granularity { - SelectionGranularity::Character if event.modifiers.shift => EditorSelection { - anchor: self.selection.anchor, - head: offset, - }, - SelectionGranularity::Character => EditorSelection { - anchor: offset, - head: offset, - }, + SelectionGranularity::Character if event.modifiers.shift => EditorSelection::range(self.selection.anchor, offset), + SelectionGranularity::Character => EditorSelection::collapsed(offset), SelectionGranularity::Word => selection_for_word_at(&self.document, offset), SelectionGranularity::Paragraph => selection_for_paragraph_at(&self.document, offset.paragraph), }; + self.fidelity_caret_set_from("on_mouse_down", &before_selection); self.drag_anchor = Some(self.selection.anchor); self.reset_caret_blink(cx); + if self.selection != before_selection { + self.note_explicit_selection_movement(); + self.emit_selection_changed(cx); + } cx.notify(); } @@ -78,6 +80,7 @@ impl RichTextEditor { paragraph: paragraph_ix, byte: paragraph_text_len(paragraph), }, + VisualGravity::Neutral, origin, ) else { return false; @@ -120,7 +123,7 @@ impl RichTextEditor { return; } if event.dragging() - && let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block + && let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block && let Some(( BlockSelection::TableCell { row_ix: hit_row, @@ -150,7 +153,12 @@ impl RichTextEditor { source_range: source_range.clone(), fragment: selected_rich_fragment(&self.document, source_range), }); + let before_selection = self.selection.clone(); self.selection = pending_drag.source_selection; + self.fidelity_caret_set_from("on_mouse_move/begin-text-drag", &before_selection); + if self.selection != before_selection { + self.emit_selection_changed(cx); + } self.pending_text_drag = None; } if self.active_text_drag.is_some() { @@ -158,11 +166,15 @@ impl RichTextEditor { self.autoscroll_for_drag(event.position); self.ensure_drag_autoscroll_task(cx); let drop = self.hit_test_document_position(event.position, window, cx); - let selection = EditorSelection { anchor: drop, head: drop }; + let selection = EditorSelection::collapsed(drop); if self.selection != selection { + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("on_mouse_move/text-drag-caret", &fid_before); self.scroll_head_into_view(); self.reset_caret_blink(cx); + self.emit_selection_changed(cx); } cx.notify(); return; @@ -193,9 +205,13 @@ impl RichTextEditor { }, ); if self.selection != selection { + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("on_mouse_move/drag-select", &fid_before); self.scroll_head_into_view(); self.reset_caret_blink(cx); + self.emit_selection_changed(cx); cx.notify(); } else { cx.notify(); @@ -230,9 +246,14 @@ impl RichTextEditor { } else if self.pending_text_drag.take().is_some() { self.clear_drop_preview(); let caret = self.hit_test_document_position(event.position, window, cx); - self.selection = EditorSelection { anchor: caret, head: caret }; + let before_selection = self.selection.clone(); + self.selection = EditorSelection::collapsed(caret); + self.fidelity_caret_set_from("on_mouse_up/cancel-text-drag", &before_selection); self.scroll_head_into_view(); self.reset_caret_blink(cx); + if self.selection != before_selection { + self.emit_selection_changed(cx); + } cx.notify(); } if self.selecting { @@ -248,59 +269,68 @@ impl RichTextEditor { self.clear_drop_preview(); } + /// Drag-drop text move, Loro-first (spec §5): ONE undo group of two typed + /// intents — delete the source range, then insert the dragged fragment at + /// the drop position. The fragment content was captured at drag start (in + /// `ActiveTextDrag`), BEFORE the delete; after the delete commits the + /// projection has changed, so the drop caret is recomputed in post-delete + /// coordinates and resolved to a `TextAnchor` through the reconciled + /// identity map inside the write helpers. No direct projection mutation, + /// no local history record — undo is the authority's grouped undo. fn move_rich_text_fragment(&mut self, drag: ActiveTextDrag, drop: DocumentOffset, cx: &mut Context) { if offset_in_range(drop, drag.source_range.clone()) { self.clear_drop_preview(); - self.selection = EditorSelection { - anchor: drag.source_range.start, - head: drag.source_range.end, - }; + let before_selection = self.selection.clone(); + self.selection = EditorSelection::range(drag.source_range.start, drag.source_range.end); + self.fidelity_caret_set_from("move_rich_text_fragment/drop-on-source", &before_selection); + if self.selection != before_selection { + self.emit_selection_changed(cx); + } cx.notify(); return; } - let before_selection = EditorSelection { - anchor: drag.source_range.start, - head: drag.source_range.end, - }; - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); + if drag.fragment.paragraphs.is_empty() { + self.clear_drop_preview(); + return; + } let source_range = drag.source_range.clone(); + // Drop position expressed in post-delete projection coordinates. let adjusted_drop = adjust_drop_after_source_delete(drop, source_range.clone()); - self.selection = before_selection.clone(); - self.delete_selection_internal(); - let inserted_start = adjusted_drop; - let inserted_end = insert_rich_fragment_at(&mut self.document, inserted_start, &drag.fragment); - self.selection = EditorSelection { - anchor: inserted_end, - head: inserted_end, - }; - self.undo_stack.push(EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::MoveRichText { - source_range, - adjusted_drop, - inserted_range: inserted_start..inserted_end, - fragment: drag.fragment, - }], - canonical_operations: vec![CanonicalOperation::ReplaceParagraphSpan { - start_paragraph: self.identity_map.paragraph_id(inserted_start.paragraph), - before: capture_document_span( - &self.document, - inserted_start.paragraph..(inserted_start.paragraph + 1).min(self.document.paragraphs.len()), - ), - after: capture_document_span( - &self.document, - inserted_start.paragraph..(inserted_end.paragraph + 1).min(self.document.paragraphs.len()), - ), - }], - }); - self.redo_stack.clear(); - self.after_text_mutation(cx); - self.mark_document_changed(after_generation, cx); + self.begin_undo_group(); + if !self.write_delete_offset_range(source_range.clone(), cx) { + self.end_undo_group(); + self.clear_drop_preview(); + return; + } + // The delete committed; the projection (and identity map) advanced. Land + // the caret at the recomputed drop position so the insert helpers anchor + // against the post-delete projection — never the stale pre-delete offset. + let caret = self.clamp_offset_to_document(adjusted_drop); + let fid_before = self.fidelity_caret_before(); + self.selection = EditorSelection::collapsed(caret); + self.fidelity_caret_set("move_rich_text_fragment/drop-caret", &fid_before); + self.emit_selection_changed(cx); + // Plain single-paragraph unstyled content moves as a plain text insert; + // anything styled or multi-paragraph moves as a rich fragment intent. + let plain_text = (drag.fragment.paragraphs.len() == 1) + .then(|| &drag.fragment.paragraphs[0]) + .filter(|paragraph| paragraph.runs.iter().all(|run| run.styles == RunStyles::default())) + .map(|paragraph| paragraph.runs.iter().map(|run| run.text.as_str()).collect::()); + match plain_text { + Some(text) => { + self.write_insert_text_at_caret(&text, cx); + }, + None => { + let blocks = drag + .fragment + .paragraphs + .into_iter() + .map(FragmentBlock::Paragraph) + .collect::>(); + self.write_insert_rich_fragment_at_caret(blocks, cx); + }, + } + self.end_undo_group(); self.clear_drop_preview(); } @@ -378,7 +408,7 @@ impl RichTextEditor { let Some(bounds) = layout.bounds else { return; }; - let Some(caret) = caret_bounds(&layout, self.selection.head, bounds.origin) else { + let Some(caret) = caret_bounds(&layout, self.selection.head, self.selection.head_gravity, bounds.origin) else { return; }; scroll_rect_into_view(&self.scroll_handle, caret, px(4.0)); @@ -432,7 +462,10 @@ impl RichTextEditor { if let Some(head) = editor.hit_test_cached_position(position) && editor.selection.head != head { + let fid_before = editor.fidelity_caret_before(); editor.selection.head = head; + editor.fidelity_caret_set("autoscroll_drag/extend-head", &fid_before); + editor.emit_selection_changed(cx); } cx.notify(); true diff --git a/crates/gpui-flowtext/src/rich_text/editor/movement_core.rs b/crates/gpui-flowtext/src/rich_text/editor/movement_core.rs index 92ae74f0..3a3efb60 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/movement_core.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/movement_core.rs @@ -1,13 +1,16 @@ #[hotpath::measure_all] impl RichTextEditor { - fn move_to_offset(&mut self, new_head: DocumentOffset, extend: bool, cx: &mut Context) { - let anchor = if extend { self.selection.anchor } else { new_head }; - let selection = EditorSelection { anchor, head: new_head }; - if self.selection == selection { + fn move_to_offset(&mut self, new_head: DocumentOffset, head_affinity: SelectionAffinity, head_gravity: VisualGravity, extend: bool, cx: &mut Context) { + let selection = self.selection.moved(new_head, head_affinity, head_gravity, extend); + if self.selection.same_positions(&selection) { self.goal_x = None; return; } + self.note_explicit_selection_movement(); + let fid_before = self.fidelity_caret_before(); self.selection = selection; + self.fidelity_caret_set("move_to_offset", &fid_before); + self.emit_selection_changed(cx); self.goal_x = None; self.scroll_head_into_view(); self.reset_caret_blink(cx); @@ -39,7 +42,7 @@ impl RichTextEditor { let new_offset = clamp_scroll_offset(&self.scroll_handle, point(old_offset.x, old_offset.y + signed_delta)); self.scroll_handle.set_offset(new_offset); - let Some(caret) = caret_bounds(&layout, head, bounds.origin) else { + let Some(caret) = caret_bounds(&layout, head, self.selection.head_gravity, bounds.origin) else { cx.notify(); return; }; @@ -50,22 +53,22 @@ impl RichTextEditor { let target = self .hit_test_cached_position(point(caret.origin.x, target_y)) .unwrap_or_else(|| layout.hit_test(point(caret.origin.x, target_y))); - self.move_to_offset(target, extend, cx); + self.move_to_offset(target, SelectionAffinity::Neutral, VisualGravity::Neutral, extend, cx); } + // §act-ten A10.12: neither hook invalidates layout caches anymore. The + // caches are CONTENT-keyed (§act-nine A9.3 — style+version keys at all four + // sites), so a mutation's own version bumps invalidate exactly the touched + // paragraphs; the unconditional whole-cache nuke these hooks used to run + // (via the dead `layout_invalidation_hint` fallback — its setters were + // deleted in the loro-first cutover) cost delete-word, toolbar formatting + // and collab asset arrival a full O(doc) re-prep each. The one caller that + // genuinely replaces the whole document (`replace_document_projection`) + // performs its own explicit invalidation. pub(super) fn after_text_mutation(&mut self, cx: &mut Context) { self.mark_text_input_interaction(); self.pending_styles = None; self.goal_x = None; - if let Some(range) = self.layout_invalidation_hint.take() { - let end = range - .end - .max(self.selection.head.paragraph.saturating_add(2)) - .min(self.document.paragraphs.len()); - self.invalidate_paragraph_layout_cache_range(range.start..end.max(range.start)); - } else { - self.invalidate_stale_paragraph_layout_caches(); - } self.pending_scroll_head_after_layout = true; self.reset_caret_blink(cx); self.notify_after_mutation(cx); @@ -74,11 +77,6 @@ impl RichTextEditor { pub(super) fn after_formatting_mutation(&mut self, cx: &mut Context) { self.pending_styles = None; self.goal_x = None; - if let Some(range) = self.layout_invalidation_hint.take() { - self.invalidate_paragraph_layout_cache_range(range); - } else { - self.invalidate_stale_paragraph_layout_caches(); - } self.reset_caret_blink(cx); self.notify_after_mutation(cx); } diff --git a/crates/gpui-flowtext/src/rich_text/editor/object_assets.rs b/crates/gpui-flowtext/src/rich_text/editor/object_assets.rs index d4e4c77a..b932c16d 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/object_assets.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/object_assets.rs @@ -1,5 +1,5 @@ #[hotpath::measure] -fn reserved_object_frame(document: &Document, row_size: Size, selected: bool) -> gpui::Div { +fn reserved_object_frame(document: &DocumentProjection, row_size: Size, selected: bool) -> gpui::Div { let object_height = (row_size.height - document.theme.paragraph_after).max(px(1.0)); let object_width = (row_size.width - document.theme.pageless_inset_x * 2.0).max(px(1.0)); div() @@ -16,15 +16,20 @@ fn reserved_object_frame(document: &Document, row_size: Size, selected: } #[hotpath::measure] -fn image_object_frame(document: &Document, image: &ImageBlock, asset: &AssetRecord, row_size: Size, selected: bool) -> gpui::Div { +fn image_object_frame(document: &DocumentProjection, image: &ImageBlock, asset: &AssetRecord, row_size: Size, selected: bool) -> gpui::Div { let available_width = (row_size.width - document.theme.pageless_inset_x * 2.0).max(px(1.0)); let intrinsic = image_asset_intrinsic_size(asset); - let object_width = match image.sizing { - ImageSizing::Fixed { width_px, .. } => px(width_px as f32).min(available_width), - ImageSizing::FitWidth => available_width, - ImageSizing::Intrinsic => intrinsic - .map(|(width, _)| width.min(available_width)) - .unwrap_or(available_width), + let loading = asset.is_loading_placeholder(); + let object_width = if loading { + px(IMAGE_LOADING_PLACEHOLDER_WIDTH_PX).min(available_width) + } else { + match image.sizing { + ImageSizing::Fixed { width_px, .. } => px(width_px as f32).min(available_width), + ImageSizing::FitWidth => available_width, + ImageSizing::Intrinsic => intrinsic + .map(|(width, _)| width.min(available_width)) + .unwrap_or(available_width), + } }; let object_height = (row_size.height - document.theme.paragraph_after).max(px(1.0)); let left_margin = document.theme.pageless_inset_x @@ -41,13 +46,22 @@ fn image_object_frame(document: &Document, image: &ImageBlock, asset: &AssetReco .mr(document.theme.pageless_inset_x) .mb(document.theme.paragraph_after) .overflow_hidden() - .bg(rgb(0xffffff)) + .bg(if loading { rgb(0xf3f4f6) } else { rgb(0xffffff) }) .border_1() - .border_color(if selected { rgb(0x0969da) } else { rgb(0xffffff) }) + .border_color(if selected { + rgb(0x0969da) + } else if loading { + rgb(0xd0d7de) + } else { + rgb(0xffffff) + }) } #[hotpath::measure] fn image_asset_intrinsic_size(asset: &AssetRecord) -> Option<(Pixels, Pixels)> { + if asset.is_loading_placeholder() { + return None; + } let size = imagesize::blob_size(asset.bytes.as_ref()).ok()?; if size.width == 0 || size.height == 0 { return None; @@ -68,14 +82,13 @@ fn image_asset_from_path(path: &Path) -> Option<(AssetRecord, SharedString)> { .file_name() .map(|name| name.to_string_lossy().to_string()); let alt_text: SharedString = original_name.clone().unwrap_or_default().into(); - let mut hasher = DefaultHasher::new(); - bytes.hash(&mut hasher); + let content_hash = AssetRecord::stable_content_hash(&bytes); Some(( AssetRecord { id: AssetId(uuid::Uuid::new_v4().as_u128()), mime_type: format.mime_type().into(), original_name: original_name.map(Into::into), - content_hash: hasher.finish(), + content_hash, bytes: Arc::new(bytes), }, alt_text, @@ -85,14 +98,13 @@ fn image_asset_from_path(path: &Path) -> Option<(AssetRecord, SharedString)> { #[hotpath::measure] fn image_asset_from_image(image: Image) -> (AssetRecord, SharedString) { let asset_id = AssetId(uuid::Uuid::new_v4().as_u128()); - let mut hasher = DefaultHasher::new(); - image.bytes.hash(&mut hasher); + let content_hash = AssetRecord::stable_content_hash(&image.bytes); ( AssetRecord { id: asset_id, mime_type: image.format.mime_type().into(), original_name: None, - content_hash: hasher.finish(), + content_hash, bytes: Arc::new(image.bytes), }, "Pasted image".into(), diff --git a/crates/gpui-flowtext/src/rich_text/editor/object_selection.rs b/crates/gpui-flowtext/src/rich_text/editor/object_selection.rs index 93d85a77..66b03e61 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/object_selection.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/object_selection.rs @@ -108,7 +108,14 @@ impl RichTextEditor { for (row_ix, row) in table.rows.iter().enumerate() { for (cell_ix, cell) in row.cells.iter().enumerate() { if cell.bounds.contains(&document_point) { - let selection = BlockSelection::TableCell { block_ix, row_ix, cell_ix }; + let (row_id, column_id) = table_cell_ids_at(&self.document, block_ix, row_ix, cell_ix); + let selection = BlockSelection::TableCell { + block_ix, + row_ix, + cell_ix, + row_id, + column_id, + }; let mut fallback = (selection, 0, 0); for block in &cell.blocks { if let LaidOutBlock::Paragraph(paragraph) = block { @@ -189,71 +196,96 @@ impl RichTextEditor { None } + /// Track a live column-resize drag. Loro-first (spec §5): the drag never + /// touches THE projection — it only accumulates the target width, and the + /// commit at drag end is a batch of typed `SetColumnWidth` intents. + /// + /// The accumulated width rides in the slot appended past the per-column + /// `start_widths` (the drag struct carries no dedicated field for it); a + /// drag that never moved has no slot and commits nothing. fn update_table_column_resize_drag(&mut self, position: Point, cx: &mut Context) -> bool { - let Some(drag) = self.table_column_resize_drag.clone() else { + let Some(drag) = self.table_column_resize_drag.as_ref() else { return false; }; - let Some(Block::Table(table)) = Arc::make_mut(&mut self.document.blocks).get_mut(drag.block_ix) else { - self.table_column_resize_drag = None; - return true; - }; + let base_columns = table_column_count(&drag.before).max(1); + let column_ix = drag.column_ix; + let start_width = drag.start_widths.get(column_ix).copied(); let delta: f32 = (position.x - drag.start_position.x).into(); - let column_count = drag - .start_widths - .len() - .max(table_column_count(table)) - .max(1); - while table.column_widths.len() < column_count { - table.column_widths.push(TableColumnWidth::FixedPx(120)); - } - for (ix, width) in drag.start_widths.iter().copied().enumerate() { - if ix < table.column_widths.len() { - table.column_widths[ix] = TableColumnWidth::FixedPx(width); - } - } - let Some(start_width) = drag.start_widths.get(drag.column_ix).copied() else { + let Some(start_width) = start_width else { self.table_column_resize_drag = None; return true; }; - table.column_widths[drag.column_ix] = TableColumnWidth::FixedPx((start_width as f32 + delta).clamp(32.0, 1600.0).round() as u32); - table.version = drag.before.version.wrapping_add(1); - self.invalidate_document_layout_caches(); + let width = (start_width as f32 + delta).clamp(32.0, 1600.0).round() as u32; + if let Some(drag) = self.table_column_resize_drag.as_mut() { + drag.start_widths.truncate(base_columns); + drag.start_widths.push(width); + } cx.notify(); true } + /// Commit the column resize at drag end through the ONE write path: one + /// typed `SetColumnWidth` intent per changed column. Every column pins to + /// its measured fixed width (freezing the table layout, matching the + /// pre-Loro-first behavior), grouped as a single undo unit. The runtime's + /// patches advance THE projection — no direct write, no history snapshot. fn finish_table_column_resize_drag(&mut self, cx: &mut Context) -> bool { let Some(drag) = self.table_column_resize_drag.take() else { return false; }; - let Some(Block::Table(after)) = self.document.blocks.get(drag.block_ix).cloned() else { + let base_columns = table_column_count(&drag.before).max(1); + let Some(final_width) = drag.start_widths.get(base_columns).copied() else { + // The drag never moved: nothing to commit. cx.notify(); return true; }; - if after == drag.before { + let Some(Block::Table(table)) = self.document.blocks.get(drag.block_ix).cloned() else { cx.notify(); return true; + }; + let Some(table_id) = self.semantic_block_id(drag.block_ix) else { + tracing::warn!( + block_ix = drag.block_ix, + "refusing table column resize: projection block has no durable id, so no intent can address it", + ); + cx.notify(); + return true; + }; + let mut targets = drag.start_widths[..base_columns].to_vec(); + if let Some(slot) = targets.get_mut(drag.column_ix) { + *slot = final_width; } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { - block_ix: drag.block_ix, - before: Block::Table(drag.before), - after: Block::Table(after), - }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(drag.block_ix), - }], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); + let mut changed = Vec::new(); + for (column_ix, column) in table.columns.iter().enumerate() { + let Some(width) = targets.get(column_ix).copied() else { + break; + }; + if input_table_column_width_from_table_column_width(&column.width) != InputTableColumnWidth::FixedPx(width) { + changed.push((column.id, width)); + } + } + if changed.is_empty() { + cx.notify(); + return true; + } + let grouped = changed.len() > 1; + if grouped { + self.begin_undo_group(); + } + for (column, width) in changed { + self.write_intent( + LocalIntent::Table(crate::local_intents::TableIntent::SetColumnWidth { + table: table_id, + column, + width: InputTableColumnWidth::FixedPx(width), + }), + cx, + ); + } + if grouped { + self.end_undo_group(); + } + cx.notify(); true } @@ -322,8 +354,10 @@ impl RichTextEditor { } let start_block = self.block_ix_for_paragraph(range.start.paragraph)?; let end_block = self.block_ix_for_paragraph(range.end.paragraph)?; - let has_object = self.document.blocks[start_block.min(end_block)..=start_block.max(end_block)] - .iter() + let has_object = self + .document + .blocks + .range(start_block.min(end_block)..start_block.max(end_block) + 1) .any(|block| !matches!(block, Block::Paragraph(_))); if !has_object { return None; @@ -369,21 +403,6 @@ impl RichTextEditor { }) } - fn selection_crosses_object_blocks(&self, range: Range) -> bool { - if !self.document_has_object_blocks() { - return false; - } - let Some(start_block) = self.block_ix_for_paragraph(range.start.paragraph) else { - return false; - }; - let Some(end_block) = self.block_ix_for_paragraph(range.end.paragraph) else { - return false; - }; - self.document.blocks[start_block.min(end_block)..=start_block.max(end_block)] - .iter() - .any(|block| !matches!(block, Block::Paragraph(_))) - } - pub(super) fn block_is_inside_text_selection(&self, block_ix: usize) -> bool { if self.selected_block.is_some() || self.selection.is_caret() { return false; @@ -401,77 +420,15 @@ impl RichTextEditor { block_ix > start_block.min(end_block) && block_ix < start_block.max(end_block) } - fn object_block_indices_in_text_range(&self, range: Range) -> Vec { - if !self.document_has_object_blocks() { - return Vec::new(); - } - let Some(start_block) = self.block_ix_for_paragraph(range.start.paragraph) else { - return Vec::new(); - }; - let Some(end_block) = self.block_ix_for_paragraph(range.end.paragraph) else { - return Vec::new(); - }; - ((start_block + 1)..end_block) - .filter(|block_ix| { - self - .document - .blocks - .get(*block_ix) - .is_some_and(|block| !matches!(block, Block::Paragraph(_))) - }) - .collect() - } - - fn delete_selection_with_document_snapshot(&mut self, cx: &mut Context) -> bool { - if self.selection.is_caret() { - return false; - } - let range = self.selection.normalized(); - let object_indices = self.object_block_indices_in_text_range(range.clone()); - if object_indices.is_empty() { - return false; - } - let before_document = self.document.clone(); - let before_selection = self.selection.clone(); - { - let blocks = Arc::make_mut(&mut self.document.blocks); - for block_ix in object_indices.iter().copied().rev() { - if block_ix < blocks.len() { - blocks.remove(block_ix); - } - } - } - for block_ix in object_indices.into_iter().rev() { - remove_block_ids(&mut self.document, block_ix..block_ix + 1); - } - self.delete_selection_internal(); - let after_document = self.document.clone(); - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceDocument { - before: Box::new(before_document), - after: Box::new(after_document), - }], - canonical_operations: vec![CanonicalOperation::ReplaceDocument], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); - true - } - + /// Delete the selected object block through the ONE write path: a typed + /// `DeleteBlocks` intent addressed by the block's durable identity. The + /// runtime retires the block canonically and its returned patches advance + /// THE projection — no direct block removal, no snapshot history. fn delete_selected_block(&mut self, cx: &mut Context) -> bool { - let Some(selection) = self.selected_block.take() else { + let Some(selection) = self.selected_block else { return false; }; if matches!(selection, BlockSelection::TableCell { .. }) { - self.selected_block = Some(selection); return false; } let block_ix = match selection { @@ -480,35 +437,27 @@ impl RichTextEditor { | BlockSelection::Table(block_ix) | BlockSelection::TableCell { block_ix, .. } => block_ix, }; - if block_ix >= self.document.blocks.len() { + if !matches!( + self.document.blocks.get(block_ix), + Some(Block::Image(_) | Block::Equation(_) | Block::Table(_)) + ) { return false; } - let blocks = Arc::make_mut(&mut self.document.blocks); - if matches!(blocks.get(block_ix), Some(Block::Paragraph(_))) { + let Some(block_id) = self.identity_map.block_id(block_ix) else { + tracing::warn!(block_ix, "refusing selected-block deletion: projection block has no durable id, so no intent can address it"); return false; + }; + let committed = self + .write_intent( + LocalIntent::DeleteBlocks(crate::local_intents::DeleteBlocksIntent { blocks: vec![block_id] }), + cx, + ) + .is_some(); + if committed { + self.clear_block_selection(); + cx.notify(); } - let block = blocks.remove(block_ix); - remove_block_ids(&mut self.document, block_ix..block_ix + 1); - let before_selection = self.selection.clone(); - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: before_selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::DeleteBlock { block_ix, block }], - canonical_operations: vec![CanonicalOperation::DeleteBlock { - block: self.identity_map.block_id(block_ix).unwrap_or(BlockId(0)), - }], - }); - self.redo_stack.clear(); - self.clear_layout_work_caches(); - self.item_sizes_cache = None; - self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); - self.mark_document_changed(after_generation, cx); - true + committed } pub fn caret_paragraph(&self) -> usize { @@ -544,10 +493,10 @@ impl RichTextEditor { } pub(super) fn drag_source_selection(&self) -> Option { - self.active_text_drag.as_ref().map(|drag| EditorSelection { - anchor: drag.source_range.start, - head: drag.source_range.end, - }) + self + .active_text_drag + .as_ref() + .map(|drag| EditorSelection::range(drag.source_range.start, drag.source_range.end)) } pub(super) fn caret_paint_width(&self) -> Pixels { @@ -558,13 +507,22 @@ impl RichTextEditor { if !self.focus_handle.is_focused(window) { return None; } - let BlockSelection::TableCell { block_ix, row_ix, cell_ix } = self.selected_block? else { + let BlockSelection::TableCell { + block_ix, + row_ix, + cell_ix, + row_id, + column_id, + } = self.selected_block? + else { return None; }; Some(TableCellCaret { block_ix, row_ix, cell_ix, + row_id, + column_id, paragraph_block_ix: self.table_cell_block_ix, anchor: self.table_cell_anchor, byte: self.table_cell_caret, diff --git a/crates/gpui-flowtext/src/rich_text/editor/paste.rs b/crates/gpui-flowtext/src/rich_text/editor/paste.rs index 3be8aca1..35ac082b 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/paste.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/paste.rs @@ -1,71 +1,90 @@ #[hotpath::measure_all] impl RichTextEditor { + /// Plain-text paste, Loro-first (spec §5): a single line is ONE `InsertText` + /// intent; multi-line text converts to ONE `InsertRichFragment` intent (one + /// gate hold, one commit, one undo unit). No direct projection mutation. + /// + /// Returns true when this path owned the paste. With an object block + /// selected the caller's block-selection flows own placement instead. fn insert_plain_text_paste_at_caret(&mut self, text: &str, cx: &mut Context) -> bool { - if !self.selection.is_caret() || self.selected_block.is_some() || text.is_empty() || text.contains('\r') || text.contains('\n') { + if self.selected_block.is_some() { return false; } - let paragraph_style = self.document.paragraphs[self.selection.head.paragraph].style; - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: vec![InputParagraph { - style: paragraph_style, - runs: vec![InputRun { - text: text.to_string(), - styles: self.styles_at_caret(), - }], - }], - blocks: Vec::new(), - assets: Vec::new(), - }; - self.insert_rich_fragment_paste_at_caret(&fragment, cx) + self.write_plain_text_paste(text, cx) } - fn insert_rich_fragment_paste_at_caret(&mut self, fragment: &RichClipboardFragment, cx: &mut Context) -> bool { - if !self.selection.is_caret() - || self.selected_block.is_some() - || !fragment.blocks.is_empty() - || !fragment.assets.is_empty() - || fragment.paragraphs.len() != 1 - { + /// Shared plain-text paste body (also the `insert_plain_text_fragment` + /// compatibility surface, which historically ignored `selected_block`). + /// Selection replacement + undo grouping are the write helpers' law. + pub(super) fn write_plain_text_paste(&mut self, text: &str, cx: &mut Context) -> bool { + let normalized = text.replace("\r\n", "\n").replace('\r', "\n"); + if normalized.is_empty() { return false; } - let Some(paragraph_id) = self - .identity_map - .paragraph_id(self.selection.head.paragraph) - else { - return false; - }; - let paragraph = &fragment.paragraphs[0]; - if paragraph.runs.iter().all(|run| run.text.is_empty()) { + if !normalized.contains('\n') { + // Single line, no structure: plain insert. Style inheritance is Loro's + // job (expand-`After` marks, spec §9) — no restated run styles. + self.write_insert_text_at_caret(&normalized, cx); return true; } + // Multi-line: one rich fragment so the whole paste is one intent and one + // undo unit. Lines keep the caret paragraph's style and the caret's run + // styles (previous paste behavior). + let start = self.selection.normalized().start; + let paragraph_style = self + .document + .paragraphs + .get(start.paragraph) + .map(|paragraph| paragraph.style) + .unwrap_or(ParagraphStyle::Normal); + let styles = self.styles_at_caret(); + let blocks = normalized + .split('\n') + .map(|line| { + FragmentBlock::Paragraph(InputParagraph { + style: paragraph_style, + runs: if line.is_empty() { + Vec::new() + } else { + vec![InputRun { + text: line.to_string(), + styles, + }] + }, + }) + }) + .collect::>(); + self.write_insert_rich_fragment_at_caret(blocks, cx); + true + } - let before_selection = self.selection.clone(); - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - let offset = self.selection.head; - let inserted_end = insert_rich_fragment_at(&mut self.document, offset, fragment); - self.selection = EditorSelection { - anchor: inserted_end, - head: inserted_end, - }; - self.undo_stack.push(EditRecord { - before_selection, - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::InsertRichFragment { - offset, - inserted_end, - fragment: fragment.clone(), - }], - canonical_operations: canonical_insert_text_operations(paragraph_id, offset.byte, paragraph), - }); - self.redo_stack.clear(); - self.layout_invalidation_hint = Some(offset.paragraph..offset.paragraph + 1); - self.after_text_mutation(cx); - self.mark_document_changed_with_reconcile(after_generation, false, cx); + /// Rich-fragment paste (the JSON-metadata clipboard path), Loro-first: the + /// fragment's paragraphs + objects convert to `FragmentBlock`s in order and + /// commit as ONE `InsertRichFragment` intent (a lone object commits as the + /// precise `InsertObject` intent instead). Caret placement comes from the + /// commit's `selection_after`, never from editor arithmetic. + pub fn insert_rich_fragment_paste_at_caret(&mut self, fragment: &RichClipboardFragment, cx: &mut Context) -> bool { + if self.selected_block.is_some() { + return false; + } + self.write_clipboard_fragment_at_caret(fragment, cx) + } + + /// Convert a clipboard fragment into intent vocabulary and commit it through + /// the write authority. Empty fragments are a handled no-op (true) rather + /// than an `EmptyIntent` round-trip through the authority. + pub(super) fn write_clipboard_fragment_at_caret(&mut self, fragment: &RichClipboardFragment, cx: &mut Context) -> bool { + let blocks = fragment_blocks_from_clipboard(fragment); + if clipboard_fragment_is_empty(&blocks) { + return true; + } + self.adopt_clipboard_assets(&fragment.assets); + if let [FragmentBlock::Object(block)] = blocks.as_slice() { + let block = block.clone(); + self.write_object_block_replacing_selection(block, cx); + return true; + } + self.write_insert_rich_fragment_at_caret(blocks, cx); true } @@ -145,9 +164,10 @@ impl RichTextEditor { fn insert_paragraphs_into_selected_table_cell(&mut self, paragraphs: &[InputParagraph], cx: &mut Context) -> bool { let Some(BlockSelection::TableCell { - block_ix: _, + block_ix, row_ix, cell_ix, + .. }) = self.selected_block else { return false; @@ -158,14 +178,7 @@ impl RichTextEditor { let current_paragraph_ix = self.table_cell_block_ix; let caret = self.table_cell_caret; let mut new_caret = None; - self.edit_selected_table(cx, |table| { - let Some(cell) = table - .rows - .get_mut(row_ix) - .and_then(|row| row.cells.get_mut(cell_ix)) - else { - return; - }; + self.edit_table_cell(block_ix, row_ix, cell_ix, cx, |cell| { new_caret = insert_table_cell_paragraphs_at(cell, current_paragraph_ix, caret, paragraphs); }); if let Some((paragraph_ix, byte)) = new_caret { @@ -233,7 +246,7 @@ impl RichTextEditor { } fn selected_table_cell(&self) -> Option<&TableCell> { - let BlockSelection::TableCell { block_ix, row_ix, cell_ix } = self.selected_block? else { + let BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. } = self.selected_block? else { return None; }; let Block::Table(table) = self.document.blocks.get(block_ix)? else { @@ -257,12 +270,11 @@ impl RichTextEditor { } fn clear_selected_table_cell(&mut self, cx: &mut Context) -> bool { - let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block else { + let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block else { return false; }; self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { paragraph.text.clear(); - paragraph.paragraph.byte_range = 0..0; paragraph.paragraph.runs.clear(); paragraph.paragraph.version = paragraph.paragraph.version.wrapping_add(1); }); @@ -270,7 +282,7 @@ impl RichTextEditor { } fn move_selected_table_cell(&mut self, forward: bool, cx: &mut Context) -> bool { - let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block else { + let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block else { return false; }; let Some(Block::Table(table)) = self.document.blocks.get(block_ix) else { @@ -292,7 +304,14 @@ impl RichTextEditor { let Some(&(row_ix, cell_ix)) = positions.get(next) else { return false; }; - self.selected_block = Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }); + let (row_id, column_id) = table_cell_ids_at(&self.document, block_ix, row_ix, cell_ix); + self.selected_block = Some(BlockSelection::TableCell { + block_ix, + row_ix, + cell_ix, + row_id, + column_id, + }); self.table_cell_block_ix = 0; self.table_cell_caret = self .selected_table_cell_text() @@ -303,3 +322,37 @@ impl RichTextEditor { } } + +/// Order-preserving conversion of a clipboard fragment into the intent +/// vocabulary: block-shaped fragments already carry paragraphs and objects in +/// document order; paragraph-only fragments are just paragraphs. +fn fragment_blocks_from_clipboard(fragment: &RichClipboardFragment) -> Vec { + if fragment.blocks.is_empty() { + fragment + .paragraphs + .iter() + .cloned() + .map(FragmentBlock::Paragraph) + .collect() + } else { + fragment + .blocks + .iter() + .map(|block| match block { + InputBlock::Paragraph(paragraph) => FragmentBlock::Paragraph(paragraph.clone()), + block => FragmentBlock::Object(block.clone()), + }) + .collect() + } +} + +/// True when the fragment would insert nothing: no blocks, or exactly one +/// paragraph with no text. Multiple empty paragraphs still insert paragraph +/// boundaries and are NOT empty. +fn clipboard_fragment_is_empty(blocks: &[FragmentBlock]) -> bool { + match blocks { + [] => true, + [FragmentBlock::Paragraph(paragraph)] => paragraph.runs.iter().all(|run| run.text.is_empty()), + _ => false, + } +} diff --git a/crates/gpui-flowtext/src/rich_text/editor/platform.rs b/crates/gpui-flowtext/src/rich_text/editor/platform.rs index e83c2827..f06f663d 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/platform.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/platform.rs @@ -19,3 +19,215 @@ fn windows_apply_capslock(text: &str) -> String { } } +#[hotpath::measure_all] +impl RichTextEditor { + pub fn ime_composition_active(&self) -> bool { + self.ime_marked_range.is_some() + } + + fn selection_utf16_range(&self) -> Range { + let text = full_document_text(&self.document); + let anchor = byte_to_utf16( + &text, + global_byte(&self.document, clamp_offset_to_document(&self.document, self.selection.anchor)), + ); + let head = byte_to_utf16( + &text, + global_byte(&self.document, clamp_offset_to_document(&self.document, self.selection.head)), + ); + anchor.min(head)..anchor.max(head) + } + + fn utf16_range_to_document_offsets(&self, range: Range) -> Range { + let text = full_document_text(&self.document); + let start = utf16_to_byte(&text, range.start); + let end = utf16_to_byte(&text, range.end).max(start); + global_to_document_offset(&self.document, start)..global_to_document_offset(&self.document, end) + } + + /// Loro-first (spec §5): one platform text replacement = typed intents + /// only. The UTF-16 range collapses into the selection, then the shared + /// caret-insert law (`write_insert_text_at_caret`) performs the replacement + /// — a non-caret selection groups its delete+insert as ONE undo group — and + /// a bare deletion is one `DeleteRange` intent. Returns whether a document + /// mutation committed. + fn replace_text_for_utf16_range(&mut self, range: Range, text: &str, cx: &mut Context) -> bool { + if self.insert_text_into_selected_object_text(text, cx) { + return true; + } + let offsets = self.utf16_range_to_document_offsets(range); + let selection = EditorSelection::range(offsets.start, offsets.end); + let before_selection = self.selection.clone(); + self.selection = selection; + if text.is_empty() && self.selection.is_caret() { + if self.selection != before_selection { + self.emit_selection_changed(cx); + self.reset_caret_blink(cx); + cx.notify(); + } + return false; + } + let committed = if !self.can_write_collaboration() { + cx.notify(); + false + } else if text.is_empty() { + self.write_delete_selection(cx) + } else { + self.write_insert_text_at_caret(text, cx) + }; + if self.selection != before_selection { + self.emit_selection_changed(cx); + } + committed + } + +} + +#[hotpath::measure_all] +impl EntityInputHandler for RichTextEditor { + fn text_for_range( + &mut self, + range: Range, + adjusted_range: &mut Option>, + _: &mut Window, + _: &mut Context, + ) -> Option { + let text = full_document_text(&self.document); + let start = utf16_to_byte(&text, range.start); + let end = utf16_to_byte(&text, range.end).max(start); + *adjusted_range = Some(byte_range_to_utf16(&text, start..end)); + Some(document_text_slice(&self.document, start..end)) + } + + fn selected_text_range(&mut self, _: bool, _: &mut Window, _: &mut Context) -> Option { + Some(UTF16Selection { + range: self.selection_utf16_range(), + reversed: self.selection.anchor > self.selection.head, + }) + } + + fn marked_text_range(&self, _: &mut Window, _: &mut Context) -> Option> { + self.ime_marked_range.clone() + } + + fn unmark_text(&mut self, _: &mut Window, cx: &mut Context) { + if self.ime_marked_range.take().is_some() { + cx.notify(); + } + } + + fn replace_text_in_range(&mut self, range: Option>, text: &str, _: &mut Window, cx: &mut Context) { + // Composition commit / plain replacement (spec §5): when a marked + // (composition) range exists it is the replacement target — the shared + // intent path deletes it (write_delete_offset_range under the hood) and + // inserts the final text (write_insert_text_at_caret) as ONE undo group. + // With no composition, write_insert_text_at_caret handles selection + // replacement itself. Never a stale visual offset: anchors resolve + // against the current projection inside the write helpers. + let range = range + .or_else(|| self.ime_marked_range.clone()) + .unwrap_or_else(|| self.selection_utf16_range()); + self.ime_marked_range = None; + let _ = self.replace_text_for_utf16_range(range, text, cx); + } + + fn replace_and_mark_text_in_range( + &mut self, + range: Option>, + new_text: &str, + new_selected_range: Option>, + _: &mut Window, + cx: &mut Context, + ) { + // Composition preview (spec §5, minimal conversion): every marked-text + // update commits through typed intents as ONE undo-grouped pair — delete + // the previous marked range, insert the new marked text — via the shared + // caret-insert law; there is no direct projection mutation. The §5 end + // state is a full overlay-only composition (marked text NEVER in Loro + // until composition commit, anchored by a composition-start Loro cursor); + // until then `ime_marked_range` is render/IME-query bookkeeping only and + // is never used as mutation authority. + let range = range + .or_else(|| self.ime_marked_range.clone()) + .unwrap_or_else(|| self.selection_utf16_range()); + let mark_start = range.start; + let committed = self.replace_text_for_utf16_range(range, new_text, cx); + if new_text.is_empty() || !committed { + // Nothing (or nothing new) is marked in the document — drop the marked + // range rather than let it drift from committed state. + self.ime_marked_range = None; + return; + } + let marked_range = mark_start..mark_start + new_text.encode_utf16().count(); + self.ime_marked_range = Some(marked_range.clone()); + let selection_range = new_selected_range + .map(|range| marked_range.start + range.start..marked_range.start + range.end) + .unwrap_or_else(|| marked_range.end..marked_range.end); + let offsets = self.utf16_range_to_document_offsets(selection_range); + let selection = EditorSelection::range(offsets.start, offsets.end); + if self.selection != selection { + self.selection = selection; + self.emit_selection_changed(cx); + self.reset_caret_blink(cx); + cx.notify(); + } + } + + fn bounds_for_range( + &mut self, + range_utf16: Range, + _: Bounds, + _: &mut Window, + _: &mut Context, + ) -> Option> { + let offsets = self.utf16_range_to_document_offsets(range_utf16); + let layout = self.layout_for_offset(offsets.start)?; + let origin = layout.bounds?.origin; + caret_bounds(&layout, offsets.start, VisualGravity::Neutral, origin) + } + + fn character_index_for_point(&mut self, point: Point, window: &mut Window, cx: &mut Context) -> Option { + let offset = self.hit_test_document_position(point, window, cx); + let text = full_document_text(&self.document); + Some(byte_to_utf16( + &text, + global_byte(&self.document, clamp_offset_to_document(&self.document, offset)), + )) + } +} + +#[hotpath::measure] +fn byte_range_to_utf16(text: &str, range: Range) -> Range { + byte_to_utf16(text, range.start)..byte_to_utf16(text, range.end) +} + +#[hotpath::measure] +fn byte_to_utf16(text: &str, byte: usize) -> usize { + let byte = previous_char_boundary(text, byte); + text[..byte].encode_utf16().count() +} + +#[hotpath::measure] +fn utf16_to_byte(text: &str, target: usize) -> usize { + let mut utf16 = 0usize; + for (byte, ch) in text.char_indices() { + if target <= utf16 { + return byte; + } + let next = utf16 + ch.len_utf16(); + if target < next { + return byte; + } + utf16 = next; + } + text.len() +} + +#[hotpath::measure] +fn previous_char_boundary(text: &str, byte: usize) -> usize { + let mut byte = byte.min(text.len()); + while !text.is_char_boundary(byte) { + byte = byte.saturating_sub(1); + } + byte +} diff --git a/crates/gpui-flowtext/src/rich_text/editor/projection_apply.rs b/crates/gpui-flowtext/src/rich_text/editor/projection_apply.rs new file mode 100644 index 00000000..f17312d5 --- /dev/null +++ b/crates/gpui-flowtext/src/rich_text/editor/projection_apply.rs @@ -0,0 +1,1472 @@ +#[hotpath::measure_all] +impl RichTextEditor { + // Asset bytes are transferred out-of-band; remote document content is applied + // by replacing the Loro projection snapshot, not by feeding document patches. + pub fn apply_synced_asset_records(&mut self, asset_records: &[(AssetId, AssetRecord)], cx: &mut Context) { + if asset_records.is_empty() { + return; + } + for (id, record) in asset_records { + self.document.assets.assets.insert(*id, record.clone()); + } + // Asset availability is out-of-band cache state: repaint, but never dirty + // the document or advance the edit generation. (§act-ten A10.12: this used + // to route through `after_formatting_mutation`, whose fallback nuked EVERY + // layout cache per asset batch — image blocks re-estimate via their own + // versioned entries when the asset actually renders.) + cx.notify(); + } + + /// Apply a REMOTE projection patch batch to THE document (Loro-first spec + /// §6): single projection, no committed/visible split, no pending-edit + /// replay, no stable-selection resolve pass — the caret clamps and the next + /// local intent re-resolves by identity. + pub fn apply_remote_patch_batch(&mut self, batch: &ProjectionPatchBatch, cx: &mut Context) -> Result<(), ProjectionApplyError> { + // In-place apply (see `sync_projection_from_authority`): the batch apply + // is internally transactional and never touches the local theme, so the + // per-batch whole-projection clone + re-theme were pure churn. + // §caret-anchor: reposition the caret across the remote batch — fast path + // (stored cursors, O(log n)) with the fork rebase as fallback — captured + // against the PRE-patch state (see sync_projection_from_authority). + let authority = self.write_authority.clone(); + let reanchored = authority.as_ref().and_then(|authority| { + self + .caret_anchor + .as_ref() + .filter(|anchor| anchor.selection == self.selection) + .and_then(|anchor| authority.resolve_selection_anchor(&anchor.head_cursor, &anchor.anchor_cursor)) + .map(|(head, anchor)| EditorSelection { anchor, head, ..self.selection.clone() }) + .or_else(|| authority.rebase_selection(&self.selection, &self.document, &self.document.frontier)) + }); + apply_projection_patch_batch(&mut self.document, batch)?; + self.identity_map.reconcile(&self.document); + let next = reanchored.unwrap_or_else(|| self.selection.clone()); + let head = self.clamp_offset_to_document(next.head); + let anchor = self.clamp_offset_to_document(next.anchor); + if head != self.selection.head || anchor != self.selection.anchor { + self.selection = EditorSelection { anchor, head, ..next }; + self.emit_selection_changed(cx); + } + self.capture_caret_anchor(); + let generation = self.next_edit_generation; + self.next_edit_generation = self.next_edit_generation.wrapping_add(1); + self.mark_document_changed(generation, false, cx); + // Remote patches update the editor in place but never scroll the viewport + // as if the local user typed. + self.pending_scroll_head_after_layout = false; + Ok(()) + } + + pub fn projection_apply_deferred(&self) -> bool { + self.selecting + || self.active_text_drag.is_some() + || self.image_resize_drag.is_some() + || self.table_column_resize_drag.is_some() + || self.ime_composition_active() + } + +} + +#[hotpath::measure] +pub fn apply_projection_patch_batch(document: &mut DocumentProjection, batch: &ProjectionPatchBatch) -> Result<(), ProjectionApplyError> { + if document.frontier != batch.base_frontier { + return Err(ProjectionApplyError::StaleFrontier { + expected: batch.base_frontier.clone(), + actual: document.frontier.clone(), + }); + } + // §perf fast path (measured 2026-07-07): SIMPLE batches — every keystroke, + // remote text batch, mass restyle, and the split/join/cross-delete shapes + // (content patches plus one trailing paragraph-row insert/delete) — apply + // IN PLACE with a rollback journal. The former clone-candidate transaction + // deep-copied the entire projection (`Arc::make_mut` over the whole + // `Vec` + `Vec`, ~3MB on the reference doc) on EVERY + // batch, and structural batches additionally REBUILT the whole document + // model (~8ms per side per Enter on the reference doc). A mid-batch + // failure restores exactly the touched rows in reverse order, so the + // transactional contract (`Err` ⇒ document unchanged) is preserved. + if simple_batch_shape(document, &batch.patches) { + let mut journal = ContentRollbackJournal::default(); + return match apply_content_patches_in_place(document, &batch.patches, &mut journal) { + Ok(outline_dirty) => { + if outline_dirty { + rebuild_document_sections(document); + } + document.frontier.clone_from(&batch.new_frontier); + Ok(()) + }, + Err(error) => { + journal.rollback(document); + Err(error) + }, + }; + } + let mut candidate = hotpath::measure_block!("editor_patch_candidate_clone", document.clone()); + apply_projection_patches_to_document(&mut candidate, &batch.patches, None)?; + candidate.frontier.clone_from(&batch.new_frontier); + *document = candidate; + Ok(()) +} + +/// The SIMPLE batch shape the in-place fast path accepts: any number of +/// content patches (paragraph text/runs/style, assets), plus AT MOST ONE +/// trailing structural patch that inserts or deletes PARAGRAPH rows — the +/// exact shapes local split, join, and cross-paragraph delete synthesize. +/// Everything else (object rows, moves, multi-structural fragments) takes the +/// legacy candidate-rebuild path. +fn simple_batch_shape(document: &DocumentProjection, patches: &[ProjectionPatch]) -> bool { + let is_content = |patch: &ProjectionPatch| { + matches!( + patch, + ProjectionPatch::ParagraphText { .. } + | ProjectionPatch::ParagraphRuns { .. } + | ProjectionPatch::ParagraphStyle { .. } + | ProjectionPatch::AssetArrived { .. } + ) + }; + let Some((last, head)) = patches.split_last() else { + return true; + }; + if !head.iter().all(is_content) { + return false; + } + if is_content(last) { + return true; + } + // Per-row splices refresh the offset index each (O(doc)); a MASS structural + // batch (select-all delete/redo touches thousands of rows) must keep the + // single-pass rebuild instead — the splice path measured O(rows × doc) + // there (24s for a redo of a select-all delete). + const STRUCTURAL_SPLICE_MAX_ROWS: usize = 8; + // The canonical materializer's coalescing/fabrication laws are LOCAL to + // object adjacency (interstitial rows, empties next to placeholders, + // trailing fabricated rows). A plain row splice matches the law only when + // no object row sits in the touched window — object-adjacent shapes keep + // the legacy full-derivation path (the object fuzz caught a trailing-row + // fabrication divergence without this gate). + let object_free_window = |center: usize| { + let start = center.saturating_sub(2); + let end = (center + 2).min(document.blocks.len().saturating_sub(1)); + (start..=end).all(|row| matches!(document.blocks.get(row), None | Some(Block::Paragraph(_)))) + }; + match last { + ProjectionPatch::InsertBlocks { before, row_hint, blocks } => { + if blocks.len() > STRUCTURAL_SPLICE_MAX_ROWS + || !blocks + .iter() + .all(|block| block.paragraph_id.is_some() && matches!(block.block, InputBlock::Paragraph(_))) + { + return false; + } + let Ok(row) = anchor_ix_for_document_insert(document, *before, *row_hint) else { + return false; + }; + // The insertion row references CURRENT rows; the ±2 window (clamped to + // the document edges) covers every local coalescing rule, including + // end-of-document object adjacency. + object_free_window(row) + }, + ProjectionPatch::DeleteBlocks { block_ids, row_hint } => { + if block_ids.len() > STRUCTURAL_SPLICE_MAX_ROWS { + return false; + } + // Every target must resolve to a paragraph row NOW with an object-free + // window, and at least one paragraph row must survive the delete. + let mut paragraph_rows = 0usize; + for (offset, block_id) in block_ids.iter().enumerate() { + let Ok(row) = block_ix_for_patch(document, *block_id, row_hint + offset) else { + return false; + }; + if !matches!(document.blocks.get(row), Some(Block::Paragraph(_))) || !object_free_window(row) { + return false; + } + paragraph_rows += 1; + } + document.paragraphs.len() > paragraph_rows + }, + _ => false, + } +} + +/// Rollback journal for the in-place fast path: one entry per touched row in +/// chronological order, restored in reverse on a mid-batch failure. +#[derive(Default)] +struct ContentRollbackJournal { + entries: Vec, +} + +enum JournalEntry { + Paragraph { + paragraph_ix: usize, + old_paragraph: Paragraph, + old_text: Option, + }, + Asset { + id: AssetId, + old: Option, + }, + InsertedRow { + row: usize, + paragraph_at: usize, + }, + DeletedRow { + row: usize, + paragraph_at: usize, + block_id: BlockId, + paragraph_id: ParagraphId, + paragraph: Paragraph, + text: String, + }, +} + +impl ContentRollbackJournal { + fn record_paragraph(&mut self, document: &DocumentProjection, paragraph_ix: usize, text_changes: bool) { + let Some(paragraph) = document.paragraphs.get(paragraph_ix) else { + return; + }; + let old_text = text_changes.then(|| paragraph_text(document, paragraph_ix)); + self.entries.push(JournalEntry::Paragraph { + paragraph_ix, + old_paragraph: paragraph.clone(), + old_text, + }); + } + + fn rollback(self, document: &mut DocumentProjection) { + for entry in self.entries.into_iter().rev() { + match entry { + JournalEntry::Asset { id, old } => { + match old { + Some(record) => document.assets.assets.insert(id, record), + None => document.assets.assets.remove(&id), + }; + }, + JournalEntry::Paragraph { + paragraph_ix, + old_paragraph, + old_text, + } => { + if paragraph_ix >= document.paragraphs.len() { + continue; + } + let current_range = crate::edit_ops::paragraph_byte_range(document, paragraph_ix); + if let Some(old_text) = old_text { + document.text.delete(current_range.clone()); + document.text.insert(current_range.start, &old_text); + } + paragraphs_mut(document).set(paragraph_ix, old_paragraph); + // Refresh the Fenwick index and mirror the restored runs into the + // block copy (byte ranges are derived on demand from the tree, so no + // stored range needs seeding). + update_paragraph_offsets_after_len_change(document, paragraph_ix); + }, + JournalEntry::InsertedRow { row, paragraph_at } => { + delete_paragraph_row_in_place(document, row, paragraph_at); + }, + JournalEntry::DeletedRow { + row, + paragraph_at, + block_id, + paragraph_id, + paragraph, + text, + } => { + insert_paragraph_row_in_place(document, row, paragraph_at, block_id, paragraph_id, paragraph, &text); + }, + } + } + } +} + +/// Splice ONE paragraph row into the projection in place: rope text, the +/// paragraph/block vectors, both id vectors, then a full offset-index + +/// range refresh (O(doc) writes, microseconds — versus the ~8ms whole-model +/// rebuild this replaces). +fn insert_paragraph_row_in_place( + document: &mut DocumentProjection, + row: usize, + paragraph_at: usize, + block_id: BlockId, + paragraph_id: ParagraphId, + paragraph: Paragraph, + text: &str, +) { + if paragraph_at < document.paragraphs.len() { + let start = document.blocks.paragraph_start(paragraph_at); + document.text.insert(start, text); + document.text.insert(start + text.len(), "\n"); + } else { + let end = document.text.byte_len(); + document.text.insert(end, "\n"); + document.text.insert(end + 1, text); + } + paragraphs_mut(document).insert(paragraph_at, paragraph.clone()); + let paragraph_id_at = paragraph_at.min(document.ids.paragraph_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.paragraph_ids).insert(paragraph_id_at, paragraph_id); + document.blocks.insert(row, Block::Paragraph(paragraph)); + let block_id_at = row.min(document.ids.block_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.block_ids).insert(block_id_at, block_id); +} + +/// Splice ONE paragraph row out of the projection in place (inverse of +/// [`insert_paragraph_row_in_place`]). +fn delete_paragraph_row_in_place(document: &mut DocumentProjection, row: usize, paragraph_at: usize) { + if paragraph_at >= document.paragraphs.len() { + return; + } + let range = crate::edit_ops::paragraph_byte_range(document, paragraph_at); + if paragraph_at > 0 { + // Remove the leading separator with the text. + document.text.delete(range.start.saturating_sub(1)..range.end); + } else { + // First paragraph: remove the text plus its trailing separator. + let end = (range.end + 1).min(document.text.byte_len()); + document.text.delete(range.start..end); + } + paragraphs_mut(document).remove(paragraph_at); + if paragraph_at < document.ids.paragraph_ids.len() { + std::sync::Arc::make_mut(&mut document.ids.paragraph_ids).remove(paragraph_at); + } + if row < document.blocks.len() { + document.blocks.remove(row); + } + if row < document.ids.block_ids.len() { + std::sync::Arc::make_mut(&mut document.ids.block_ids).remove(row); + } +} + +/// The content-only arms of [`apply_projection_patches_to_document`], applied +/// directly to the live projection with journaling. Returns whether a style +/// change dirtied the outline. +fn apply_content_patches_in_place( + document: &mut DocumentProjection, + patches: &[ProjectionPatch], + journal: &mut ContentRollbackJournal, +) -> Result { + let mut outline_dirty = false; + for patch in patches { + match patch { + ProjectionPatch::ParagraphText { + block_id, + paragraph_id, + row_hint, + new, + .. + } => { + let paragraph_ix = paragraph_ix_for_patch(document, *block_id, *paragraph_id, *row_hint)?; + journal.record_paragraph(document, paragraph_ix, true); + outline_dirty |= document.paragraphs[paragraph_ix].style != new.style; + replace_paragraph_content(document, paragraph_ix, new); + }, + ProjectionPatch::ParagraphRuns { + block_id, + paragraph_id, + row_hint, + runs, + } => { + let (row, paragraph_ix) = patch_row_and_paragraph_ix(document, *block_id, *paragraph_id, *row_hint)?; + let text = paragraph_text(document, paragraph_ix); + validate_text_runs(&text, runs)?; + journal.record_paragraph(document, paragraph_ix, false); + let paragraph = paragraphs_mut(document) + .get_mut(paragraph_ix) + .ok_or(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + })?; + paragraph.runs.clone_from(runs); + bump_paragraph_version(paragraph); + update_paragraph_block_at(document, paragraph_ix, row); + }, + ProjectionPatch::ParagraphStyle { + block_id, + paragraph_id, + row_hint, + style, + } => { + let (row, paragraph_ix) = patch_row_and_paragraph_ix(document, *block_id, *paragraph_id, *row_hint)?; + journal.record_paragraph(document, paragraph_ix, false); + let paragraph = paragraphs_mut(document) + .get_mut(paragraph_ix) + .ok_or(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + })?; + outline_dirty |= paragraph.style != *style; + paragraph.style = *style; + bump_paragraph_version(paragraph); + update_paragraph_block_at(document, paragraph_ix, row); + }, + ProjectionPatch::AssetArrived { id, record } => { + journal.entries.push(JournalEntry::Asset { + id: *id, + old: document.assets.assets.get(id).cloned(), + }); + document.assets.assets.insert(*id, record.clone()); + }, + ProjectionPatch::InsertBlocks { + before, + row_hint, + blocks: inserted, + } => { + // Gate-verified: every inserted block is a paragraph row with a + // durable paragraph id (the local split shape). + let row = anchor_ix_for_document_insert(document, *before, *row_hint)?; + for (offset, structural) in inserted.iter().enumerate() { + let (InputBlock::Paragraph(input), Some(paragraph_id)) = (&structural.block, structural.paragraph_id) else { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "non-paragraph insert reached the in-place fast path", + )); + }; + let row = row + offset; + let paragraph_at = paragraph_rows_before(document, row); + let text = input_paragraph_text(input); + let mut paragraph = paragraph_from_input_paragraph(input); + // §act-nine A9.3 version discipline: `paragraph_from_input_paragraph` + // resets `version` to 0. If this paragraph id ALREADY exists in the + // projection, advance its old version instead — the layout caches + // key on (style, version), and reusing an old pair for + // possibly-different content would serve stale layout. Genuinely new + // ids keep 0. + if let Some(previous) = document + .ids + .paragraph_ids + .iter() + .position(|id| *id == paragraph_id) + .and_then(|old_ix| document.paragraphs.get(old_ix)) + { + paragraph.version = previous.version.wrapping_add(1); + } + insert_paragraph_row_in_place(document, row, paragraph_at, structural.block_id, paragraph_id, paragraph, &text); + journal.entries.push(JournalEntry::InsertedRow { row, paragraph_at }); + } + outline_dirty = true; + }, + ProjectionPatch::DeleteBlocks { block_ids, row_hint } => { + // Gate-verified: every target row is a paragraph block and at least + // one paragraph survives (the join / cross-paragraph-delete shape). + // Delete in DESCENDING row order so earlier removals never shift the + // later targets. + let mut rows = Vec::with_capacity(block_ids.len()); + for (offset, block_id) in block_ids.iter().enumerate() { + rows.push(block_ix_for_patch(document, *block_id, row_hint + offset)?); + } + rows.sort_unstable(); + for &row in rows.iter().rev() { + let paragraph_at = paragraph_rows_before(document, row); + let (Some(Block::Paragraph(_)), Some(paragraph)) = (document.blocks.get(row), document.paragraphs.get(paragraph_at)) else { + return Err(ProjectionApplyError::WrongBlockKind { + block_id: document.ids.block_ids.get(row).copied().unwrap_or(BlockId(0)), + expected: "a paragraph", + }); + }; + let entry = JournalEntry::DeletedRow { + row, + paragraph_at, + block_id: document.ids.block_ids.get(row).copied().unwrap_or(BlockId(0)), + paragraph_id: document.ids.paragraph_ids.get(paragraph_at).copied().unwrap_or(ParagraphId(0)), + paragraph: paragraph.clone(), + text: paragraph_text(document, paragraph_at), + }; + delete_paragraph_row_in_place(document, row, paragraph_at); + journal.entries.push(entry); + } + outline_dirty = true; + }, + _ => { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "structural patch reached the content-only fast path", + )); + }, + } + } + Ok(outline_dirty) +} + +/// Count of paragraph rows strictly before `row` — the paragraph index a row +/// splice at `row` maps to. +fn paragraph_rows_before(document: &DocumentProjection, row: usize) -> usize { + if document.blocks.len() == document.paragraphs.len() { + return row.min(document.paragraphs.len()); + } + document + .blocks + .iter() + .take(row) + .filter(|block| matches!(block, Block::Paragraph(_))) + .count() +} + +fn apply_projection_patches_to_document( + document: &mut DocumentProjection, + patches: &[ProjectionPatch], + invalidation: Option<&mut Option>>, +) -> Result<(), ProjectionApplyError> { + let mut structural_blocks: Option> = None; + let mut structural_changed = false; + let mut outline_dirty = false; + let mut paragraph_invalidation: Option> = None; + + for patch in patches { + match patch { + ProjectionPatch::ParagraphText { + block_id, + paragraph_id, + row_hint, + new, + .. + } => { + let paragraph_ix = if let Some(blocks) = structural_blocks.as_mut() { + let row = structural_block_ix_for_patch(blocks, *block_id, *row_hint)?; + let paragraph_ix = paragraph_ix_for_structural_row(blocks, row); + let target = blocks.get_mut(row).ok_or(ProjectionApplyError::MissingBlock { + block_id: *block_id, + row_hint: *row_hint, + })?; + if target.paragraph_id != Some(*paragraph_id) { + return Err(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + }); + } + outline_dirty |= projection_patch_block_paragraph_style(document, target, *block_id)? != new.style; + target.payload = ProjectionPatchBlockPayload::Paragraph(new.clone()); + paragraph_ix + } else { + let paragraph_ix = paragraph_ix_for_patch(document, *block_id, *paragraph_id, *row_hint)?; + outline_dirty |= document.paragraphs[paragraph_ix].style != new.style; + replace_paragraph_content(document, paragraph_ix, new); + paragraph_ix + }; + extend_invalidation(&mut paragraph_invalidation, paragraph_ix..paragraph_ix + 1); + }, + ProjectionPatch::ParagraphStyle { + block_id, + paragraph_id, + row_hint, + style, + } => { + let paragraph_ix = if let Some(blocks) = structural_blocks.as_mut() { + let row = structural_block_ix_for_patch(blocks, *block_id, *row_hint)?; + let paragraph_ix = paragraph_ix_for_structural_row(blocks, row); + let target = blocks.get_mut(row).ok_or(ProjectionApplyError::MissingBlock { + block_id: *block_id, + row_hint: *row_hint, + })?; + if target.paragraph_id != Some(*paragraph_id) { + return Err(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + }); + } + let paragraph = materialize_projection_patch_paragraph(document, target, *block_id)?; + outline_dirty |= paragraph.style != *style; + paragraph.style = *style; + paragraph_ix + } else { + let (row, paragraph_ix) = patch_row_and_paragraph_ix(document, *block_id, *paragraph_id, *row_hint)?; + let paragraph = paragraphs_mut(document) + .get_mut(paragraph_ix) + .ok_or(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + })?; + outline_dirty |= paragraph.style != *style; + paragraph.style = *style; + bump_paragraph_version(paragraph); + update_paragraph_block_at(document, paragraph_ix, row); + paragraph_ix + }; + extend_invalidation(&mut paragraph_invalidation, paragraph_ix..paragraph_ix + 1); + }, + ProjectionPatch::ParagraphRuns { + block_id, + paragraph_id, + row_hint, + runs, + } => { + let paragraph_ix = if let Some(blocks) = structural_blocks.as_mut() { + let row = structural_block_ix_for_patch(blocks, *block_id, *row_hint)?; + let paragraph_ix = paragraph_ix_for_structural_row(blocks, row); + let target = blocks.get_mut(row).ok_or(ProjectionApplyError::MissingBlock { + block_id: *block_id, + row_hint: *row_hint, + })?; + if target.paragraph_id != Some(*paragraph_id) { + return Err(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + }); + } + let paragraph = materialize_projection_patch_paragraph(document, target, *block_id)?; + let text = input_paragraph_text(paragraph); + paragraph.runs = input_runs_from_text_runs(&text, runs)?; + paragraph_ix + } else { + let (row, paragraph_ix) = patch_row_and_paragraph_ix(document, *block_id, *paragraph_id, *row_hint)?; + let text = paragraph_text(document, paragraph_ix); + validate_text_runs(&text, runs)?; + let paragraph = paragraphs_mut(document) + .get_mut(paragraph_ix) + .ok_or(ProjectionApplyError::MissingParagraph { + paragraph_id: *paragraph_id, + block_id: *block_id, + })?; + paragraph.runs.clone_from(runs); + bump_paragraph_version(paragraph); + update_paragraph_block_at(document, paragraph_ix, row); + paragraph_ix + }; + extend_invalidation(&mut paragraph_invalidation, paragraph_ix..paragraph_ix + 1); + }, + ProjectionPatch::ReplaceObjectBlock { + block_id, + row_hint, + block, + } => { + if let Some(blocks) = structural_blocks.as_mut() { + let row = structural_block_ix_for_patch(blocks, *block_id, *row_hint)?; + if blocks[row].paragraph_id.is_some() + || block.paragraph_id.is_some() + || matches!(&block.block, InputBlock::Paragraph(_)) + { + return Err(ProjectionApplyError::WrongBlockKind { + block_id: *block_id, + expected: "an object block", + }); + } + if block.block_id != *block_id { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "object replacement must preserve the target block id", + )); + } + blocks[row] = ProjectionPatchBlock::from_structural_block(block)?; + } else { + replace_object_projection_patch(document, *block_id, *row_hint, block)?; + } + structural_changed = true; + }, + ProjectionPatch::InsertBlocks { + before, + row_hint, + blocks: inserted, + } => { + if structural_blocks.is_none() && inserted.iter().all(projection_structural_block_is_object) { + let row = anchor_ix_for_document_insert(document, *before, *row_hint)?; + for (offset, block) in inserted.iter().enumerate() { + insert_projection_structural_block(document, row + offset, (*block).clone())?; + } + structural_changed = true; + } else { + let blocks = structural_blocks + .get_or_insert_with(|| projection_patch_blocks_from_document(document)); + let row = anchor_ix_for_insert_blocks(blocks, *before, *row_hint)?; + let inserted = inserted + .iter() + .map(ProjectionPatchBlock::from_structural_block) + .collect::, _>>()?; + blocks.splice(row..row, inserted); + structural_changed = true; + outline_dirty = true; + } + }, + ProjectionPatch::DeleteBlocks { block_ids, row_hint } => { + if structural_blocks.is_none() && block_ids.iter().all(|block_id| document_block_is_object(document, *block_id, *row_hint)) { + let mut rows = block_ids + .iter() + .map(|block_id| block_ix_for_patch(document, *block_id, *row_hint)) + .collect::, _>>()?; + rows.sort_unstable(); + for window in rows.windows(2) { + if window[0] == window[1] { + return Err(ProjectionApplyError::DuplicateBlockId(document.ids.block_ids[window[0]])); + } + } + for row in rows.into_iter().rev() { + delete_projection_block_at(document, row)?; + } + structural_changed = true; + } else { + let blocks = structural_blocks + .get_or_insert_with(|| projection_patch_blocks_from_document(document)); + let mut rows = block_ids + .iter() + .map(|block_id| structural_block_ix_for_patch(blocks, *block_id, *row_hint)) + .collect::, _>>()?; + rows.sort_unstable(); + for window in rows.windows(2) { + if window[0] == window[1] { + return Err(ProjectionApplyError::DuplicateBlockId(blocks[window[0]].block_id)); + } + } + for row in rows.into_iter().rev() { + blocks.remove(row); + } + structural_changed = true; + outline_dirty = true; + } + }, + ProjectionPatch::MoveBlock { + block_id, + before, + from_hint, + to_hint, + } => { + if structural_blocks.is_none() && document_block_is_object(document, *block_id, *from_hint) { + let from = block_ix_for_patch(document, *block_id, *from_hint)?; + let to = anchor_ix_for_document_move(document, from, *before, *to_hint)?; + move_projection_block(document, from, to)?; + structural_changed = true; + } else { + let blocks = structural_blocks + .get_or_insert_with(|| projection_patch_blocks_from_document(document)); + let from = structural_block_ix_for_patch(blocks, *block_id, *from_hint)?; + let block = blocks.remove(from); + let to = anchor_ix_for_insert_blocks(blocks, *before, *to_hint)?; + blocks.insert(to, block); + structural_changed = true; + outline_dirty = true; + } + }, + ProjectionPatch::AssetArrived { id, record } => { + document.assets.assets.insert(*id, record.clone()); + }, + } + } + + let rebuilt_structural = structural_blocks.is_some(); + if let Some(blocks) = structural_blocks { + validate_projection_patch_blocks(&blocks)?; + rebuild_document_from_projection_patch_blocks(document, blocks)?; + } else if outline_dirty { + rebuild_document_sections(document); + } + if rebuilt_structural && outline_dirty { + rebuild_document_sections(document); + } + + if let Some(invalidation) = invalidation { + if structural_changed { + extend_invalidation(invalidation, 0..document.paragraphs.len()); + } else if let Some(range) = paragraph_invalidation { + extend_invalidation(invalidation, range); + } + } + Ok(()) +} + +#[derive(Clone)] +struct ProjectionPatchBlock { + block_id: BlockId, + paragraph_id: Option, + payload: ProjectionPatchBlockPayload, +} + +#[derive(Clone)] +enum ProjectionPatchBlockPayload { + ExistingParagraph { paragraph_ix: usize }, + Paragraph(InputParagraph), + ExistingObject { block_ix: usize }, + Object(InputBlock), +} + +impl ProjectionPatchBlock { + fn from_structural_block(block: &ProjectionStructuralBlock) -> Result { + match &block.block { + InputBlock::Paragraph(paragraph) => { + let Some(paragraph_id) = block.paragraph_id else { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "paragraph structural block must carry a paragraph id", + )); + }; + Ok(Self { + block_id: block.block_id, + paragraph_id: Some(paragraph_id), + payload: ProjectionPatchBlockPayload::Paragraph(paragraph.clone()), + }) + }, + InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_) => { + if block.paragraph_id.is_some() { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "object structural block must not carry a paragraph id", + )); + } + Ok(Self { + block_id: block.block_id, + paragraph_id: None, + payload: ProjectionPatchBlockPayload::Object(block.block.clone()), + }) + }, + } + } +} + +fn projection_patch_blocks_from_document(document: &DocumentProjection) -> Vec { + let mut paragraph_ix = 0; + document + .blocks + .iter() + .enumerate() + .map(|(block_ix, block)| { + let block_id = document.ids.block_ids.get(block_ix).copied().unwrap_or_else(new_block_id); + match block { + Block::Paragraph(_) => { + let structural = ProjectionPatchBlock { + block_id, + paragraph_id: Some( + document + .ids + .paragraph_ids + .get(paragraph_ix) + .copied() + .unwrap_or_else(new_paragraph_id), + ), + payload: ProjectionPatchBlockPayload::ExistingParagraph { paragraph_ix }, + }; + paragraph_ix += 1; + structural + }, + Block::Image(_) | Block::Equation(_) | Block::Table(_) => ProjectionPatchBlock { + block_id, + paragraph_id: None, + payload: ProjectionPatchBlockPayload::ExistingObject { block_ix }, + }, + } + }) + .collect() +} + +fn structural_block_ix_for_patch( + blocks: &[ProjectionPatchBlock], + block_id: BlockId, + row_hint: usize, +) -> Result { + if blocks.get(row_hint).is_some_and(|block| block.block_id == block_id) { + return Ok(row_hint); + } + blocks + .iter() + .position(|block| block.block_id == block_id) + .ok_or(ProjectionApplyError::MissingBlock { block_id, row_hint }) +} + +fn paragraph_ix_for_structural_row(blocks: &[ProjectionPatchBlock], row: usize) -> usize { + blocks.iter().take(row).filter(|block| block.paragraph_id.is_some()).count() +} + +fn paragraph_ix_for_block_row(document: &DocumentProjection, row: usize) -> Option { + // Aligned fast path: with no object rows, block row == paragraph index. + if document.blocks.len() == document.paragraphs.len() { + return matches!(document.blocks.get(row), Some(Block::Paragraph(_))).then_some(row); + } + // §perf-heaven T7.13: O(log N) tree rank replaces the object-doc O(row) count. + document.blocks.paragraph_ix_for_block_row(row) +} + +/// Resolve a paragraph patch to `(block_row, paragraph_ix)` in one shot — the +/// block row (used to write the mirrored block copy without a re-scan) is +/// computed anyway while deriving `paragraph_ix`, so callers that need both +/// take it from here instead of re-scanning via `block_ix_for_paragraph` +/// (§perf-heaven T2). +fn patch_row_and_paragraph_ix( + document: &DocumentProjection, + block_id: BlockId, + paragraph_id: ParagraphId, + row_hint: usize, +) -> Result<(usize, usize), ProjectionApplyError> { + let row = block_ix_for_patch(document, block_id, row_hint)?; + let paragraph_ix = paragraph_ix_for_block_row(document, row).ok_or(ProjectionApplyError::WrongBlockKind { + block_id, + expected: "a paragraph", + })?; + if document.ids.paragraph_ids.get(paragraph_ix).copied() != Some(paragraph_id) { + return Err(ProjectionApplyError::MissingParagraph { paragraph_id, block_id }); + } + Ok((row, paragraph_ix)) +} + +fn paragraph_ix_for_patch( + document: &DocumentProjection, + block_id: BlockId, + paragraph_id: ParagraphId, + row_hint: usize, +) -> Result { + patch_row_and_paragraph_ix(document, block_id, paragraph_id, row_hint).map(|(_, paragraph_ix)| paragraph_ix) +} + +fn block_ix_for_patch(document: &DocumentProjection, block_id: BlockId, row_hint: usize) -> Result { + if document.ids.block_ids.get(row_hint).copied() == Some(block_id) { + return Ok(row_hint); + } + document + .ids + .block_ids + .iter() + .position(|id| *id == block_id) + .ok_or(ProjectionApplyError::MissingBlock { block_id, row_hint }) +} + +fn anchor_ix_for_insert_blocks( + blocks: &[ProjectionPatchBlock], + before: Option, + row_hint: usize, +) -> Result { + let Some(before) = before else { + return Ok(blocks.len()); + }; + if blocks.get(row_hint).is_some_and(|block| block.block_id == before) { + return Ok(row_hint); + } + blocks + .iter() + .position(|block| block.block_id == before) + .ok_or(ProjectionApplyError::InvalidAnchor(before)) +} + +fn anchor_ix_for_document_insert( + document: &DocumentProjection, + before: Option, + row_hint: usize, +) -> Result { + let Some(before) = before else { + return Ok(document.blocks.len()); + }; + if document.ids.block_ids.get(row_hint).copied() == Some(before) { + return Ok(row_hint); + } + document + .ids + .block_ids + .iter() + .position(|block_id| *block_id == before) + .ok_or(ProjectionApplyError::InvalidAnchor(before)) +} + +fn anchor_ix_for_document_move( + document: &DocumentProjection, + from: usize, + before: Option, + row_hint: usize, +) -> Result { + let Some(before) = before else { + return Ok(document.blocks.len().saturating_sub(1)); + }; + let adjusted_hint = if row_hint > from { row_hint.saturating_sub(1) } else { row_hint }; + if document + .ids + .block_ids + .get(row_hint) + .copied() + .or_else(|| document.ids.block_ids.get(adjusted_hint).copied()) + == Some(before) + { + return Ok(adjusted_hint); + } + document + .ids + .block_ids + .iter() + .enumerate() + .filter(|(ix, _)| *ix != from) + .map(|(_, id)| *id) + .position(|block_id| block_id == before) + .ok_or(ProjectionApplyError::InvalidAnchor(before)) +} + +fn projection_structural_block_is_object(block: &ProjectionStructuralBlock) -> bool { + block.paragraph_id.is_none() && matches!(&block.block, InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_)) +} + +fn document_block_is_object(document: &DocumentProjection, block_id: BlockId, row_hint: usize) -> bool { + block_ix_for_patch(document, block_id, row_hint) + .ok() + .and_then(|row| document.blocks.get(row)) + .is_some_and(|block| matches!(block, Block::Image(_) | Block::Equation(_) | Block::Table(_))) +} + +fn replace_object_projection_patch( + document: &mut DocumentProjection, + block_id: BlockId, + row_hint: usize, + block: &ProjectionStructuralBlock, +) -> Result<(), ProjectionApplyError> { + if block.block_id != block_id { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "object replacement must preserve the target block id", + )); + } + if !projection_structural_block_is_object(block) { + return Err(ProjectionApplyError::WrongBlockKind { + block_id, + expected: "an object block", + }); + } + let row = block_ix_for_patch(document, block_id, row_hint)?; + if !matches!(document.blocks.get(row), Some(Block::Image(_) | Block::Equation(_) | Block::Table(_))) { + return Err(ProjectionApplyError::WrongBlockKind { + block_id, + expected: "an object block", + }); + } + replace_projection_block(document, row, block_id, block.block.clone()) +} + +fn validate_text_runs(text: &str, runs: &[TextRun]) -> Result<(), ProjectionApplyError> { + let mut byte = 0usize; + for run in runs { + let end = byte + .checked_add(run.len) + .ok_or(ProjectionApplyError::InvalidStructuralPatch("paragraph run lengths overflow"))?; + if end > text.len() || !text.is_char_boundary(byte) || !text.is_char_boundary(end) { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "paragraph run lengths do not align to UTF-8 boundaries", + )); + } + byte = end; + } + if byte != text.len() { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "paragraph runs do not cover the complete paragraph text", + )); + } + Ok(()) +} + +fn input_runs_from_text_runs(text: &str, runs: &[TextRun]) -> Result, ProjectionApplyError> { + validate_text_runs(text, runs)?; + let mut byte = 0usize; + Ok( + runs + .iter() + .map(|run| { + let start = byte; + byte += run.len; + InputRun { + text: text[start..byte].to_string(), + styles: run.styles, + } + }) + .collect(), + ) +} + +fn validate_projection_patch_blocks(blocks: &[ProjectionPatchBlock]) -> Result<(), ProjectionApplyError> { + let mut block_ids = rustc_hash::FxHashSet::default(); + let mut paragraph_ids = rustc_hash::FxHashSet::default(); + for block in blocks { + if !block_ids.insert(block.block_id) { + return Err(ProjectionApplyError::DuplicateBlockId(block.block_id)); + } + if let Some(paragraph_id) = block.paragraph_id + && !paragraph_ids.insert(paragraph_id) + { + return Err(ProjectionApplyError::DuplicateParagraphId(paragraph_id)); + } + } + Ok(()) +} + +#[hotpath::measure] +fn replace_paragraph_content(document: &mut DocumentProjection, paragraph_ix: usize, paragraph: &InputParagraph) { + if paragraph_ix >= document.paragraphs.len() { + return; + } + let text = input_paragraph_text(paragraph); + let byte_range = paragraph_byte_range(document, paragraph_ix); + document.text.delete(byte_range.clone()); + document.text.insert(byte_range.start, &text); + let mut replacement = paragraph_from_input_paragraph(paragraph); + replacement.version = document.paragraphs[paragraph_ix].version.wrapping_add(1); + paragraphs_mut(document).set(paragraph_ix, replacement); + update_paragraph_offsets_after_len_change(document, paragraph_ix); +} + +#[hotpath::measure] +fn input_paragraph_from_document_paragraph(document: &DocumentProjection, paragraph_ix: usize, paragraph: &Paragraph) -> InputParagraph { + let text = paragraph_text(document, paragraph_ix); + let mut byte = 0; + InputParagraph { + style: paragraph.style, + runs: paragraph + .runs + .iter() + .map(|run| { + let start = byte; + let end = (start + run.len).min(text.len()); + byte = end; + InputRun { + text: text.get(start..end).unwrap_or("").to_string(), + styles: run.styles, + } + }) + .collect(), + } +} + +fn projection_patch_block_paragraph_style( + document: &DocumentProjection, + block: &ProjectionPatchBlock, + block_id: BlockId, +) -> Result { + match &block.payload { + ProjectionPatchBlockPayload::ExistingParagraph { paragraph_ix } => document + .paragraphs + .get(*paragraph_ix) + .map(|paragraph| paragraph.style) + .ok_or(ProjectionApplyError::MissingBlock { + block_id, + row_hint: *paragraph_ix, + }), + ProjectionPatchBlockPayload::Paragraph(paragraph) => Ok(paragraph.style), + ProjectionPatchBlockPayload::ExistingObject { .. } | ProjectionPatchBlockPayload::Object(_) => { + Err(ProjectionApplyError::WrongBlockKind { + block_id, + expected: "a paragraph", + }) + }, + } +} + +fn materialize_projection_patch_paragraph<'a>( + document: &DocumentProjection, + block: &'a mut ProjectionPatchBlock, + block_id: BlockId, +) -> Result<&'a mut InputParagraph, ProjectionApplyError> { + match &block.payload { + ProjectionPatchBlockPayload::ExistingParagraph { paragraph_ix } => { + let Some(paragraph) = document.paragraphs.get(*paragraph_ix) else { + return Err(ProjectionApplyError::MissingBlock { + block_id, + row_hint: *paragraph_ix, + }); + }; + block.payload = ProjectionPatchBlockPayload::Paragraph(input_paragraph_from_document_paragraph( + document, + *paragraph_ix, + paragraph, + )); + }, + ProjectionPatchBlockPayload::Paragraph(_) => {}, + ProjectionPatchBlockPayload::ExistingObject { .. } | ProjectionPatchBlockPayload::Object(_) => { + return Err(ProjectionApplyError::WrongBlockKind { + block_id, + expected: "a paragraph", + }); + }, + } + match &mut block.payload { + ProjectionPatchBlockPayload::Paragraph(paragraph) => Ok(paragraph), + ProjectionPatchBlockPayload::ExistingParagraph { .. } + | ProjectionPatchBlockPayload::ExistingObject { .. } + | ProjectionPatchBlockPayload::Object(_) => unreachable!("paragraph payload was just materialized"), + } +} + +#[hotpath::measure] +fn rebuild_document_from_projection_patch_blocks( + document: &mut DocumentProjection, + patch_blocks: Vec, +) -> Result<(), ProjectionApplyError> { + let mut text = String::new(); + let mut paragraphs = Vec::new(); + let mut blocks = Vec::with_capacity(patch_blocks.len().max(1)); + let mut block_ids = Vec::with_capacity(patch_blocks.len().max(1)); + let mut paragraph_ids = Vec::new(); + // §act-nine A9.3 version discipline: rebuilt `Paragraph` payloads come from + // `paragraph_from_input_paragraph` (version 0). A paragraph id that SURVIVES + // the rebuild must ADVANCE its old version — the layout caches key on + // (style, version), so resetting to 0 could reuse a key for different + // content (stale render). Conservative: always advance for surviving ids, + // even if the content happens to be unchanged. Genuinely new ids keep 0. + let mut previous_versions: rustc_hash::FxHashMap = rustc_hash::FxHashMap::default(); + for (old_ix, id) in document.ids.paragraph_ids.iter().enumerate() { + if let Some(paragraph) = document.paragraphs.get(old_ix) { + previous_versions.insert(*id, paragraph.version); + } + } + + for patch_block in patch_blocks { + block_ids.push(patch_block.block_id); + match patch_block.payload { + ProjectionPatchBlockPayload::ExistingParagraph { paragraph_ix } => { + let Some(paragraph) = document.paragraphs.get(paragraph_ix).cloned() else { + return Err(ProjectionApplyError::MissingBlock { + block_id: patch_block.block_id, + row_hint: paragraph_ix, + }); + }; + let Some(paragraph_id) = patch_block.paragraph_id else { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "paragraph structural block must carry a paragraph id", + )); + }; + let paragraph_text = paragraph_text(document, paragraph_ix); + push_rebuilt_paragraph( + &mut text, + &mut paragraphs, + &mut blocks, + &mut paragraph_ids, + paragraph, + paragraph_id, + ¶graph_text, + ); + }, + ProjectionPatchBlockPayload::Paragraph(input) => { + let Some(paragraph_id) = patch_block.paragraph_id else { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "paragraph structural block must carry a paragraph id", + )); + }; + let paragraph_text = input_paragraph_text(&input); + let mut paragraph = paragraph_from_input_paragraph(&input); + if let Some(previous_version) = previous_versions.get(¶graph_id) { + paragraph.version = previous_version.wrapping_add(1); + } + push_rebuilt_paragraph( + &mut text, + &mut paragraphs, + &mut blocks, + &mut paragraph_ids, + paragraph, + paragraph_id, + ¶graph_text, + ); + }, + ProjectionPatchBlockPayload::ExistingObject { block_ix } => { + let Some(block) = document.blocks.get(block_ix).cloned() else { + return Err(ProjectionApplyError::MissingBlock { + block_id: patch_block.block_id, + row_hint: block_ix, + }); + }; + if matches!(block, Block::Paragraph(_)) { + return Err(ProjectionApplyError::WrongBlockKind { + block_id: patch_block.block_id, + expected: "an object block", + }); + } + blocks.push(block); + }, + ProjectionPatchBlockPayload::Object(input) => { + if matches!(&input, InputBlock::Paragraph(_)) { + return Err(ProjectionApplyError::WrongBlockKind { + block_id: patch_block.block_id, + expected: "an object block", + }); + } + blocks.push(block_from_input_block(&input)); + }, + } + } + + if paragraphs.is_empty() { + let paragraph = Paragraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + version: 0, + }; + let paragraph_id = new_paragraph_id(); + let block_id = new_block_id(); + block_ids.push(block_id); + push_rebuilt_paragraph( + &mut text, + &mut paragraphs, + &mut blocks, + &mut paragraph_ids, + paragraph, + paragraph_id, + "", + ); + } + + document.text = Rope::from(text); + document.paragraphs = ParagraphSeq::from_vec(paragraphs); + document.blocks = BlockSeq::from_vec(blocks); + document.ids.block_ids = std::sync::Arc::new(block_ids); + document.ids.paragraph_ids = std::sync::Arc::new(paragraph_ids); + reconcile_document_ids(document); + Ok(()) +} + +#[allow(clippy::too_many_arguments, reason = "rebuilding projection shape threads parallel output vectors")] +fn push_rebuilt_paragraph( + text: &mut String, + paragraphs: &mut Vec, + blocks: &mut Vec, + paragraph_ids: &mut Vec, + paragraph: Paragraph, + paragraph_id: ParagraphId, + paragraph_text: &str, +) { + if !paragraphs.is_empty() { + text.push('\n'); + } + text.push_str(paragraph_text); + paragraph_ids.push(paragraph_id); + paragraphs.push(paragraph.clone()); + blocks.push(Block::Paragraph(paragraph)); +} + +fn insert_projection_structural_block( + document: &mut DocumentProjection, + row: usize, + block: ProjectionStructuralBlock, +) -> Result<(), ProjectionApplyError> { + if matches!(&block.block, InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_)) { + if block.paragraph_id.is_some() { + return Err(ProjectionApplyError::InvalidStructuralPatch( + "object structural block must not carry a paragraph id", + )); + } + if document.ids.block_ids.contains(&block.block_id) { + return Err(ProjectionApplyError::DuplicateBlockId(block.block_id)); + } + let row = row.min(document.blocks.len()); + document.blocks.insert(row, block_from_input_block(&block.block)); + let insert_at = row.min(document.ids.block_ids.len()); + std::sync::Arc::make_mut(&mut document.ids.block_ids).insert(insert_at, block.block_id); + reconcile_document_ids(document); + return Ok(()); + } + + let mut blocks = projection_patch_blocks_from_document(document); + let row = row.min(blocks.len()); + blocks.insert(row, ProjectionPatchBlock::from_structural_block(&block)?); + validate_projection_patch_blocks(&blocks)?; + rebuild_document_from_projection_patch_blocks(document, blocks)?; + rebuild_document_sections(document); + Ok(()) +} + +fn delete_projection_block_at(document: &mut DocumentProjection, row: usize) -> Result<(), ProjectionApplyError> { + let missing_block_id = document.ids.block_ids.get(row).copied().unwrap_or(BlockId(0)); + let Some(block) = document.blocks.get(row) else { + return Err(ProjectionApplyError::MissingBlock { + block_id: missing_block_id, + row_hint: row, + }); + }; + if matches!(block, Block::Image(_) | Block::Equation(_) | Block::Table(_)) { + document.blocks.remove(row); + remove_block_ids(document, row..row + 1); + reconcile_document_ids(document); + return Ok(()); + } + + let mut blocks = projection_patch_blocks_from_document(document); + if row >= blocks.len() { + return Err(ProjectionApplyError::MissingBlock { + block_id: missing_block_id, + row_hint: row, + }); + } + blocks.remove(row); + validate_projection_patch_blocks(&blocks)?; + rebuild_document_from_projection_patch_blocks(document, blocks)?; + rebuild_document_sections(document); + Ok(()) +} + +fn move_projection_block(document: &mut DocumentProjection, from: usize, to: usize) -> Result<(), ProjectionApplyError> { + let missing_block_id = document.ids.block_ids.get(from).copied().unwrap_or(BlockId(0)); + let Some(block) = document.blocks.get(from) else { + return Err(ProjectionApplyError::MissingBlock { + block_id: missing_block_id, + row_hint: from, + }); + }; + if matches!(block, Block::Image(_) | Block::Equation(_) | Block::Table(_)) { + let mut blocks = document.blocks.make_mut(); + let block = blocks.remove(from); + let to = to.min(blocks.len()); + blocks.insert(to, block); + drop(blocks); + if from < document.ids.block_ids.len() { + let ids = std::sync::Arc::make_mut(&mut document.ids.block_ids); + let block_id = ids.remove(from); + ids.insert(to.min(ids.len()), block_id); + } + reconcile_document_ids(document); + return Ok(()); + } + + let mut blocks = projection_patch_blocks_from_document(document); + if from >= blocks.len() { + return Err(ProjectionApplyError::MissingBlock { + block_id: missing_block_id, + row_hint: from, + }); + } + let block = blocks.remove(from); + blocks.insert(to.min(blocks.len()), block); + validate_projection_patch_blocks(&blocks)?; + rebuild_document_from_projection_patch_blocks(document, blocks)?; + rebuild_document_sections(document); + Ok(()) +} + +fn replace_projection_block( + document: &mut DocumentProjection, + row: usize, + block_id: BlockId, + after: InputBlock, +) -> Result<(), ProjectionApplyError> { + let paragraph_id = paragraph_ix_for_block_row(document, row).and_then(|paragraph_ix| document.ids.paragraph_ids.get(paragraph_ix).copied()); + if matches!(document.blocks.get(row), Some(Block::Image(_) | Block::Equation(_) | Block::Table(_))) + && matches!(&after, InputBlock::Image(_) | InputBlock::Equation(_) | InputBlock::Table(_)) + { + if document + .ids + .block_ids + .iter() + .enumerate() + .any(|(ix, id)| ix != row && *id == block_id) + { + return Err(ProjectionApplyError::DuplicateBlockId(block_id)); + } + document.blocks.set(row, block_from_input_block(&after)); + if row < document.ids.block_ids.len() { + std::sync::Arc::make_mut(&mut document.ids.block_ids)[row] = block_id; + } + reconcile_document_ids(document); + return Ok(()); + } + + let mut blocks = projection_patch_blocks_from_document(document); + if row >= blocks.len() { + return Err(ProjectionApplyError::MissingBlock { block_id, row_hint: row }); + } + blocks[row] = ProjectionPatchBlock::from_structural_block(&structural_block_for_input(block_id, paragraph_id, after))?; + validate_projection_patch_blocks(&blocks)?; + rebuild_document_from_projection_patch_blocks(document, blocks)?; + rebuild_document_sections(document); + Ok(()) +} + + +#[hotpath::measure] +fn clamp_selection_to_document(document: &DocumentProjection, selection: &mut EditorSelection) { + selection.anchor = clamp_offset_to_document(document, selection.anchor); + selection.head = clamp_offset_to_document(document, selection.head); +} + +#[hotpath::measure] +fn clamp_offset_to_document(document: &DocumentProjection, offset: DocumentOffset) -> DocumentOffset { + let paragraph = offset.paragraph.min(document.paragraphs.len().saturating_sub(1)); + let byte = clamp_paragraph_byte_to_char_boundary(document, paragraph, offset.byte); + DocumentOffset { paragraph, byte } +} + +#[hotpath::measure] +fn extend_invalidation(target: &mut Option>, range: Range) { + if range.is_empty() { + return; + } + match target { + Some(existing) => { + existing.start = existing.start.min(range.start); + existing.end = existing.end.max(range.end); + }, + None => *target = Some(range), + } +} diff --git a/crates/gpui-flowtext/src/rich_text/editor/recovery.rs b/crates/gpui-flowtext/src/rich_text/editor/recovery.rs index e131564f..a39227e3 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/recovery.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/recovery.rs @@ -71,23 +71,20 @@ impl RichTextEditor { self .paragraph_chunk_layout_cache .resize(paragraph_count, None); - self - .paragraph_shaping_cache - .resize_with(paragraph_count, || None); - self.paragraph_prep_cache.resize_with(paragraph_count, ParagraphPrepSlot::default); + // §perf-heaven T8.12: prep/shaping caches are id-keyed maps; evict the + // offscreen paragraphs by removing their id entries (no positional resize). for paragraph_ix in 0..paragraph_count { if !layout_keep_ranges.contains(paragraph_ix) { - let entry = &mut self.paragraph_chunk_layout_cache[paragraph_ix]; - *entry = None; - if let Some(shape_cache) = self.paragraph_shaping_cache.get_mut(paragraph_ix) { - *shape_cache = None; + self.paragraph_chunk_layout_cache[paragraph_ix] = None; + if let Some(paragraph_id) = self.paragraph_id_at(paragraph_ix) { + self.paragraph_shaping_cache.remove(¶graph_id); } } if !prep_keep_ranges.contains(paragraph_ix) - && let Some(slot) = self.paragraph_prep_cache.get_mut(paragraph_ix) + && let Some(paragraph_id) = self.paragraph_id_at(paragraph_ix) { - slot.clear(); + self.paragraph_prep_cache.remove(¶graph_id); } } self @@ -148,6 +145,7 @@ impl RichTextEditor { let Some(path) = self.recovery_path.clone() else { return; }; + let source_path = self.document_path.clone(); if !self.has_unsaved_changes() { return; } @@ -159,9 +157,10 @@ impl RichTextEditor { return; } + let delay = self.recovery_write_delay(); self.recovery_write_in_progress = true; cx.spawn(async move |editor, cx| { - Timer::after(Duration::from_millis(750)).await; + Timer::after(delay).await; let snapshot_timing = Instant::now(); let decision = editor .update(cx, |editor, cx| { @@ -174,9 +173,18 @@ impl RichTextEditor { editor.recovery_write_in_progress = false; editor.schedule_recovery_write(cx); RecoveryWriteDecision::Rescheduled + } else if editor.last_text_input_at.is_some_and(|last_input| last_input.elapsed() < RECOVERY_TYPING_IDLE_WINDOW) { + editor.recovery_write_in_progress = false; + editor.schedule_recovery_write(cx); + RecoveryWriteDecision::Rescheduled } else if !editor.has_unsaved_changes() || editor.last_recovery_generation == editor.edit_generation { editor.recovery_write_in_progress = false; RecoveryWriteDecision::Idle + } else if let Some(hook) = editor.native_recovery_hook.clone() { + RecoveryWriteDecision::WriteRuntime { + generation: editor.edit_generation, + hook, + } } else { RecoveryWriteDecision::Write { generation: editor.edit_generation, @@ -186,19 +194,29 @@ impl RichTextEditor { }) .ok(); log_timing("recovery snapshot", snapshot_timing, ""); - let Some(RecoveryWriteDecision::Write { generation, document }) = decision else { - return; + let (generation, write_result, detail) = match decision { + Some(RecoveryWriteDecision::Write { generation, document }) => { + let write_timing = Instant::now(); + let paragraph_count = document.paragraphs.len(); + let result = cx + .background_executor() + .spawn(async move { + let document = detach_document_for_background_write(&document); + write_recovery_snapshot(&path, source_path.as_deref(), &document) + }) + .await; + log_timing_lazy("recovery write", write_timing, || format!("paragraphs={paragraph_count}")); + (generation, result, format!("paragraphs={paragraph_count}")) + }, + Some(RecoveryWriteDecision::WriteRuntime { generation, hook }) => { + let write_timing = Instant::now(); + let result = hook(path).await; + log_timing("runtime recovery write", write_timing, ""); + (generation, result, "runtime".to_string()) + }, + _ => return, }; - let write_timing = Instant::now(); - let paragraph_count = document.paragraphs.len(); - let write_result = cx - .background_executor() - .spawn(async move { - let document = detach_document_for_background_write(&document); - write_document(path, &document) - }) - .await; - log_timing_lazy("recovery write", write_timing, || format!("paragraphs={paragraph_count}")); + let _ = detail; match write_result { Ok(()) => { let _ = editor.update(cx, |editor, _| { @@ -206,7 +224,7 @@ impl RichTextEditor { }); }, Err(error) => { - eprintln!("failed to write recovery file: {error}"); + tracing::warn!(%error, "failed to write recovery file"); }, } let _ = editor.update(cx, |editor, cx| { @@ -224,4 +242,12 @@ impl RichTextEditor { .detach(); } + fn recovery_write_delay(&self) -> Duration { + self + .last_text_input_at + .and_then(|last_input| RECOVERY_TYPING_IDLE_WINDOW.checked_sub(last_input.elapsed())) + .filter(|delay| *delay > RECOVERY_WRITE_DEBOUNCE) + .unwrap_or(RECOVERY_WRITE_DEBOUNCE) + } + } diff --git a/crates/gpui-flowtext/src/rich_text/editor/render_blocks.rs b/crates/gpui-flowtext/src/rich_text/editor/render_blocks.rs index 9faa7b9e..b99d49c6 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/render_blocks.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/render_blocks.rs @@ -1,6 +1,6 @@ #[hotpath::measure] fn render_image_block( - document: &Document, + document: &DocumentProjection, image: &ImageBlock, block_ix: usize, row_size: Size, @@ -13,6 +13,9 @@ fn render_image_block( .child("Missing image") .into_any_element(); }; + if asset.is_loading_placeholder() { + return render_loading_image_placeholder(document, image, asset, row_size, selected, editor, block_ix); + } let Some(format) = ImageFormat::from_mime_type(asset.mime_type.as_ref()) else { return reserved_object_frame(document, row_size, selected) .child("Unsupported image") @@ -37,6 +40,34 @@ fn render_image_block( .into_any_element() } +#[hotpath::measure] +fn render_loading_image_placeholder( + document: &DocumentProjection, + image: &ImageBlock, + asset: &AssetRecord, + row_size: Size, + selected: bool, + editor: Entity, + block_ix: usize, +) -> gpui::AnyElement { + image_object_frame(document, image, asset, row_size, selected) + .child( + div() + .size_full() + .flex() + .flex_col() + .items_center() + .justify_center() + .gap_1() + .text_sm() + .text_color(rgb(0x6b7280)) + .child(div().text_xs().text_color(rgb(0x9ca3af)).child("...")) + .child("Loading image"), + ) + .when(selected, |this| this.children(image_resize_handles(editor, block_ix))) + .into_any_element() +} + #[hotpath::measure] fn image_resize_handles(editor: Entity, block_ix: usize) -> Vec { [ @@ -100,7 +131,7 @@ fn image_resize_handle(editor: Entity, block_ix: usize, handle: #[hotpath::measure] fn render_equation_block( - document: &Document, + document: &DocumentProjection, equation: &EquationBlock, block_ix: usize, row_size: Size, @@ -254,4 +285,3 @@ type EquationRenderCache = FxHashMap>, Stri static EQUATION_SVG_CACHE: OnceLock> = OnceLock::new(); static EQUATION_PNG_CACHE: OnceLock> = OnceLock::new(); - diff --git a/crates/gpui-flowtext/src/rich_text/editor/search_highlights.rs b/crates/gpui-flowtext/src/rich_text/editor/search_highlights.rs index fd26b9b3..b48e3f06 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/search_highlights.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/search_highlights.rs @@ -29,10 +29,11 @@ impl RichTextEditor { return false; }; - self.selection = EditorSelection { - anchor: range.start, - head: range.end, - }; + let before_selection = self.selection.clone(); + self.selection = EditorSelection::range(range.start, range.end); + if self.selection != before_selection { + self.emit_selection_changed(cx); + } self.apply_document_edit(cx, |editor, cx| { editor.insert_text(replacement, cx); }); @@ -58,57 +59,51 @@ impl RichTextEditor { .then_with(|| left.end.cmp(&right.end)) }); let count = ranges.len(); - let paragraph_count = self.document.paragraphs.len(); - self.apply_document_edit_with_capture_range(cx, Some(0..paragraph_count), |editor, cx| { - let mut final_caret = None; - let mut paragraph_groups = Vec::new(); - let mut cross_paragraph_ranges = Vec::new(); - for range in ranges { - if range.start.paragraph == range.end.paragraph { - if paragraph_groups - .last() - .is_none_or(|(paragraph_ix, _): &(usize, Vec>)| *paragraph_ix != range.start.paragraph) - { - paragraph_groups.push((range.start.paragraph, Vec::new())); - } - paragraph_groups - .last_mut() - .expect("paragraph group was just inserted") - .1 - .push(range.start.byte..range.end.byte); - } else { - cross_paragraph_ranges.push(range); - } - } - for range in cross_paragraph_ranges.into_iter().rev() { - editor.selection = EditorSelection { - anchor: range.start, - head: range.end, + // Invariant 5/6: every replacement goes through the write authority. + // Same-paragraph matches ride ONE compound `ReplaceMatches` intent (one + // gate hold, one commit, one undo member regardless of match count — the + // §11 anti-storm law); cross-paragraph matches (a match spanning a + // boundary) take the selection + insert-text path, which owns the + // structural join. The old direct-projection mutation silently never + // reached canonical state — peers, saves, and undo all lost it. + let mut matches = Vec::with_capacity(ranges.len()); + let mut cross_paragraph_ranges = Vec::new(); + for range in ranges { + if range.start.paragraph == range.end.paragraph { + let (Some(start), Some(end)) = (self.text_anchor_at(range.start), self.text_anchor_at(range.end)) else { + continue; }; - editor.insert_text(replacement, cx); - final_caret = Some(editor.selection.head); + let styles = self + .document + .paragraphs + .get(range.start.paragraph) + .map(|paragraph| styles_at_byte(paragraph, range.start.byte)); + matches.push(ReplaceMatch { start, end, styles }); + } else { + cross_paragraph_ranges.push(range); } + } - let affected_start = paragraph_groups - .first() - .map(|(paragraph_ix, _)| *paragraph_ix); - let affected_end = paragraph_groups - .last() - .map(|(paragraph_ix, _)| paragraph_ix + 1); - for (paragraph_ix, matches) in paragraph_groups.into_iter().rev() { - final_caret = replace_paragraph_matches(&mut editor.document, paragraph_ix, &matches, replacement).or(final_caret); - } - if let (Some(start), Some(end)) = (affected_start, affected_end) { - rebuild_document_offset_index(&mut editor.document); - let replacements = editor.document.paragraphs[start..end.min(editor.document.paragraphs.len())].to_vec(); - replace_paragraph_blocks(&mut editor.document, start, replacements.len(), &replacements); - } - if let Some(caret) = final_caret { - editor.selection = EditorSelection { anchor: caret, head: caret }; + let legs = cross_paragraph_ranges.len() + usize::from(!matches.is_empty()); + let grouped = legs > 1; + if grouped { + self.begin_undo_group(); + } + for range in cross_paragraph_ranges.into_iter().rev() { + self.selection = EditorSelection::range(range.start, range.end); + if replacement.is_empty() { + self.write_delete_selection(cx); + } else { + self.insert_text(replacement, cx); } - editor.after_text_mutation(cx); - }); + } + if !matches.is_empty() { + self.write_replace_matches(matches, replacement, cx); + } + if grouped { + self.end_undo_group(); + } self.search_highlights.clear(); self.active_search_highlight = None; cx.notify(); @@ -129,84 +124,6 @@ impl RichTextEditor { } } -fn replace_paragraph_matches( - document: &mut Document, - paragraph_ix: usize, - matches: &[Range], - replacement: &str, -) -> Option { - if matches.is_empty() { - return None; - } - let old_text = paragraph_text(document, paragraph_ix); - let old_paragraph = document.paragraphs.get(paragraph_ix)?.clone(); - let final_len = old_text - .len() - .saturating_add(matches.len().saturating_mul(replacement.len())) - .saturating_sub(matches.iter().map(|range| range.len()).sum::()); - let mut new_text = String::with_capacity(final_len); - let mut new_runs = Vec::with_capacity(old_paragraph.runs.len() + matches.len().saturating_mul(2)); - let mut cursor = 0; - for range in matches { - append_original_paragraph_span(&old_text, &old_paragraph, cursor..range.start, &mut new_text, &mut new_runs); - let styles = styles_at_byte(&old_paragraph, range.start); - new_text.push_str(replacement); - push_merged_run( - &mut new_runs, - TextRun { - len: replacement.len(), - styles, - }, - ); - cursor = range.end; - } - append_original_paragraph_span(&old_text, &old_paragraph, cursor..old_text.len(), &mut new_text, &mut new_runs); - - let paragraph_range = paragraph_byte_range(document, paragraph_ix); - document.text.delete(paragraph_range.clone()); - document.text.insert(paragraph_range.start, &new_text); - let paragraph = &mut paragraphs_mut(document)[paragraph_ix]; - paragraph.runs = new_runs; - paragraph.byte_range = paragraph_range.start..paragraph_range.start + new_text.len(); - bump_paragraph_version(paragraph); - Some(DocumentOffset { - paragraph: paragraph_ix, - byte: matches.last()?.start + replacement.len(), - }) -} - -fn append_original_paragraph_span( - old_text: &str, - old_paragraph: &Paragraph, - span: Range, - new_text: &mut String, - new_runs: &mut Vec, -) { - if span.is_empty() { - return; - } - new_text.push_str(&old_text[span.clone()]); - let mut run_start = 0; - for run in &old_paragraph.runs { - let run_end = run_start + run.len; - let start = span.start.max(run_start); - let end = span.end.min(run_end); - if start < end { - push_merged_run( - new_runs, - TextRun { - len: end - start, - styles: run.styles, - }, - ); - } - run_start = run_end; - if run_start >= span.end { - break; - } - } -} - fn styles_at_byte(paragraph: &Paragraph, byte: usize) -> RunStyles { let (run_ix, _) = run_containing(paragraph, byte); paragraph @@ -215,15 +132,3 @@ fn styles_at_byte(paragraph: &Paragraph, byte: usize) -> RunStyles { .map_or_else(RunStyles::default, |run| run.styles) } -fn push_merged_run(runs: &mut Vec, run: TextRun) { - if run.len == 0 { - return; - } - if let Some(last) = runs.last_mut() - && last.styles == run.styles - { - last.len += run.len; - return; - } - runs.push(run); -} diff --git a/crates/gpui-flowtext/src/rich_text/editor/send_export.rs b/crates/gpui-flowtext/src/rich_text/editor/send_export.rs index 9d4c2cb6..d6b99bd3 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/send_export.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/send_export.rs @@ -6,15 +6,24 @@ pub trait DocumentExportAdapter: Send + Sync + 'static { .map(Path::to_path_buf) } - fn write_document_export(&self, output_path: &Path, document: &Document, format: DocumentExportFormat) -> io::Result<()>; + fn write_document_export(&self, output_path: &Path, document: &DocumentProjection, format: DocumentExportFormat) -> io::Result<()>; +} + +pub trait DocumentRecoveryAdapter: Send + Sync + 'static { + fn write_recovery_snapshot(&self, recovery_path: &Path, source_path: Option<&Path>, document: &DocumentProjection) -> io::Result<()>; } static DOCUMENT_EXPORT_ADAPTER: OnceLock> = OnceLock::new(); +static DOCUMENT_RECOVERY_ADAPTER: OnceLock> = OnceLock::new(); pub fn set_document_export_adapter(adapter: Arc) -> Result<(), Arc> { DOCUMENT_EXPORT_ADAPTER.set(adapter) } +pub fn set_document_recovery_adapter(adapter: Arc) -> Result<(), Arc> { + DOCUMENT_RECOVERY_ADAPTER.set(adapter) +} + #[hotpath::measure_all] impl RichTextEditor { pub fn send_document(&mut self, format: DocumentExportFormat, cx: &mut Context) -> Task> { @@ -29,6 +38,22 @@ impl RichTextEditor { }; let generation = self.edit_generation; let document = self.document.clone(); + if let Some(export_hook) = self.native_export_hook.clone() { + let assets = document.assets.assets.values().cloned().collect(); + return cx.spawn(async move |editor, cx| { + let result = export_hook(output_path.clone(), format, assets).await; + match result { + Ok(()) => { + let _ = editor.update(cx, |editor, cx| { + editor.last_send_document_generation = Some(generation); + cx.notify(); + }); + Ok(output_path) + }, + Err(error) => Err(error), + } + }); + } cx.spawn(async move |editor, cx| { let result = cx .background_executor() @@ -59,6 +84,22 @@ impl RichTextEditor { }; let generation = self.edit_generation; let document = self.document.clone(); + if let Some(export_hook) = self.native_export_hook.clone() { + let assets = document.assets.assets.values().cloned().collect(); + return cx.spawn(async move |editor, cx| { + let result = export_hook(output_path.clone(), format, assets).await; + match result { + Ok(()) => { + let _ = editor.update(cx, |editor, cx| { + editor.last_format_export_generation = Some(generation); + cx.notify(); + }); + Ok(output_path) + }, + Err(error) => Err(error), + } + }); + } cx.spawn(async move |editor, cx| { let result = cx .background_executor() @@ -195,15 +236,83 @@ impl DocumentExportFormat { } #[hotpath::measure] -fn write_document_export(output_path: &Path, document: &Document, format: DocumentExportFormat) -> io::Result<()> { +fn write_document_export(output_path: &Path, document: &DocumentProjection, format: DocumentExportFormat) -> io::Result<()> { if let Some(adapter) = DOCUMENT_EXPORT_ADAPTER.get() { return adapter.write_document_export(output_path, document, format); } - match format { - DocumentExportFormat::Native | DocumentExportFormat::NativeWithExtension(_) => write_document(output_path, document), - DocumentExportFormat::Docx | DocumentExportFormat::Pdf => Err(io::Error::new( - io::ErrorKind::Unsupported, - "DOCX and PDF export are host-application adapters; gpui-flowtext only writes its native binary format directly", - )), + let _ = (output_path, document, format); + Err(io::Error::new( + io::ErrorKind::Unsupported, + "document export requires a host-application adapter", + )) +} + +#[hotpath::measure] +fn write_native_document(output_path: &Path, document: &DocumentProjection) -> io::Result<()> { + if let Some(adapter) = DOCUMENT_EXPORT_ADAPTER.get() { + return adapter.write_document_export(output_path, document, DocumentExportFormat::Native); + } + let _ = (output_path, document); + Err(io::Error::new( + io::ErrorKind::Unsupported, + "native save requires a host-application adapter", + )) +} + +#[hotpath::measure] +fn write_recovery_snapshot(recovery_path: &Path, source_path: Option<&Path>, document: &DocumentProjection) -> io::Result<()> { + if let Some(adapter) = DOCUMENT_RECOVERY_ADAPTER.get() { + return adapter.write_recovery_snapshot(recovery_path, source_path, document); + } + let _ = (recovery_path, source_path, document); + Err(io::Error::new( + io::ErrorKind::Unsupported, + "recovery snapshots require a host-application adapter", + )) +} + +#[cfg(test)] +mod send_export_tests { + use super::*; + + #[test] + fn native_extension_without_adapter_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("document.db8"); + + let error = write_document_export(&path, &blank_document(), DocumentExportFormat::NativeWithExtension("db8")).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::Unsupported); + assert!(!path.exists()); + } + + #[test] + fn native_save_without_adapter_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("document.db8"); + + let error = write_native_document(&path, &blank_document()).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::Unsupported); + assert!(!path.exists()); + } + + #[test] + fn recovery_for_db8_source_without_adapter_is_unsupported() { + let error = write_recovery_snapshot( + Path::new("document.db8.recovery"), + Some(Path::new("document.db8")), + &blank_document(), + ) + .unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::Unsupported); + } + + #[test] + fn recovery_for_db8_path_without_source_without_adapter_is_unsupported() { + let error = write_recovery_snapshot(Path::new("collaboration.db8"), None, &blank_document()).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::Unsupported); } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/serialization.rs b/crates/gpui-flowtext/src/rich_text/editor/serialization.rs index 4049c090..ccdca2cd 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/serialization.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/serialization.rs @@ -1,21 +1,3 @@ -#[hotpath::measure] -fn canonical_insert_text_operations(paragraph_id: ParagraphId, start_byte: usize, paragraph: &InputParagraph) -> Vec { - let mut byte = start_byte; - let mut operations = Vec::with_capacity(paragraph.runs.len()); - for run in ¶graph.runs { - if !run.text.is_empty() { - operations.push(CanonicalOperation::InsertText { - paragraph: paragraph_id, - byte, - text: run.text.clone(), - styles: run.styles, - }); - } - byte += run.text.len(); - } - operations -} - #[hotpath::measure] pub(super) fn input_paragraph_text(paragraph: &InputParagraph) -> String { paragraph.runs.iter().map(|run| run.text.as_str()).collect() @@ -51,7 +33,7 @@ fn collect_block_assets(block: &Block, assets: &AssetStore, output: &mut Vec InputBlock { +pub fn input_block_from_block(block: &Block) -> InputBlock { match block { Block::Paragraph(paragraph) => InputBlock::Paragraph(input_paragraph_from_paragraph(paragraph)), Block::Image(image) => InputBlock::Image(InputImageBlock { @@ -64,6 +46,7 @@ fn input_block_from_block(block: &Block) -> InputBlock { ImageSizing::Fixed { width_px, height_px } => InputImageSizing::Fixed { width_px, height_px }, }, alignment: input_alignment_from_alignment(image.alignment), + external_url: image.external_url.as_ref().map(ToString::to_string), }), Block::Equation(equation) => InputBlock::Equation(InputEquationBlock { source: equation.source.to_string(), @@ -78,7 +61,7 @@ fn input_block_from_block(block: &Block) -> InputBlock { } #[hotpath::measure] -fn block_from_input_block(block: &InputBlock) -> Block { +pub fn block_from_input_block(block: &InputBlock) -> Block { match block { InputBlock::Paragraph(paragraph) => Block::Paragraph(paragraph_from_input_paragraph(paragraph)), InputBlock::Image(image) => Block::Image(ImageBlock { @@ -91,6 +74,7 @@ fn block_from_input_block(block: &InputBlock) -> Block { InputImageSizing::Fixed { width_px, height_px } => ImageSizing::Fixed { width_px, height_px }, }, alignment: alignment_from_input_alignment(image.alignment), + external_url: image.external_url.clone().map(Into::into), version: 0, }), InputBlock::Equation(equation) => Block::Equation(EquationBlock { @@ -106,6 +90,58 @@ fn block_from_input_block(block: &InputBlock) -> Block { } } +#[hotpath::measure] +#[must_use] +pub fn document_from_input_blocks(theme: DocumentTheme, input_blocks: Vec) -> DocumentProjection { + let mut text = String::new(); + let mut paragraphs = Vec::new(); + let mut blocks = Vec::with_capacity(input_blocks.len()); + + for input_block in input_blocks { + match input_block { + InputBlock::Paragraph(paragraph) => { + if !paragraphs.is_empty() { + text.push('\n'); + } + text.push_str(&input_paragraph_text(¶graph)); + let paragraph = paragraph_from_input_paragraph(¶graph); + paragraphs.push(paragraph.clone()); + blocks.push(Block::Paragraph(paragraph)); + }, + InputBlock::Image(image) => blocks.push(block_from_input_block(&InputBlock::Image(image))), + InputBlock::Equation(equation) => blocks.push(block_from_input_block(&InputBlock::Equation(equation))), + InputBlock::Table(table) => blocks.push(block_from_input_block(&InputBlock::Table(table))), + } + } + + if paragraphs.is_empty() { + let paragraph = Paragraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + version: 0, + }; + paragraphs.push(paragraph.clone()); + blocks.push(Block::Paragraph(paragraph)); + } + + let paragraph_count = paragraphs.len(); + let block_count = blocks.len(); + let mut document = DocumentProjection { + frontier: Vec::new(), + text: Rope::from(text), + paragraphs: ParagraphSeq::from_vec(paragraphs), + blocks: BlockSeq::from_vec(blocks), + assets: AssetStore::default(), + ids: document_ids_for_shape(paragraph_count, block_count), + sections: Arc::new(Vec::new()), + outline: Arc::new(Vec::new()), + theme, + }; + reconcile_document_ids(&mut document); + rebuild_document_sections(&mut document); + document +} + #[hotpath::measure] fn input_paragraph_from_paragraph(paragraph: &Paragraph) -> InputParagraph { InputParagraph { @@ -122,7 +158,7 @@ fn input_paragraph_from_paragraph(paragraph: &Paragraph) -> InputParagraph { } #[hotpath::measure] -fn input_paragraph_from_document_range(document: &Document, paragraph_ix: usize, range: Range) -> InputParagraph { +pub fn input_paragraph_from_document_range(document: &DocumentProjection, paragraph_ix: usize, range: Range) -> InputParagraph { let paragraph = &document.paragraphs[paragraph_ix]; let paragraph_range = paragraph_byte_range(document, paragraph_ix); let start = range.start.min(paragraph_text_len(paragraph)); @@ -201,10 +237,8 @@ fn input_paragraph_from_table_cell_range(paragraph: &TableCellParagraph, range: #[hotpath::measure] fn paragraph_from_input_paragraph(paragraph: &InputParagraph) -> Paragraph { - let len = paragraph.runs.iter().map(|run| run.text.len()).sum(); Paragraph { style: paragraph.style, - byte_range: 0..len, runs: merge_adjacent_runs( paragraph .runs @@ -234,32 +268,14 @@ fn input_table_from_table(table: &TableBlock) -> InputTableBlock { rows: table .rows .iter() - .map(|row| InputTableRow { - cells: row - .cells - .iter() - .map(|cell| InputTableCell { - blocks: cell - .blocks - .iter() - .map(|block| match block { - TableCellBlock::Paragraph(paragraph) => InputTableCellBlock::Paragraph(input_paragraph_from_table_cell_paragraph(paragraph)), - TableCellBlock::Table(table) => InputTableCellBlock::Table(input_table_from_table(table)), - }) - .collect(), - row_span: cell.row_span, - col_span: cell.col_span, - }) - .collect(), - }) + .map(input_table_row_from_table_row) .collect(), - column_widths: table - .column_widths + columns: table + .columns .iter() - .map(|width| match *width { - TableColumnWidth::Auto => InputTableColumnWidth::Auto, - TableColumnWidth::FixedPx(px) => InputTableColumnWidth::FixedPx(px), - TableColumnWidth::Fraction(fraction) => InputTableColumnWidth::Fraction(fraction), + .map(|column| InputTableColumn { + id: column.id, + width: input_table_column_width_from_table_column_width(&column.width), }) .collect(), style: InputTableStyle { @@ -268,6 +284,46 @@ fn input_table_from_table(table: &TableBlock) -> InputTableBlock { } } +#[hotpath::measure] +fn input_table_row_from_table_row(row: &TableRow) -> InputTableRow { + InputTableRow { + id: row.id, + cells: row + .cells + .iter() + .map(input_table_cell_from_table_cell) + .collect(), + } +} + +#[hotpath::measure] +fn input_table_cell_from_table_cell(cell: &TableCell) -> InputTableCell { + InputTableCell { + id: cell.id, + row_id: cell.row_id, + column_id: cell.column_id, + blocks: cell + .blocks + .iter() + .map(|block| match block { + TableCellBlock::Paragraph(paragraph) => InputTableCellBlock::Paragraph(input_paragraph_from_table_cell_paragraph(paragraph)), + TableCellBlock::Table(table) => InputTableCellBlock::Table(input_table_from_table(table)), + }) + .collect(), + row_span: cell.row_span, + col_span: cell.col_span, + } +} + +#[hotpath::measure] +fn input_table_column_width_from_table_column_width(width: &TableColumnWidth) -> InputTableColumnWidth { + match *width { + TableColumnWidth::Auto => InputTableColumnWidth::Auto, + TableColumnWidth::FixedPx(px) => InputTableColumnWidth::FixedPx(px), + TableColumnWidth::Fraction(fraction) => InputTableColumnWidth::Fraction(fraction), + } +} + #[hotpath::measure] fn table_from_input_table(table: &InputTableBlock) -> TableBlock { TableBlock { @@ -275,10 +331,14 @@ fn table_from_input_table(table: &InputTableBlock) -> TableBlock { .rows .iter() .map(|row| TableRow { + id: row.id, cells: row .cells .iter() .map(|cell| TableCell { + id: cell.id, + row_id: cell.row_id, + column_id: cell.column_id, blocks: cell .blocks .iter() @@ -293,13 +353,12 @@ fn table_from_input_table(table: &InputTableBlock) -> TableBlock { .collect(), }) .collect(), - column_widths: table - .column_widths + columns: table + .columns .iter() - .map(|width| match *width { - InputTableColumnWidth::Auto => TableColumnWidth::Auto, - InputTableColumnWidth::FixedPx(px) => TableColumnWidth::FixedPx(px), - InputTableColumnWidth::Fraction(fraction) => TableColumnWidth::Fraction(fraction), + .map(|column| TableColumn { + id: column.id, + width: table_column_width_from_input_width(&column.width), }) .collect(), style: TableStyle { diff --git a/crates/gpui-flowtext/src/rich_text/editor/style_mutation.rs b/crates/gpui-flowtext/src/rich_text/editor/style_mutation.rs index 47ea3274..cb9feac2 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/style_mutation.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/style_mutation.rs @@ -1,7 +1,7 @@ #[hotpath::measure_all] impl RichTextEditor { fn toggle_underline_kind(&mut self, explicit_direct: Option, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { let Some(selection_range) = self.table_cell_selection_range() else { return; }; @@ -50,7 +50,12 @@ impl RichTextEditor { return; } if self.selection.is_caret() { - let paragraph_style = self.document.paragraphs[self.selection.head.paragraph].style; + let paragraph_style = self + .document + .paragraphs + .get(self.selection.head.paragraph) + .map(|paragraph| paragraph.style) + .unwrap_or(ParagraphStyle::Normal); let direct = explicit_direct.unwrap_or(matches!(paragraph_style, ParagraphStyle::Custom(3) | ParagraphStyle::Custom(4))); let mut styles = self.styles_at_caret(); if direct { @@ -72,26 +77,21 @@ impl RichTextEditor { let range = self.selection.normalized(); let direct = explicit_direct.unwrap_or_else(|| selection_prefers_direct_underline(&self.document, range.clone())); let all_selected = selection_all_underline_kind(&self.document, range.clone(), direct); - self.apply_document_edit(cx, |editor, cx| { - mutate_runs_in_range(&mut editor.document, range, |styles| { - if direct { - styles.direct_underline = !all_selected; - } else { - let new_value = !all_selected; - if new_value { - styles.semantic = RunSemanticStyle::Custom(3); - styles.direct_underline = false; - } else { - styles.semantic = RunSemanticStyle::Plain; - } - } - }); - editor.after_formatting_mutation(cx); - }); + let mut styles = run_styles_at_offset(&self.document, range.start); + if direct { + styles.direct_underline = !all_selected; + } else if all_selected { + styles.semantic = RunSemanticStyle::Plain; + } else { + styles.semantic = RunSemanticStyle::Custom(3); + styles.direct_underline = false; + } + self.pending_styles = None; + self.write_set_marks(range, styles, cx); } fn toggle_semantic_style(&mut self, semantic: RunSemanticStyle, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { let Some(selection_range) = self.table_cell_selection_range() else { return; }; @@ -135,16 +135,14 @@ impl RichTextEditor { let range = self.selection.normalized(); let all_selected = selection_all_run_styles(&self.document, range.clone(), |styles| styles.semantic == semantic); - self.apply_document_edit(cx, |editor, cx| { - mutate_runs_in_range(&mut editor.document, range, |styles| { - styles.semantic = if all_selected { RunSemanticStyle::Plain } else { semantic }; - }); - editor.after_formatting_mutation(cx); - }); + let mut styles = run_styles_at_offset(&self.document, range.start); + styles.semantic = if all_selected { RunSemanticStyle::Plain } else { semantic }; + self.pending_styles = None; + self.write_set_marks(range, styles, cx); } fn set_highlight_internal(&mut self, highlight: Option, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block { let Some(selection_range) = self.table_cell_selection_range() else { return; }; @@ -190,25 +188,32 @@ impl RichTextEditor { false }; let target_highlight = if all_selected { None } else { highlight }; - self.apply_document_edit(cx, |editor, cx| { - mutate_runs_in_range(&mut editor.document, range, |styles| styles.highlight = target_highlight); - editor.after_formatting_mutation(cx); - }); + let mut styles = run_styles_at_offset(&self.document, range.start); + styles.highlight = target_highlight; + self.pending_styles = None; + self.write_set_marks(range, styles, cx); } pub(super) fn styles_at_caret(&self) -> RunStyles { if let Some(styles) = self.pending_styles { return styles; } - let caret = self.selection.head; - let paragraph = &self.document.paragraphs[caret.paragraph]; - let (run_ix, _) = run_containing(paragraph, caret.byte); - paragraph - .runs - .get(run_ix) - .map(|run| run.styles) - .unwrap_or_default() + run_styles_at_offset(&self.document, self.selection.head) } // -------- Movement primitives ---------------------------------------- } + +/// Pure projection read: the run styles governing `offset` (used as the base +/// a `SetMarks` intent restates over a selection). +fn run_styles_at_offset(document: &DocumentProjection, offset: DocumentOffset) -> RunStyles { + let Some(paragraph) = document.paragraphs.get(offset.paragraph) else { + return RunStyles::default(); + }; + let (run_ix, _) = run_containing(paragraph, offset.byte.min(paragraph_text_len(paragraph))); + paragraph + .runs + .get(run_ix) + .map(|run| run.styles) + .unwrap_or_default() +} diff --git a/crates/gpui-flowtext/src/rich_text/editor/style_state.rs b/crates/gpui-flowtext/src/rich_text/editor/style_state.rs index 07a2bd5e..8bcf037a 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/style_state.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/style_state.rs @@ -107,6 +107,19 @@ impl RichTextEditor { self.edit_generation } + pub fn mark_saved_generation(&mut self, generation: u64, cx: &mut Context) { + self.saved_generation = self.saved_generation.max(generation); + self.refresh_save_status(); + cx.notify(); + } + + pub fn set_save_failed(&mut self, generation: u64, message: String, cx: &mut Context) { + if generation >= self.saved_generation { + self.save_status = SaveStatus::SaveFailed(message); + } + cx.notify(); + } + pub fn update_document_theme(&mut self, update: impl FnOnce(&mut DocumentTheme), cx: &mut Context) { update(&mut self.document.theme); rebuild_document_sections(&mut self.document); @@ -115,6 +128,11 @@ impl RichTextEditor { fn invalidate_document_theme_layout(&mut self, cx: &mut Context) { self.clear_layout_work_caches(); + // §act-nine A9.3 (pre-existing stale-prep hazard): the theme feeds + // invisibility-mode prep (`run_is_visible_for_theme` decides projected + // text/runs), but theme identity is not part of `ParagraphPrepKey` — a + // theme change must clear the prep map too, or stale projections survive. + self.clear_all_layout_prep(); self.paragraph_chunk_layout_cache = vec![None; self.document.paragraphs.len()]; self.paragraph_height_cache = vec![None; self.document.paragraphs.len()]; self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); @@ -146,28 +164,10 @@ impl RichTextEditor { self.paragraph_height_cache.resize(paragraph_count, None); for paragraph_ix in 0..paragraph_count { - let Some(paragraph) = self.document.paragraphs.get(paragraph_ix) else { - self.paragraph_chunk_layout_cache[paragraph_ix] = None; - self.paragraph_height_cache[paragraph_ix] = None; - continue; - }; - let key = paragraph_cache_key(&self.document, paragraph); - let chunk_valid = self.valid_chunk_cache_entry(paragraph_ix, width).is_some(); - if !chunk_valid { + if self.valid_chunk_cache_entry(paragraph_ix, width).is_none() { self.paragraph_chunk_layout_cache[paragraph_ix] = None; } - - let height_valid = self - .paragraph_height_cache - .get(paragraph_ix) - .and_then(|entry| entry.as_ref()) - .is_some_and(|entry| { - entry.key == key - && entry.width == width - && entry.invisibility_mode == self.invisibility_mode - && entry.edit_generation == self.edit_generation - }); - if !height_valid { + if self.valid_paragraph_height(paragraph_ix, width).is_none() { self.paragraph_height_cache[paragraph_ix] = None; } } @@ -178,32 +178,6 @@ impl RichTextEditor { self.height_prefix_index = HeightPrefixIndex::default(); } - fn invalidate_paragraph_layout_cache_range(&mut self, range: Range) { - let paragraph_count = self.document.paragraphs.len(); - let expanded_range = expand_paragraph_range(range, paragraph_count, 2); - self.clear_layout_work_cache_range(expanded_range.clone()); - self.clear_layout_prep_range(expanded_range.clone()); - self - .paragraph_chunk_layout_cache - .resize(paragraph_count, None); - self.paragraph_height_cache.resize(paragraph_count, None); - - for paragraph_ix in expanded_range.clone() { - if let Some(cache) = self.paragraph_chunk_layout_cache.get_mut(paragraph_ix) { - *cache = None; - } - if let Some(cache) = self.paragraph_height_cache.get_mut(paragraph_ix) { - *cache = None; - } - } - - self.paragraph_height_cache_revision = self.paragraph_height_cache_revision.wrapping_add(1); - self.pending_item_sizes_patch_range = Some(match self.pending_item_sizes_patch_range.take() { - Some(previous) => previous.start.min(expanded_range.start)..previous.end.max(expanded_range.end), - None => expanded_range, - }); - } - pub fn invisibility_mode(&self) -> bool { self.invisibility_mode } @@ -266,12 +240,46 @@ impl RichTextEditor { let recovery_path = self.recovery_path.clone(); self.save_status = SaveStatus::Saving; cx.notify(); + if let Some(save_hook) = self.native_save_hook.clone() { + // Loro-first: every local edit is already committed to the doc core, so + // the hook persists canonical state — there is nothing to flush, replay, + // or re-install; the projection stays untouched. + let assets = document.assets.assets.values().cloned().collect(); + return cx.spawn(async move |editor, cx| { + let write_result = save_hook(path, assets).await; + if write_result.is_ok() + && let Some(recovery_path) = recovery_path + { + let _ = fs::remove_file(recovery_path); + } + match write_result { + Ok(()) => { + let _ = editor.update(cx, |editor, cx| { + editor.saved_generation = editor.saved_generation.max(generation); + editor.refresh_save_status(); + cx.notify(); + }); + Ok(()) + }, + Err(error) => { + let message = error.to_string(); + let _ = editor.update(cx, |editor, cx| { + if generation >= editor.saved_generation { + editor.save_status = SaveStatus::SaveFailed(message); + } + cx.notify(); + }); + Err(error) + }, + } + }); + } cx.spawn(async move |editor, cx| { let write_result = cx .background_executor() .spawn(async move { let document = detach_document_for_background_write(&document); - let result = write_document(&path, &document); + let result = write_native_document(&path, &document); if result.is_ok() && let Some(recovery_path) = recovery_path { diff --git a/crates/gpui-flowtext/src/rich_text/editor/table_equation_editing.rs b/crates/gpui-flowtext/src/rich_text/editor/table_equation_editing.rs index a5fe2d24..7ae42b87 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/table_equation_editing.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/table_equation_editing.rs @@ -1,7 +1,17 @@ #[hotpath::measure_all] impl RichTextEditor { + fn insert_text_into_selected_object_text(&mut self, text: &str, cx: &mut Context) -> bool { + if self.insert_text_into_selected_table_cell(text, cx) { + return true; + } + if self.insert_text_into_selected_equation(text, cx) { + return true; + } + false + } + fn insert_text_into_selected_table_cell(&mut self, text: &str, cx: &mut Context) -> bool { - let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block else { + let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block else { return false; }; if text.is_empty() { @@ -16,63 +26,86 @@ impl RichTextEditor { .selected_table_cell_paragraph() .map(|paragraph| table_cell_styles_at(paragraph, insert_at)) .unwrap_or_default(); - self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { + let committed = self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { if let Some(range) = selection_range.clone() { delete_range_in_table_cell_paragraph(paragraph, range); } insert_text_in_table_cell_paragraph(paragraph, insert_at, text, styles); }); - self.table_cell_caret = insert_at.saturating_add(text.len()); - self.table_cell_anchor = self.table_cell_caret; + if committed { + // Local caret state inside the cell stays editor-side (spec §5); the + // projection itself advanced through the intent's returned patches. + self.table_cell_caret = insert_at.saturating_add(text.len()); + self.table_cell_anchor = self.table_cell_caret; + } true } fn split_selected_table_cell_paragraph(&mut self, cx: &mut Context) -> bool { - let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block else { + let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block else { return false; }; - let Some(Block::Table(table)) = self.document.blocks.get(block_ix).cloned() else { - return false; + let paragraph_ix = self.table_cell_block_ix; + let caret = self.table_cell_caret; + let mut new_paragraph_ix = None; + let committed = self.edit_table_cell(block_ix, row_ix, cell_ix, cx, |cell| { + new_paragraph_ix = split_table_cell_paragraph_at(cell, paragraph_ix, caret); + }); + if committed && let Some(paragraph_ix) = new_paragraph_ix { + self.table_cell_block_ix = paragraph_ix; + self.table_cell_caret = 0; + cx.notify(); + } + true + } + + /// Table-cell text edits keep whole-cell granularity: build the after-cell + /// from the projected cell (via `update`) and issue ONE + /// `TableIntent::ReplaceCell`. The document is never mutated directly — the + /// intent's returned patches advance the projection inside `write_intent`. + fn edit_table_cell( + &mut self, + block_ix: usize, + row_ix: usize, + cell_ix: usize, + cx: &mut Context, + update: impl FnOnce(&mut TableCell), + ) -> bool { + let before = match self.document.blocks.get(block_ix) { + Some(Block::Table(table)) => table + .rows + .get(row_ix) + .and_then(|row| row.cells.get(cell_ix)) + .cloned(), + _ => None, }; - let mut updated = table.clone(); - let Some(cell) = updated - .rows - .get_mut(row_ix) - .and_then(|row| row.cells.get_mut(cell_ix)) - else { + let Some(before) = before else { return false; }; - let Some(new_paragraph_ix) = split_table_cell_paragraph_at(cell, self.table_cell_block_ix, self.table_cell_caret) else { - return true; - }; - if updated == table { - return true; - } - updated.version = updated.version.wrapping_add(1); - let before = Block::Table(table); - let after = Block::Table(updated); - if let Some(block) = Arc::make_mut(&mut self.document.blocks).get_mut(block_ix) { - *block = after.clone(); + let mut after = before.clone(); + update(&mut after); + if after == before { + return false; } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { block_ix, before, after }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(block_ix), - }], - }); - self.redo_stack.clear(); - self.table_cell_block_ix = new_paragraph_ix; - self.table_cell_caret = 0; - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); - true + let Some(table_id) = self.semantic_block_id(block_ix) else { + tracing::warn!(block_ix, "refusing table-cell edit: projection block has no durable id, so no intent can address it"); + return false; + }; + // Durable coordinate from the id-bearing model at the edited cell's + // position (§P2b); the cell already carries these ids. + let row = after.row_id; + let column = after.column_id; + self + .write_intent( + LocalIntent::Table(crate::local_intents::TableIntent::ReplaceCell { + table: table_id, + row, + column, + cell: input_table_cell_from_table_cell(&after), + }), + cx, + ) + .is_some() } fn insert_text_into_selected_equation(&mut self, text: &str, cx: &mut Context) -> bool { @@ -87,48 +120,26 @@ impl RichTextEditor { .as_ref() .map(|range| range.start) .unwrap_or(self.equation_source_caret); - self.edit_selected_equation(block_ix, cx, |equation| { - let mut source = equation.source.to_string(); - let insert_at = insert_at.min(source.len()); - if !source.is_char_boundary(insert_at) { - return; - } - if let Some(range) = selection_range.clone() - && range.start <= range.end - && range.end <= source.len() - && source.is_char_boundary(range.start) - && source.is_char_boundary(range.end) - { - source.replace_range(range, ""); - } - source.insert_str(insert_at, text); - equation.source = source.into(); - equation.version = equation.version.wrapping_add(1); - }); - self.equation_source_caret = insert_at.saturating_add(text.len()); - self.equation_source_anchor = self.equation_source_caret; + let range = selection_range.unwrap_or(insert_at..insert_at); + if self.edit_selected_equation_source_range(block_ix, range, text, cx) { + self.equation_source_caret = insert_at.saturating_add(text.len()); + self.equation_source_anchor = self.equation_source_caret; + } true } fn backspace_selected_table_cell(&mut self, cx: &mut Context) -> bool { - let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block else { + let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block else { return false; }; let caret = self.table_cell_caret; if caret == 0 { let mut merged_caret = None; let current_paragraph_ix = self.table_cell_block_ix; - self.edit_selected_table(cx, |table| { - let Some(cell) = table - .rows - .get_mut(row_ix) - .and_then(|row| row.cells.get_mut(cell_ix)) - else { - return; - }; + let committed = self.edit_table_cell(block_ix, row_ix, cell_ix, cx, |cell| { merged_caret = merge_table_cell_paragraph_with_previous(cell, current_paragraph_ix); }); - if let Some((paragraph_ix, byte)) = merged_caret { + if committed && let Some((paragraph_ix, byte)) = merged_caret { self.table_cell_block_ix = paragraph_ix; self.table_cell_caret = byte; cx.notify(); @@ -148,7 +159,7 @@ impl RichTextEditor { }) }) .unwrap_or(caret); - self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { + let committed = self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { let caret = caret.min(paragraph.text.len()); if caret == 0 { return; @@ -160,12 +171,14 @@ impl RichTextEditor { .unwrap_or(0); delete_range_in_table_cell_paragraph(paragraph, prev..caret); }); - self.table_cell_caret = new_caret; + if committed { + self.table_cell_caret = new_caret; + } true } fn delete_forward_selected_table_cell(&mut self, cx: &mut Context) -> bool { - let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block else { + let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix, .. }) = self.selected_block else { return false; }; let Some(text) = self.selected_table_cell_text() else { @@ -182,30 +195,24 @@ impl RichTextEditor { caret }; if next > caret { - self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { + let committed = self.edit_table_cell_paragraph(block_ix, row_ix, cell_ix, cx, |paragraph| { delete_range_in_table_cell_paragraph(paragraph, caret..next); }); + if committed { + self.table_cell_caret = caret; + } } else { let mut merged_caret = None; let current_paragraph_ix = self.table_cell_block_ix; - self.edit_selected_table(cx, |table| { - let Some(cell) = table - .rows - .get_mut(row_ix) - .and_then(|row| row.cells.get_mut(cell_ix)) - else { - return; - }; + let committed = self.edit_table_cell(block_ix, row_ix, cell_ix, cx, |cell| { merged_caret = merge_table_cell_paragraph_with_next(cell, current_paragraph_ix); }); - if let Some((paragraph_ix, byte)) = merged_caret { + if committed && let Some((paragraph_ix, byte)) = merged_caret { self.table_cell_block_ix = paragraph_ix; self.table_cell_caret = byte; cx.notify(); } - return true; } - self.table_cell_caret = caret; true } @@ -223,67 +230,68 @@ impl RichTextEditor { } let selection_range = self.equation_source_selection_range(); let caret = self.equation_source_caret; - let mut next_caret = caret; - self.edit_selected_equation(block_ix, cx, |equation| { - let mut source = equation.source.to_string(); - if let Some(range) = selection_range.clone() - && range.start <= range.end - && range.end <= source.len() - && source.is_char_boundary(range.start) - && source.is_char_boundary(range.end) - { - source.replace_range(range.clone(), ""); - next_caret = range.start; - equation.source = source.into(); - equation.version = equation.version.wrapping_add(1); - return; - } + let Some(source) = self.selected_equation_source() else { + return true; + }; + let (range, next_caret) = if let Some(range) = selection_range + && range.start <= range.end + && range.end <= source.len() + && source.is_char_boundary(range.start) + && source.is_char_boundary(range.end) + { + let next_caret = range.start; + (range, next_caret) + } else { let caret = caret.min(source.len()); - if caret > 0 - && source.is_char_boundary(caret) - && let Some((byte, _)) = source[..caret].char_indices().next_back() - { - source.replace_range(byte..caret, ""); - next_caret = byte; - equation.source = source.into(); - equation.version = equation.version.wrapping_add(1); - } - }); - self.equation_source_caret = next_caret; - self.equation_source_anchor = next_caret; + let Some(byte) = (caret > 0 && source.is_char_boundary(caret)) + .then(|| source[..caret].char_indices().next_back().map(|(byte, _)| byte)) + .flatten() + else { + return true; + }; + (byte..caret, byte) + }; + if self.edit_selected_equation_source_range(block_ix, range, "", cx) { + self.equation_source_caret = next_caret; + self.equation_source_anchor = next_caret; + } true } - fn edit_selected_equation(&mut self, block_ix: usize, cx: &mut Context, update: impl FnOnce(&mut EquationBlock)) { - let Some(Block::Equation(equation)) = self.document.blocks.get(block_ix).cloned() else { - return; + /// Equation source edits are identity-addressed range replacements: one + /// `ReplaceEquationSourceRangeIntent` through the write authority. No direct + /// projection mutation, no history record — the intent's patches are the + /// only way the projection changes. + fn edit_selected_equation_source_range(&mut self, block_ix: usize, range: Range, text: &str, cx: &mut Context) -> bool { + let Some(Block::Equation(equation)) = self.document.blocks.get(block_ix) else { + return false; }; - let mut updated = equation.clone(); - update(&mut updated); - if updated == equation { - return; + let source: &str = &equation.source; + if range.start > range.end + || range.end > source.len() + || !source.is_char_boundary(range.start) + || !source.is_char_boundary(range.end) + { + return false; } - let before = Block::Equation(equation); - let after = Block::Equation(updated); - if let Some(block) = Arc::make_mut(&mut self.document.blocks).get_mut(block_ix) { - *block = after.clone(); + if &source[range.clone()] == text { + // No-op replacement; don't emit an intent. + return false; } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { block_ix, before, after }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(block_ix), - }], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); + let Some(equation_id) = self.semantic_block_id(block_ix) else { + tracing::warn!(block_ix, "refusing equation source edit: projection block has no durable id, so no intent can address it"); + return false; + }; + self + .write_intent( + LocalIntent::ReplaceEquationSourceRange(crate::local_intents::ReplaceEquationSourceRangeIntent { + equation: equation_id, + range, + text: text.to_string(), + }), + cx, + ) + .is_some() } pub(super) fn edit_table_cell_paragraph( @@ -293,53 +301,20 @@ impl RichTextEditor { cell_ix: usize, cx: &mut Context, update: impl FnOnce(&mut TableCellParagraph), - ) { - let Some(Block::Table(table)) = self.document.blocks.get(block_ix).cloned() else { - return; - }; - let mut updated = table.clone(); - let Some(cell) = updated - .rows - .get_mut(row_ix) - .and_then(|row| row.cells.get_mut(cell_ix)) - else { - return; - }; - let paragraph_ix = table_cell_paragraph_block_ix(cell, self.table_cell_block_ix).unwrap_or_else(|| { - cell - .blocks - .push(TableCellBlock::Paragraph(default_table_cell_paragraph())); - cell.blocks.len() - 1 - }); - let TableCellBlock::Paragraph(paragraph) = &mut cell.blocks[paragraph_ix] else { - return; - }; - update(paragraph); - if updated == table { - return; - } - updated.version = updated.version.wrapping_add(1); - let before = Block::Table(table); - let after = Block::Table(updated); - if let Some(block) = Arc::make_mut(&mut self.document.blocks).get_mut(block_ix) { - *block = after.clone(); - } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { block_ix, before, after }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(block_ix), - }], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); + ) -> bool { + let preferred_paragraph_ix = self.table_cell_block_ix; + self.edit_table_cell(block_ix, row_ix, cell_ix, cx, |cell| { + let paragraph_ix = table_cell_paragraph_block_ix(cell, preferred_paragraph_ix).unwrap_or_else(|| { + cell + .blocks + .push(TableCellBlock::Paragraph(default_table_cell_paragraph())); + cell.blocks.len() - 1 + }); + let TableCellBlock::Paragraph(paragraph) = &mut cell.blocks[paragraph_ix] else { + return; + }; + update(paragraph); + }) } } diff --git a/crates/gpui-flowtext/src/rich_text/editor/table_helpers.rs b/crates/gpui-flowtext/src/rich_text/editor/table_helpers.rs index 04688097..ed6cdeef 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/table_helpers.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/table_helpers.rs @@ -1,19 +1,47 @@ #[hotpath::measure] -fn default_table_row(columns: usize) -> TableRow { +fn default_table_row(row_id: RowId, column_ids: &[ColumnId]) -> TableRow { TableRow { - cells: (0..columns).map(|_| default_table_cell()).collect(), + id: row_id, + cells: column_ids + .iter() + .map(|&column_id| default_table_cell(row_id, column_id)) + .collect(), } } #[hotpath::measure] -fn default_table_cell() -> TableCell { +fn default_table_cell(row_id: RowId, column_id: ColumnId) -> TableCell { TableCell { + id: CellId::from_coordinate(row_id, column_id), + row_id, + column_id, blocks: vec![TableCellBlock::Paragraph(default_table_cell_paragraph())], row_span: 1, col_span: 1, } } +/// Resolve the durable `(RowId, ColumnId)` for a cell at a positional +/// `(row_ix, cell_ix)` from the id-bearing projection model (§P2b). Used when +/// constructing an id-carrying [`BlockSelection::TableCell`]. Falls back to the +/// zero sentinel when the coordinate cannot be resolved (e.g. an empty table), +/// which the emission paths treat as "no durable identity". +#[hotpath::measure] +fn table_cell_ids_at(document: &DocumentProjection, block_ix: usize, row_ix: usize, cell_ix: usize) -> (RowId, ColumnId) { + let Some(Block::Table(table)) = document.blocks.get(block_ix) else { + return (RowId(0), ColumnId(0)); + }; + let row_id = table.rows.get(row_ix).map(|row| row.id).unwrap_or(RowId(0)); + let column_id = table + .rows + .get(row_ix) + .and_then(|row| row.cells.get(cell_ix)) + .map(|cell| cell.column_id) + .or_else(|| table.columns.get(cell_ix).map(|column| column.id)) + .unwrap_or(ColumnId(0)); + (row_id, column_id) +} + #[hotpath::measure] fn table_column_count(table: &TableBlock) -> usize { table @@ -28,16 +56,16 @@ fn table_column_count(table: &TableBlock) -> usize { }) .max() .unwrap_or(0) - .max(table.column_widths.len()) + .max(table.columns.len()) } #[hotpath::measure] fn fixed_table_column_widths_from_layout(table: &TableBlock, layout: &LaidOutTable) -> Vec { let column_count = table_column_count(table).max(1); let mut widths = vec![120; column_count]; - for (ix, width) in table.column_widths.iter().enumerate() { + for (ix, column) in table.columns.iter().enumerate() { if ix < widths.len() - && let TableColumnWidth::FixedPx(width) = width + && let TableColumnWidth::FixedPx(width) = &column.width { widths[ix] = *width; } @@ -58,7 +86,7 @@ fn fixed_table_column_widths_from_layout(table: &TableBlock, layout: &LaidOutTab let cell_width: f32 = cell_layout.bounds.size.width.into(); let per_column = (cell_width / span as f32).max(32.0).round() as u32; for ix in logical_column_ix..logical_column_ix.saturating_add(span).min(widths.len()) { - if !matches!(table.column_widths.get(ix), Some(TableColumnWidth::FixedPx(_))) { + if !matches!(table.columns.get(ix).map(|column| &column.width), Some(TableColumnWidth::FixedPx(_))) { widths[ix] = per_column; } } @@ -72,7 +100,6 @@ fn default_table_cell_paragraph() -> TableCellParagraph { TableCellParagraph { paragraph: Paragraph { style: ParagraphStyle::Normal, - byte_range: 0..0, runs: Vec::new(), version: 0, }, @@ -129,12 +156,10 @@ pub(super) fn split_table_cell_paragraph_at(cell: &mut TableCell, paragraph_bloc paragraph.text.truncate(byte); let (left_runs, right_runs) = split_runs_at(¶graph.paragraph.runs, byte); paragraph.paragraph.runs = left_runs; - paragraph.paragraph.byte_range = 0..paragraph.text.len(); paragraph.paragraph.version = paragraph.paragraph.version.wrapping_add(1); let new_paragraph = TableCellParagraph { paragraph: Paragraph { style: paragraph.paragraph.style, - byte_range: 0..right_text.len(), runs: right_runs, version: paragraph.paragraph.version, }, @@ -174,7 +199,6 @@ pub(super) fn insert_table_cell_paragraphs_at( paragraph.text.truncate(byte); let (left_runs, right_runs) = split_runs_at(¶graph.paragraph.runs, byte); paragraph.paragraph.runs = left_runs; - paragraph.paragraph.byte_range = 0..paragraph.text.len(); paragraph.paragraph.version = paragraph.paragraph.version.wrapping_add(1); let mut inserted = paragraphs @@ -187,7 +211,6 @@ pub(super) fn insert_table_cell_paragraphs_at( let mut paragraph_runs = std::mem::take(&mut paragraph.paragraph.runs); paragraph_runs.extend(first.paragraph.runs); paragraph.paragraph.runs = merge_adjacent_runs(paragraph_runs); - paragraph.paragraph.byte_range = 0..paragraph.text.len(); paragraph.paragraph.version = paragraph.paragraph.version.wrapping_add(1); let mut caret_block_ix = paragraph_ix; let mut caret_byte = first_insert_start + first.text.len(); @@ -207,7 +230,6 @@ pub(super) fn insert_table_cell_paragraphs_at( let mut caret_runs = std::mem::take(&mut caret_paragraph.paragraph.runs); caret_runs.extend(right_runs); caret_paragraph.paragraph.runs = merge_adjacent_runs(caret_runs); - caret_paragraph.paragraph.byte_range = 0..caret_paragraph.text.len(); caret_paragraph.paragraph.version = caret_paragraph.paragraph.version.wrapping_add(1); Some((caret_block_ix, caret_byte)) } @@ -227,7 +249,6 @@ pub(super) fn merge_table_cell_paragraph_with_previous(cell: &mut TableCell, par let mut runs = std::mem::take(&mut previous.paragraph.runs); runs.extend(current.paragraph.runs); previous.paragraph.runs = merge_adjacent_runs(runs); - previous.paragraph.byte_range = 0..previous.text.len(); previous.paragraph.version = previous.paragraph.version.wrapping_add(1); Some((previous_ix, caret)) } @@ -247,7 +268,6 @@ fn merge_table_cell_paragraph_with_next(cell: &mut TableCell, paragraph_block_ix let mut runs = std::mem::take(&mut current.paragraph.runs); runs.extend(next.paragraph.runs); current.paragraph.runs = merge_adjacent_runs(runs); - current.paragraph.byte_range = 0..current.text.len(); current.paragraph.version = current.paragraph.version.wrapping_add(1); Some((current_ix, caret)) } @@ -278,7 +298,6 @@ fn insert_text_in_table_cell_paragraph(cell_paragraph: &mut TableCellParagraph, left.push(TextRun { len: insert_len, styles }); left.extend(right); cell_paragraph.paragraph.runs = merge_adjacent_runs(left); - cell_paragraph.paragraph.byte_range = 0..cell_paragraph.text.len(); cell_paragraph.paragraph.version = cell_paragraph.paragraph.version.wrapping_add(1); } @@ -314,7 +333,6 @@ fn delete_range_in_table_cell_paragraph(cell_paragraph: &mut TableCellParagraph, } } cell_paragraph.paragraph.runs = merge_adjacent_runs(output); - cell_paragraph.paragraph.byte_range = 0..cell_paragraph.text.len(); cell_paragraph.paragraph.version = cell_paragraph.paragraph.version.wrapping_add(1); } diff --git a/crates/gpui-flowtext/src/rich_text/editor/tables.rs b/crates/gpui-flowtext/src/rich_text/editor/tables.rs index 29fb890c..e85563f5 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/tables.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/tables.rs @@ -3,33 +3,25 @@ impl RichTextEditor { pub fn insert_default_table(&mut self, rows: usize, columns: usize, cx: &mut Context) { let rows = rows.clamp(1, 20); let columns = columns.clamp(1, 12); + let column_ids: Vec = (0..columns).map(|_| ColumnId(uuid::Uuid::new_v4().as_u128())).collect(); let table = TableBlock { rows: (0..rows) - .map(|_| TableRow { - cells: (0..columns) - .map(|_| TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }, - text: String::new(), - })], - row_span: 1, - col_span: 1, - }) - .collect(), - }) + .map(|_| default_table_row(RowId(uuid::Uuid::new_v4().as_u128()), &column_ids)) .collect(), - column_widths: (0..columns) - .map(|_| TableColumnWidth::Fraction(1)) + columns: column_ids + .iter() + .map(|&id| TableColumn { + id, + width: TableColumnWidth::Fraction(1), + }) .collect(), style: TableStyle { header_row: false }, version: 0, }; - self.insert_blocks_after_caret(vec![Block::Table(table)], cx); + // Loro-first (spec §5): the table enters the document as one InsertObject + // intent; the block's durable identity is minted by the write path while + // the row/column ids above stay editor-minted (they ride in the payload). + self.write_insert_object_at_caret(input_block_from_block(&Block::Table(table)), cx); } pub fn insert_row_after_selected_table(&mut self, cx: &mut Context) { @@ -37,21 +29,34 @@ impl RichTextEditor { Some(BlockSelection::TableCell { row_ix, .. }) => Some(row_ix), _ => None, }; - self.edit_selected_table(cx, |table| { - let columns = table - .rows - .iter() - .map(|row| row.cells.len()) - .max() - .unwrap_or(1) - .max(table.column_widths.len()) - .max(1); - let insert_ix = target_row - .map(|row| row + 1) - .unwrap_or(table.rows.len()) - .min(table.rows.len()); - table.rows.insert(insert_ix, default_table_row(columns)); - }); + let Some((block_ix, table_id)) = self.selected_table_identity() else { + return; + }; + let Some(Block::Table(table)) = self.document.blocks.get(block_ix) else { + return; + }; + let insert_ix = target_row + .map(|row| row + 1) + .unwrap_or(table.rows.len()) + .min(table.rows.len()); + // Durable anchor: the row currently before the insertion point in the + // projection (`None` when inserting at the head). + let after_row = insert_ix + .checked_sub(1) + .and_then(|ix| table.rows.get(ix)) + .map(|row| row.id); + let column_ids: Vec = table.columns.iter().map(|column| column.id).collect(); + // The new row's identity is editor-minted (InputTableRow carries its id); + // the runtime materializes the row and returns the projection patches. + let row = default_table_row(RowId(uuid::Uuid::new_v4().as_u128()), &column_ids); + self.write_intent( + LocalIntent::Table(crate::local_intents::TableIntent::InsertRow { + table: table_id, + after_row, + row: input_table_row_from_table_row(&row), + }), + cx, + ); } pub fn delete_last_row_from_selected_table(&mut self, cx: &mut Context) { @@ -59,14 +64,20 @@ impl RichTextEditor { Some(BlockSelection::TableCell { row_ix, .. }) => Some(row_ix), _ => None, }; - self.edit_selected_table(cx, |table| { - if table.rows.len() > 1 { - let row_ix = target_row - .unwrap_or(table.rows.len() - 1) - .min(table.rows.len() - 1); - table.rows.remove(row_ix); - } - }); + let Some((block_ix, table_id)) = self.selected_table_identity() else { + return; + }; + let Some(Block::Table(table)) = self.document.blocks.get(block_ix) else { + return; + }; + if table.rows.len() <= 1 { + return; + } + let row_ix = target_row + .unwrap_or(table.rows.len() - 1) + .min(table.rows.len() - 1); + let row = table.rows[row_ix].id; + self.write_intent(LocalIntent::Table(crate::local_intents::TableIntent::DeleteRow { table: table_id, row }), cx); } pub fn insert_column_after_selected_table(&mut self, cx: &mut Context) { @@ -74,19 +85,31 @@ impl RichTextEditor { Some(BlockSelection::TableCell { cell_ix, .. }) => Some(cell_ix), _ => None, }; - self.edit_selected_table(cx, |table| { - let insert_ix = target_column - .map(|column| column + 1) - .unwrap_or(table.column_widths.len()) - .min(table.column_widths.len()); - table - .column_widths - .insert(insert_ix, TableColumnWidth::Fraction(1)); - for row in &mut table.rows { - let cell_ix = insert_ix.min(row.cells.len()); - row.cells.insert(cell_ix, default_table_cell()); - } - }); + let Some((block_ix, table_id)) = self.selected_table_identity() else { + return; + }; + let Some(Block::Table(table)) = self.document.blocks.get(block_ix) else { + return; + }; + let insert_ix = target_column + .map(|column| column + 1) + .unwrap_or(table.columns.len()) + .min(table.columns.len()); + // Durable anchor: the column currently before the insertion point (`None` + // at the head). The new column's identity AND its empty cells are minted + // by the runtime — the intent carries only the anchor and the width. + let after_column = insert_ix + .checked_sub(1) + .and_then(|ix| table.columns.get(ix)) + .map(|column| column.id); + self.write_intent( + LocalIntent::Table(crate::local_intents::TableIntent::InsertColumn { + table: table_id, + after_column, + width: InputTableColumnWidth::Fraction(1), + }), + cx, + ); } pub fn delete_last_column_from_selected_table(&mut self, cx: &mut Context) { @@ -94,29 +117,23 @@ impl RichTextEditor { Some(BlockSelection::TableCell { cell_ix, .. }) => Some(cell_ix), _ => None, }; - self.edit_selected_table(cx, |table| { - if table.column_widths.len() > 1 { - let column_ix = target_column - .unwrap_or(table.column_widths.len() - 1) - .min(table.column_widths.len() - 1); - table.column_widths.remove(column_ix); - for row in &mut table.rows { - if row.cells.len() > 1 { - let cell_ix = column_ix.min(row.cells.len() - 1); - row.cells.remove(cell_ix); - } - } - } else { - for row in &mut table.rows { - if row.cells.len() > 1 { - let cell_ix = target_column - .unwrap_or(row.cells.len() - 1) - .min(row.cells.len() - 1); - row.cells.remove(cell_ix); - } - } - } - }); + let Some((block_ix, table_id)) = self.selected_table_identity() else { + return; + }; + let Some(Block::Table(table)) = self.document.blocks.get(block_ix) else { + return; + }; + // Deleting the last remaining column is refused, symmetric with rows. (The + // old positional per-row cell trim for degenerate id-less tables has no + // intent form; topology repair is the materializer's job now.) + if table.columns.len() <= 1 { + return; + } + let column_ix = target_column + .unwrap_or(table.columns.len() - 1) + .min(table.columns.len() - 1); + let column = table.columns[column_ix].id; + self.write_intent(LocalIntent::Table(crate::local_intents::TableIntent::DeleteColumn { table: table_id, column }), cx); } pub fn widen_selected_table_column(&mut self, cx: &mut Context) { @@ -132,52 +149,48 @@ impl RichTextEditor { Some(BlockSelection::TableCell { cell_ix, .. }) => cell_ix, _ => return, }; - self.edit_selected_table(cx, |table| { - if target_column >= table.column_widths.len() { - return; - } - let current = match table.column_widths[target_column] { - TableColumnWidth::FixedPx(width) => width as i32, - TableColumnWidth::Fraction(_) | TableColumnWidth::Auto => 120, - }; - table.column_widths[target_column] = TableColumnWidth::FixedPx((current + delta_px).clamp(32, 1600) as u32); - }); - } - - fn edit_selected_table(&mut self, cx: &mut Context, update: impl FnOnce(&mut TableBlock)) { - let Some(block_ix) = self.selected_table_block_ix() else { + let Some((block_ix, table_id)) = self.selected_table_identity() else { return; }; - let Some(Block::Table(table)) = self.document.blocks.get(block_ix).cloned() else { + let Some(Block::Table(table)) = self.document.blocks.get(block_ix) else { return; }; - let mut updated = table.clone(); - update(&mut updated); - if updated == table { + // SetColumnWidth is identity-addressed: resolve the durable column id from + // the projected table's columns at the selected index. + let Some(target) = table.columns.get(target_column) else { + return; + }; + let current = match target.width { + TableColumnWidth::FixedPx(width) => width as i32, + TableColumnWidth::Fraction(_) | TableColumnWidth::Auto => 120, + }; + let width = TableColumnWidth::FixedPx((current + delta_px).clamp(32, 1600) as u32); + if width == target.width { return; } - updated.version = updated.version.wrapping_add(1); - let before = Block::Table(table); - let after = Block::Table(updated.clone()); - if let Some(block) = Arc::make_mut(&mut self.document.blocks).get_mut(block_ix) { - *block = after.clone(); + let column = target.id; + self.write_intent( + LocalIntent::Table(crate::local_intents::TableIntent::SetColumnWidth { + table: table_id, + column, + width: input_table_column_width_from_table_column_width(&width), + }), + cx, + ); + } + + /// Resolve the selected table's `(block_ix, durable BlockId)` pair. A table + /// without a durable identity cannot be addressed by an intent; the edit is + /// refused loudly instead of diverging local state (spec I-2). + fn selected_table_identity(&self) -> Option<(usize, BlockId)> { + let block_ix = self.selected_table_block_ix()?; + match self.semantic_block_id(block_ix) { + Some(table_id) => Some((block_ix, table_id)), + None => { + tracing::warn!(block_ix, "refusing table edit: projection block has no durable id, so no intent can address it"); + None + }, } - let before_generation = self.edit_generation; - let after_generation = self.next_edit_generation; - self.next_edit_generation = self.next_edit_generation.wrapping_add(1); - self.undo_stack.push(EditRecord { - before_selection: self.selection.clone(), - before_generation, - after_selection: self.selection.clone(), - after_generation, - operations: vec![EditOperation::ReplaceBlock { block_ix, before, after }], - canonical_operations: vec![CanonicalOperation::ReplaceBlock { - block: self.identity_map.block_id(block_ix), - }], - }); - self.redo_stack.clear(); - self.invalidate_document_layout_caches(); - self.mark_document_changed(after_generation, cx); } fn selected_table_block_ix(&self) -> Option { diff --git a/crates/gpui-flowtext/src/rich_text/editor/traits.rs b/crates/gpui-flowtext/src/rich_text/editor/traits.rs index d093ea7b..80390957 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/traits.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/traits.rs @@ -136,8 +136,8 @@ impl Render for RichTextEditor { .clone() .map(|preview| render_drop_preview(preview, editor.invisibility_mode, cx).into_any_element()) .unwrap_or_else(|| EmptyVirtualItemElement.into_any_element()), - RenderVirtualItem::Document(VirtualItem::HiddenBlock { .. }) => EmptyVirtualItemElement.into_any_element(), - RenderVirtualItem::Document(VirtualItem::ParagraphChunk { + RenderVirtualItem::DocumentProjection(VirtualItem::HiddenBlock { .. }) => EmptyVirtualItemElement.into_any_element(), + RenderVirtualItem::DocumentProjection(VirtualItem::ParagraphChunk { paragraph_ix, chunk_ix, .. @@ -150,7 +150,7 @@ impl Render for RichTextEditor { layout: WordElementLayout::default(), } .into_any_element(), - RenderVirtualItem::Document(VirtualItem::ParagraphRemainder { paragraph_ix, .. }) => { + RenderVirtualItem::DocumentProjection(VirtualItem::ParagraphRemainder { paragraph_ix, .. }) => { let width = editor.current_layout_width(); let chunk_ix = editor.materialize_paragraph_remainder_for_render(paragraph_ix, width, window, cx); if let Some(chunk_ix) = chunk_ix { @@ -167,7 +167,7 @@ impl Render for RichTextEditor { EmptyVirtualItemElement.into_any_element() } }, - RenderVirtualItem::Document(VirtualItem::StructuralBlock { block_ix }) => { + RenderVirtualItem::DocumentProjection(VirtualItem::StructuralBlock { block_ix }) => { let editor_entity = cx.entity(); let selection = match editor.document.blocks.get(block_ix) { Some(Block::Image(_)) => Some(BlockSelection::Image(block_ix)), diff --git a/crates/gpui-flowtext/src/rich_text/editor/virtual_helpers.rs b/crates/gpui-flowtext/src/rich_text/editor/virtual_helpers.rs index bdedb19e..50b636f7 100644 --- a/crates/gpui-flowtext/src/rich_text/editor/virtual_helpers.rs +++ b/crates/gpui-flowtext/src/rich_text/editor/virtual_helpers.rs @@ -5,9 +5,7 @@ fn item_lookup_for_virtual_items(items: &[VirtualItem], paragraph_count: usize) for (item_ix, item) in items.iter().enumerate() { match item { - VirtualItem::ParagraphChunk { - paragraph_ix, .. - } => { + VirtualItem::ParagraphChunk { paragraph_ix, .. } => { if let Some(range) = paragraph_chunk_item_ranges.get_mut(*paragraph_ix) { if range.start == range.end { *range = item_ix..item_ix + 1; @@ -47,12 +45,14 @@ fn patch_item_lookup_for_paragraph_range( } } - for (relative_item_ix, item) in items.get(replace_start..replace_start + new_len)?.iter().enumerate() { + for (relative_item_ix, item) in items + .get(replace_start..replace_start + new_len)? + .iter() + .enumerate() + { let item_ix = replace_start + relative_item_ix; match item { - VirtualItem::ParagraphChunk { - paragraph_ix, .. - } if range.contains(paragraph_ix) => { + VirtualItem::ParagraphChunk { paragraph_ix, .. } if range.contains(paragraph_ix) => { if let Some(chunk_range) = paragraph_chunk_item_ranges.get_mut(*paragraph_ix) { if chunk_range.start == chunk_range.end { *chunk_range = item_ix..item_ix + 1; @@ -102,7 +102,7 @@ fn expand_paragraph_range(range: Range, paragraph_count: usize, padding: } #[hotpath::measure] -fn byte_at_ratio_in_paragraph(document: &Document, paragraph_ix: usize, start_byte: usize, end_byte: usize, ratio: f32) -> usize { +fn byte_at_ratio_in_paragraph(document: &DocumentProjection, paragraph_ix: usize, start_byte: usize, end_byte: usize, ratio: f32) -> usize { let Some(paragraph) = document.paragraphs.get(paragraph_ix) else { return 0; }; @@ -117,15 +117,16 @@ fn byte_at_ratio_in_paragraph(document: &Document, paragraph_ix: usize, start_by } #[hotpath::measure] -fn detach_document_for_background_write(document: &Document) -> Document { - Document { +fn detach_document_for_background_write(document: &DocumentProjection) -> DocumentProjection { + DocumentProjection { + frontier: document.frontier.clone(), text: document.text.clone(), - paragraphs: Arc::new(document.paragraphs.as_ref().clone()), - blocks: Arc::new(document.blocks.as_ref().clone()), + paragraphs: document.paragraphs.clone(), + blocks: document.blocks.clone(), assets: document.assets.clone(), ids: document.ids.clone(), sections: Arc::new(document.sections.as_ref().clone()), - offset_index: document.offset_index.clone(), + outline: Arc::new(document.outline.as_ref().clone()), theme: document.theme.clone(), } } @@ -138,4 +139,3 @@ fn floor_char_boundary(text: &str, mut byte: usize) -> usize { } byte } - diff --git a/crates/gpui-flowtext/src/rich_text/element.rs b/crates/gpui-flowtext/src/rich_text/element.rs index c719585b..f9af322e 100644 --- a/crates/gpui-flowtext/src/rich_text/element.rs +++ b/crates/gpui-flowtext/src/rich_text/element.rs @@ -1,21 +1,21 @@ use std::{cell::RefCell, rc::Rc}; use gpui::{ - App, AvailableSpace, Background, Bounds, Element, ElementId, Entity, GlobalElementId, InspectorElementId, IntoElement, LayoutId, Pixels, - Style, Window, fill, px, relative, + App, AvailableSpace, Background, Bounds, Element, ElementId, ElementInputHandler, Entity, GlobalElementId, InspectorElementId, IntoElement, + LayoutId, Pixels, Style, Window, fill, px, relative, }; use super::*; pub struct RichTextDocumentElement { - pub(super) document: Document, + pub(super) document: DocumentProjection, pub(super) layout: WordElementLayout, pub(super) invisibility_mode: bool, } #[hotpath::measure_all] impl RichTextDocumentElement { - pub fn new(document: Document) -> Self { + pub fn new(document: DocumentProjection) -> Self { Self { document, layout: WordElementLayout::default(), @@ -161,7 +161,23 @@ impl Element for RichTextDocumentElement { cx: &mut App, ) { if let Some((layout, bounds)) = self.layout.positioned() { - paint_layout(layout.as_ref(), bounds, None, None, false, px(1.0), &[], &[], None, window, cx); + paint_layout( + layout.as_ref(), + bounds, + None, + None, + false, + px(1.0), + None, + gpui::black(), + &[], + &[], + &[], + &[], + None, + window, + cx, + ); } } } @@ -239,10 +255,24 @@ impl Element for VirtualParagraphChunkElement { window: &mut Window, cx: &mut App, ) { - let (selection, drag_selection, caret_offset, caret_width, external_carets, search_highlights, active_search_highlight) = { + let ( + selection, + drag_selection, + caret_offset, + caret_width, + caret_color_rgb, + default_caret_color, + external_carets, + external_selections, + annotation_selections, + search_highlights, + active_search_highlight, + ) = { let editor = self.editor.read(cx); let drag_selection = editor.drag_source_selection(); let external_carets = editor.external_carets_for_paragraph(self.paragraph_ix); + let external_selections = editor.external_selections_for_paragraph(self.paragraph_ix); + let annotation_selections = editor.annotation_selections_for_paragraph(self.paragraph_ix); ( editor.selection.clone(), drag_selection, @@ -253,12 +283,21 @@ impl Element for VirtualParagraphChunkElement { && editor.focus_handle.is_focused(window)) .then_some(editor.selection.head), editor.caret_paint_width(), + editor.local_caret_color_rgb(), + // The caret's fallback color = the theme's default text color, so it + // contrasts with the background (a hardcoded black caret is invisible on a + // dark theme). + editor.document.theme.default_text_color, external_carets, + external_selections, + annotation_selections, editor.search_highlights.clone(), editor.active_search_highlight, ) }; if let Some((layout, bounds)) = self.layout.positioned() { + let focus_handle = self.editor.read(cx).focus_handle.clone(); + window.handle_input(&focus_handle, ElementInputHandler::new(bounds, self.editor.clone()), cx); if self.chunk_ix == 0 { let collapse_state = self .editor @@ -298,7 +337,11 @@ impl Element for VirtualParagraphChunkElement { drag_selection.as_ref(), show_caret, caret_width, + caret_color_rgb, + default_caret_color, &external_carets, + &external_selections, + &annotation_selections, &search_highlights, active_search_highlight, window, @@ -489,7 +532,7 @@ fn packed_element_pair(first: usize, second: usize) -> u64 { #[hotpath::measure] pub(super) fn request_word_layout( - document: Document, + document: DocumentProjection, layout_cell: WordElementLayout, invisibility_mode: bool, window: &mut Window, diff --git a/crates/gpui-flowtext/src/rich_text/invisibility.rs b/crates/gpui-flowtext/src/rich_text/invisibility.rs index f612f7e5..429080dd 100644 --- a/crates/gpui-flowtext/src/rich_text/invisibility.rs +++ b/crates/gpui-flowtext/src/rich_text/invisibility.rs @@ -37,7 +37,7 @@ pub(super) struct VisibilityIndex { #[hotpath::measure_all] impl VisibilityIndex { - pub(super) fn build(document: &Document, invisibility_mode: bool) -> Self { + pub(super) fn build(document: &DocumentProjection, invisibility_mode: bool) -> Self { let mut visible_blocks = Vec::with_capacity(document.blocks.len()); for block in document.blocks.iter() { @@ -60,7 +60,7 @@ impl VisibilityIndex { } #[hotpath::measure] -pub(super) fn paragraph_is_visible(document: &Document, paragraph: &Paragraph) -> bool { +pub(super) fn paragraph_is_visible(document: &DocumentProjection, paragraph: &Paragraph) -> bool { paragraph_is_visible_for_theme(&document.theme, paragraph) } @@ -86,7 +86,7 @@ pub(super) fn paragraph_is_visible_for_theme(theme: &DocumentTheme, paragraph: & pub(super) const INVISIBILITY_PROJECTED_VERSION_OFFSET: u64 = 0x9E37_79B9_7F4A_7C15; #[hotpath::measure] -pub(super) fn invisibility_projected_document(document: &Document, paragraph_ix: usize) -> Option { +pub(super) fn invisibility_projected_document(document: &DocumentProjection, paragraph_ix: usize) -> Option { let paragraph = document.paragraphs.get(paragraph_ix)?; if !matches!(paragraph.style, ParagraphStyle::Normal) { return None; @@ -96,7 +96,6 @@ pub(super) fn invisibility_projected_document(document: &Document, paragraph_ix: let paragraph = Paragraph { style: ParagraphStyle::Normal, - byte_range: 0..text.len(), runs, // Give the projected paragraph a distinct cache key from the source // paragraph so invisible-mode layout cannot reuse a full-text layout. @@ -104,15 +103,17 @@ pub(super) fn invisibility_projected_document(document: &Document, paragraph_ix: .version .wrapping_add(INVISIBILITY_PROJECTED_VERSION_OFFSET), }; - let paragraphs = Arc::new(vec![paragraph.clone()]); - let mut projected = Document { + let paragraphs = vec![paragraph.clone()]; + let paragraph_count = paragraphs.len(); + let mut projected = DocumentProjection { + frontier: document.frontier.clone(), text: Rope::from(text), - blocks: Arc::new(vec![Block::Paragraph(paragraph)]), - paragraphs: paragraphs.clone(), + blocks: BlockSeq::from_vec(vec![Block::Paragraph(paragraph)]), + paragraphs: ParagraphSeq::from_vec(paragraphs), assets: document.assets.clone(), - ids: document_ids_for_shape(paragraphs.len(), 1), + ids: document_ids_for_shape(paragraph_count, 1), sections: Arc::new(Vec::new()), - offset_index: ParagraphOffsetIndex::new(¶graphs), + outline: Arc::new(Vec::new()), theme: document.theme.clone(), }; rebuild_document_sections(&mut projected); @@ -120,7 +121,7 @@ pub(super) fn invisibility_projected_document(document: &Document, paragraph_ix: } #[hotpath::measure] -pub(super) fn run_is_visible(document: &Document, styles: RunStyles) -> bool { +pub(super) fn run_is_visible(document: &DocumentProjection, styles: RunStyles) -> bool { run_is_visible_for_theme(&document.theme, styles) } @@ -158,7 +159,7 @@ pub(super) fn decode_remainder_item_ix(encoded: u32) -> Option { } #[hotpath::measure] -pub(super) fn projected_visible_paragraph_text_and_runs(document: &Document, paragraph_ix: usize) -> Option<(String, Vec)> { +pub(super) fn projected_visible_paragraph_text_and_runs(document: &DocumentProjection, paragraph_ix: usize) -> Option<(String, Vec)> { let paragraph = document.paragraphs.get(paragraph_ix)?; let paragraph_start = paragraph_byte_range(document, paragraph_ix).start; let paragraph_len = paragraph_text_len(paragraph); diff --git a/crates/gpui-flowtext/src/rich_text/layout/block_layout.rs b/crates/gpui-flowtext/src/rich_text/layout/block_layout.rs index 9d45f715..3266073d 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/block_layout.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/block_layout.rs @@ -1,6 +1,6 @@ #[hotpath::measure] pub(super) fn build_layout( - document: &Document, + document: &DocumentProjection, width: Pixels, previous_layout: Option<&LayoutState>, window: &mut Window, @@ -53,7 +53,7 @@ pub(super) fn build_layout( #[hotpath::measure] pub(super) fn build_layout_with_visibility( - document: &Document, + document: &DocumentProjection, width: Pixels, previous_layout: Option<&LayoutState>, invisibility_mode: bool, @@ -133,7 +133,7 @@ pub(super) fn build_layout_with_visibility( #[hotpath::measure] pub(super) fn build_single_paragraph_layout_with_visibility( - document: &Document, + document: &DocumentProjection, paragraph_ix: usize, width: Pixels, previous_layout: Option<&LayoutState>, @@ -207,7 +207,7 @@ pub(super) fn build_single_paragraph_layout_with_visibility( #[allow(dead_code, reason = "Block layout helper is retained for incremental object-layout work.")] #[hotpath::measure] pub(super) fn build_structural_block_layout( - document: &Document, + document: &DocumentProjection, width: Pixels, previous_layout: Option<&LayoutState>, window: &mut Window, @@ -266,14 +266,22 @@ pub(super) fn build_structural_block_layout( } #[hotpath::measure] -fn structural_block_bounds(document: &Document, width: Pixels, y: Pixels, height: Pixels) -> Bounds { +fn structural_block_bounds(document: &DocumentProjection, width: Pixels, y: Pixels, height: Pixels) -> Bounds { let left = document.theme.pageless_inset_x; let block_width = (width - document.theme.pageless_inset_x * 2.0).max(px(1.0)); Bounds::new(point(left, y), size(block_width, height.max(px(1.0)))) } #[hotpath::measure] -fn image_placeholder_height(document: &Document, image: &ImageBlock, width: Pixels) -> Pixels { +fn image_placeholder_height(document: &DocumentProjection, image: &ImageBlock, width: Pixels) -> Pixels { + if document + .assets + .assets + .get(&image.asset_id) + .is_some_and(AssetRecord::is_loading_placeholder) + { + return px(IMAGE_LOADING_PLACEHOLDER_HEIGHT_PX); + } let available_width = (width - document.theme.pageless_inset_x * 2.0).max(px(1.0)); let intrinsic = image_intrinsic_size(document, image); match image.sizing { @@ -286,15 +294,12 @@ fn image_placeholder_height(document: &Document, image: &ImageBlock, width: Pixe } } -#[cfg(test)] #[hotpath::measure] -pub(super) fn image_layout_height_for_test(document: &Document, image: &ImageBlock, width: Pixels) -> Pixels { - image_placeholder_height(document, image, width) -} - -#[hotpath::measure] -fn image_intrinsic_size(document: &Document, image: &ImageBlock) -> Option<(Pixels, Pixels)> { +fn image_intrinsic_size(document: &DocumentProjection, image: &ImageBlock) -> Option<(Pixels, Pixels)> { let asset = document.assets.assets.get(&image.asset_id)?; + if asset.is_loading_placeholder() { + return None; + } let size = imagesize::blob_size(asset.bytes.as_ref()).ok()?; if size.width == 0 || size.height == 0 { return None; @@ -315,7 +320,7 @@ fn image_height_for_width(intrinsic: Option<(Pixels, Pixels)>, width: Pixels) -> } #[hotpath::measure] -fn equation_placeholder_height(document: &Document, equation: &EquationBlock) -> Pixels { +fn equation_placeholder_height(document: &DocumentProjection, equation: &EquationBlock) -> Pixels { match equation.display { EquationDisplay::Display => (document.theme.body_font_size * document.theme.zoom_factor.max(0.01) * 3.7).max(px(72.0)), EquationDisplay::InlineLikeParagraph => (document.theme.body_font_size * document.theme.zoom_factor.max(0.01) * 2.75).max(px(56.0)), @@ -324,7 +329,7 @@ fn equation_placeholder_height(document: &Document, equation: &EquationBlock) -> #[hotpath::measure] pub(super) fn layout_structural_block_at( - document: &Document, + document: &DocumentProjection, block_ix: usize, width: Pixels, y: Pixels, diff --git a/crates/gpui-flowtext/src/rich_text/layout/decorations.rs b/crates/gpui-flowtext/src/rich_text/layout/decorations.rs index 8f2f5383..c35c2bf0 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/decorations.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/decorations.rs @@ -1,5 +1,5 @@ #[hotpath::measure] -pub(super) fn rects_for_line(document: &Document, line: &LaidOutLine) -> Vec { +pub(super) fn rects_for_line(document: &DocumentProjection, line: &LaidOutLine) -> Vec { let mut backgrounds = Vec::new(); let mut borders = Vec::new(); let text_top = line.baseline_y() - line.ascent; @@ -148,8 +148,10 @@ pub(super) fn push_box_rules(rects: &mut Vec, bounds: Bounds, t } #[hotpath::measure] -pub(super) fn underlines_for_line(document: &Document, line: &LaidOutLine, cx: &mut App) -> Vec { - let mut underlines = Vec::with_capacity(line.segments.len().saturating_mul(2)); +pub(super) fn underlines_for_line(document: &DocumentProjection, line: &LaidOutLine, cx: &mut App) -> Vec { + // §perf: the common prose line has no underlines, so start empty (no allocation + // until the first push) rather than eagerly allocating a buffer that stays empty. + let mut underlines = Vec::new(); let baseline = line.baseline_y(); for segment in &line.segments { match segment.format.underline { @@ -188,9 +190,10 @@ pub(super) fn underlines_for_line(document: &Document, line: &LaidOutLine, cx: & } #[hotpath::measure] -pub(super) fn strikethroughs_for_line(document: &Document, line: &LaidOutLine) -> Vec { +pub(super) fn strikethroughs_for_line(document: &DocumentProjection, line: &LaidOutLine) -> Vec { let baseline = line.baseline_y(); - let mut decorations = Vec::with_capacity(line.segments.len()); + // §perf: most lines have no strikethrough; defer allocation until the first push. + let mut decorations = Vec::new(); for segment in &line.segments { if segment.format.strikethrough { let thickness = document.theme.underline_rule_thickness.max(px(1.0)); @@ -232,6 +235,22 @@ pub(super) fn merge_inline_decorations(decorations: Vec) -> Vec bool { } #[hotpath::measure] -pub(super) fn single_underline_metrics_for_segment(segment: &LaidOutSegment, document: &Document, cx: &mut App) -> (Pixels, Pixels) { +pub(super) fn single_underline_metrics_for_segment(segment: &LaidOutSegment, document: &DocumentProjection, cx: &mut App) -> (Pixels, Pixels) { // GPUI exposes glyph bounds in font coordinates. For Calibri, the // underscore bbox is below the baseline. The origin is the lower // edge of the glyph box on this metric path; Word positions an @@ -297,7 +316,7 @@ pub(super) fn single_underline_metrics_for_segment(segment: &LaidOutSegment, doc } #[hotpath::measure] -pub(super) fn double_underline_metrics_for_segment(document: &Document) -> (Pixels, Pixels) { +pub(super) fn double_underline_metrics_for_segment(document: &DocumentProjection) -> (Pixels, Pixels) { (document.theme.double_underline_top_from_baseline, document.theme.underline_rule_thickness) } diff --git a/crates/gpui-flowtext/src/rich_text/layout/estimates.rs b/crates/gpui-flowtext/src/rich_text/layout/estimates.rs index 6adfd675..0a1f6664 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/estimates.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/estimates.rs @@ -1,43 +1,88 @@ +/// §perf-heaven T5: fraction of a line's raw char capacity actually filled once +/// word-wrap (break at word boundaries, not mid-word) is accounted for. Tuned +/// against the estimate-accuracy net so the estimate approaches the exact height +/// without over-shooting. +const WORD_WRAP_FILL: f32 = 0.82; + #[hotpath::measure] -pub(super) fn estimate_paragraph_item_height(document: &Document, paragraph_ix: usize, width: Pixels) -> Pixels { +pub(super) fn estimate_paragraph_item_height(document: &DocumentProjection, paragraph_ix: usize, width: Pixels) -> Pixels { estimate_paragraph_item_height_with_visibility(document, paragraph_ix, width, false) } #[hotpath::measure] pub(super) fn estimate_paragraph_item_height_with_visibility( - document: &Document, + document: &DocumentProjection, paragraph_ix: usize, width: Pixels, invisibility_mode: bool, ) -> Pixels { - if invisibility_mode - && document + if invisibility_mode { + if document .paragraphs .get(paragraph_ix) .is_some_and(|paragraph| !paragraph_is_visible(document, paragraph)) - { - return px(0.0); + { + return px(0.0); + } + // §act-ten A10.9: estimate the projected paragraph from its visible + // text/runs DIRECTLY. The old path assembled a full throwaway + // `DocumentProjection` (rope + paragraph/block trees + assets clone + + // sections rebuild) per paragraph per estimate — 30k calls / 320 MB in + // hotpath1. Same inputs, same numbers: the projected paragraph is always + // `Normal`-styled and its text length equals the sum of projected run + // lengths. + if document + .paragraphs + .get(paragraph_ix) + .is_some_and(|paragraph| matches!(paragraph.style, ParagraphStyle::Normal)) + && let Some((text, _runs)) = projected_visible_paragraph_text_and_runs(document, paragraph_ix) + { + return estimate_item_height_from_parts( + document, + paragraph_ix, + ParagraphStyle::Normal, + text.len(), + text.matches(SOFT_LINE_BREAK).count(), + width, + ); + } + // No projection (non-Normal visible style, or no non-empty visible runs): + // fall through to the full-paragraph estimate, as before. } - let projected_document = invisibility_mode - .then(|| invisibility_projected_document(document, paragraph_ix)) - .flatten(); - let estimate_document = projected_document.as_ref().unwrap_or(document); - let estimate_paragraph_ix = if projected_document.is_some() { 0 } else { paragraph_ix }; - let paragraph = &estimate_document.paragraphs[estimate_paragraph_ix]; - let p_format = paragraph_format(estimate_document, paragraph.style); + let paragraph = &document.paragraphs[paragraph_ix]; + estimate_item_height_from_parts( + document, + paragraph_ix, + paragraph.style, + paragraph_text_len(paragraph), + paragraph_char_count(document, paragraph_ix, SOFT_LINE_BREAK), + width, + ) +} + +/// Shared core of the §perf-heaven T5 line-count height heuristic, over +/// pre-extracted paragraph facts (style, text byte length, forced-break count). +fn estimate_item_height_from_parts( + document: &DocumentProjection, + paragraph_ix: usize, + style: ParagraphStyle, + text_len: usize, + forced_line_count: usize, + width: Pixels, +) -> Pixels { + let p_format = paragraph_format(document, style); let border = p_format.border; let border_inset = border.map_or(px(0.0), |border| border.width + border.space_x); let content_top = border.map_or(px(0.0), |border| border.width + border.space_y); - let content_width = (width - estimate_document.theme.pageless_inset_x * 2.0 - border_inset * 2.0).max(px(1.0)); + let content_width = (width - document.theme.pageless_inset_x * 2.0 - border_inset * 2.0).max(px(1.0)); let avg_char_width = (p_format.font_size * 0.52).max(px(1.0)); - let chars_per_line = ((content_width / avg_char_width).floor() as usize).max(1); - let text_len = paragraph_text_len(paragraph); - let forced_line_count = paragraph_char_count(estimate_document, estimate_paragraph_ix, SOFT_LINE_BREAK); + // §perf-heaven T5: word-wrap allowance — see the prep variant. + let chars_per_line = ((content_width / avg_char_width * WORD_WRAP_FILL).floor() as usize).max(1); let estimated_lines = (text_len / chars_per_line) .saturating_add(1) .saturating_add(forced_line_count) .max(1); - let line_gap = p_format.font_size * estimate_document.theme.line_gap_fraction; + let line_gap = p_format.font_size * document.theme.line_gap_fraction; let line_height = (p_format.font_size + line_gap) * p_format.line_spacing; let mut height = p_format.spacing_before + content_top + line_height * estimated_lines as f32 + content_top + p_format.spacing_after; if paragraph_ix == 0 { @@ -50,7 +95,16 @@ pub(super) fn estimate_paragraph_item_height_with_visibility( } #[hotpath::measure] -pub(super) fn estimate_paragraph_prep_item_height(document: &Document, prep: &ParagraphPrep, width: Pixels) -> Pixels { +// §act-eleven A11.7: takes the CURRENT paragraph index — the prep's stored +// `paragraph_ix` is its build-time position, which a structural shift above +// invalidated positionally but not contentually (the whole point of dropping +// the positional validity check is that the shifted tail keeps its prep). +pub(super) fn estimate_paragraph_prep_item_height( + document: &DocumentProjection, + prep: &ParagraphPrep, + paragraph_ix: usize, + width: Pixels, +) -> Pixels { if !prep.visible { return px(0.0); } @@ -60,8 +114,19 @@ pub(super) fn estimate_paragraph_prep_item_height(document: &Document, prep: &Pa let content_top = border.map_or(px(0.0), |border| border.width + border.space_y); let content_width = (width - document.theme.pageless_inset_x * 2.0 - border_inset * 2.0).max(px(1.0)); let avg_char_width = (p_format.font_size * 0.52).max(px(1.0)); - let chars_per_line = ((content_width / avg_char_width).floor() as usize).max(1); - let text_len = prep.paragraph_text.len(); + // §perf-heaven T5: word-wrap breaks at word boundaries, so a line holds fewer + // chars than its raw char capacity (the last word spills to the next line). + // Applying `WORD_WRAP_FILL` lifts the estimated line count toward the exact + // one, cutting the estimate's height under-shoot (guarded by the + // estimate-accuracy net — it must not tip into over-shoot). + let chars_per_line = ((content_width / avg_char_width * WORD_WRAP_FILL).floor() as usize).max(1); + // §perf-heaven T7.17: `chars_per_line` is a CHARACTER capacity, so measure the + // text in characters, not bytes. `prep.paragraph_text` is already materialized + // here, so the char count is cheap; using the byte length made a multibyte + // (e.g. CJK) paragraph estimate up to 3–4× too many lines, which over-shoots + // the exact height (guarded by the estimate-accuracy net, now including a + // non-ASCII fixture). + let text_len = prep.paragraph_text.chars().count(); let forced_line_count = prep.paragraph_text.matches(SOFT_LINE_BREAK).count(); let estimated_lines = (text_len / chars_per_line) .saturating_add(1) @@ -70,17 +135,17 @@ pub(super) fn estimate_paragraph_prep_item_height(document: &Document, prep: &Pa let line_gap = p_format.font_size * document.theme.line_gap_fraction; let line_height = (p_format.font_size + line_gap) * p_format.line_spacing; let mut height = p_format.spacing_before + content_top + line_height * estimated_lines as f32 + content_top + p_format.spacing_after; - if prep.paragraph_ix == 0 { + if paragraph_ix == 0 { height += document.theme.pageless_inset_top; } - if prep.paragraph_ix + 1 == document.paragraphs.len() { + if paragraph_ix + 1 == document.paragraphs.len() { height += document.theme.pageless_inset_bottom; } height.max(line_height) } #[hotpath::measure] -pub(super) fn estimate_structural_block_item_height(document: &Document, block_ix: usize, width: Pixels) -> Pixels { +pub(super) fn estimate_structural_block_item_height(document: &DocumentProjection, block_ix: usize, width: Pixels) -> Pixels { let Some(block) = document.blocks.get(block_ix) else { return px(1.0); }; @@ -102,10 +167,10 @@ pub(super) fn estimate_structural_block_item_height(document: &Document, block_i } #[hotpath::measure] -fn table_placeholder_height(document: &Document, table: &TableBlock, width: Pixels) -> Pixels { +fn table_placeholder_height(document: &DocumentProjection, table: &TableBlock, width: Pixels) -> Pixels { let line_height = (document.theme.body_font_size * document.theme.zoom_factor.max(0.01) * document.theme.line_spacing).max(px(16.0)); let column_count = table - .column_widths + .columns .len() .max( table @@ -152,11 +217,11 @@ fn table_placeholder_height(document: &Document, table: &TableBlock, width: Pixe } #[hotpath::measure] -fn layout_table_block_without_text(document: &Document, table: &TableBlock, width: Pixels, y: Pixels) -> LaidOutTable { +fn layout_table_block_without_text(document: &DocumentProjection, table: &TableBlock, width: Pixels, y: Pixels) -> LaidOutTable { let table_left = document.theme.pageless_inset_x; let table_width = (width - document.theme.pageless_inset_x * 2.0).max(px(1.0)); let column_count = table - .column_widths + .columns .len() .max( table diff --git a/crates/gpui-flowtext/src/rich_text/layout/format.rs b/crates/gpui-flowtext/src/rich_text/layout/format.rs index 9f09fb48..09983c14 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/format.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/format.rs @@ -1,5 +1,5 @@ #[hotpath::measure] -pub(super) fn paragraph_format(document: &Document, style: ParagraphStyle) -> EffectiveParagraphFormat { +pub(super) fn paragraph_format(document: &DocumentProjection, style: ParagraphStyle) -> EffectiveParagraphFormat { let theme = &document.theme; let zoom = theme.zoom_factor.max(0.01); let normal = EffectiveParagraphFormat { @@ -46,7 +46,7 @@ pub(super) fn paragraph_format(document: &Document, style: ParagraphStyle) -> Ef } #[hotpath::measure] -pub(super) fn run_format(document: &Document, paragraph: &EffectiveParagraphFormat, styles: RunStyles) -> EffectiveRunFormat { +pub(super) fn run_format(document: &DocumentProjection, paragraph: &EffectiveParagraphFormat, styles: RunStyles) -> EffectiveRunFormat { let theme = &document.theme; let zoom = theme.zoom_factor.max(0.01); let mut format = EffectiveRunFormat { diff --git a/crates/gpui-flowtext/src/rich_text/layout/hit_test.rs b/crates/gpui-flowtext/src/rich_text/layout/hit_test.rs index c2412724..be193ffd 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/hit_test.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/hit_test.rs @@ -44,16 +44,45 @@ pub(super) fn first_line_with_bottom_at_or_after(lines: &[LaidOutLine], y: Pixel } #[hotpath::measure] -pub(super) fn caret_bounds(layout: &LayoutState, offset: DocumentOffset, origin: Point) -> Option> { +pub(super) fn caret_bounds(layout: &LayoutState, offset: DocumentOffset, gravity: VisualGravity, origin: Point) -> Option> { // Use locate_line so the caret is drawn on the same visual line that - // Up/Down/Home/End navigate from — in particular the wrap-seam bias - // (byte at end of line k == start of line k+1 → paint on k+1) must be - // identical in both paths, otherwise the caret appears at the wrong - // position after the cursor reaches a soft-wrap boundary. - let (p_ix, l_ix) = locate_line(layout, offset)?; + // Up/Down/Home/End navigate from — in particular the wrap-seam bias must be + // identical in both paths, otherwise the caret appears at the wrong position + // after the cursor reaches a soft-wrap boundary. The seam choice is now + // driven by §16 visual gravity (Upstream → end of upper line, otherwise → + // start of lower line). + let (p_ix, l_ix) = locate_line(layout, offset, gravity)?; let line = layout.paragraphs[p_ix].lines.get(l_ix)?; let x = x_for_byte(line, offset.byte); - Some(Bounds::new(origin + line.origin + point(x, px(0.0)), size(px(1.0), line.line_height))) + let bounds = Bounds::new(origin + line.origin + point(x, px(0.0)), size(px(1.0), line.line_height)); + // Fidelity: round-trip the caret through offset -> x -> hit-test. `x` is + // line-local (same basis as `hit_test_x`), so for an interior caret offset the + // recovered byte must equal the requested byte; a mismatch means the byte->x + // and x->byte maps disagree (the painted caret and mouse hit-testing would + // land on different positions). Seam/line-edge offsets legitimately snap, so + // the assertion is limited to strictly interior offsets to avoid noise. + if flowstate_fidelity::enabled() { + let recovered = line.hit_test_x(x); + flowstate_fidelity::event(flowstate_fidelity::FidelityClass::Caret, "resolve", || { + format!( + "offset={offset:?} gravity={gravity:?} paragraph_index={} line={l_ix} line_bytes={}..{} x={x:?} recovered_byte={recovered} rect={bounds:?}", + layout.paragraphs[p_ix].index, line.start_byte, line.end_byte, + ) + }); + let interior = offset.byte > line.start_byte && offset.byte < line.end_byte; + flowstate_fidelity::check( + recovered == offset.byte || !interior, + flowstate_fidelity::FidelityClass::Caret, + "hit-test-roundtrip", + || { + format!( + "offset_byte={} x={x:?} recovered_byte={recovered} line_bytes={}..{} paragraph_index={}", + offset.byte, line.start_byte, line.end_byte, layout.paragraphs[p_ix].index, + ) + }, + ); + } + Some(bounds) } #[hotpath::measure] @@ -104,11 +133,14 @@ fn line_ix_for_byte(paragraph: &LaidOutParagraph, byte: usize) -> Option // Locate the `LaidOutLine` containing the given offset. Returns // `(paragraph_layout_index, line_index)`. When the byte sits exactly on a -// soft-wrap seam (== end_byte of line k and start_byte of line k+1), we bias -// to the next line — matching Word's "caret-at-start-of-next-line" -// convention. This is exactly the disambiguation called out in the plan. +// soft-wrap seam (== end_byte of line k and start_byte of line k+1), the §16 +// visual `gravity` disambiguates which visual line the caret belongs to: +// `Upstream` keeps it on the upper line (end of line k); `Downstream`/`Neutral` +// bias to the lower line (start of line k+1), matching the historical +// "caret-at-start-of-next-line" convention. This is exactly the disambiguation +// called out in the plan. #[hotpath::measure] -pub(super) fn locate_line(layout: &LayoutState, off: DocumentOffset) -> Option<(usize, usize)> { +pub(super) fn locate_line(layout: &LayoutState, off: DocumentOffset, gravity: VisualGravity) -> Option<(usize, usize)> { let p_ix = paragraph_layout_index_for_offset(layout, off)?; let para = &layout.paragraphs[p_ix]; let mut low = 0; @@ -125,8 +157,13 @@ pub(super) fn locate_line(layout: &LayoutState, off: DocumentOffset) -> Option<( && off.byte >= line.start_byte && off.byte <= line.end_byte { - // Bias to next line at exact wrap seam. - if off.byte == line.end_byte && low + 1 < para.lines.len() && para.lines[low + 1].start_byte == off.byte { + // At an exact wrap seam, choose the visual line from gravity. Upstream stays + // on the line that ends here (`low`); otherwise bias to the next line. + if off.byte == line.end_byte + && low + 1 < para.lines.len() + && para.lines[low + 1].start_byte == off.byte + && !matches!(gravity, VisualGravity::Upstream) + { return Some((p_ix, low + 1)); } return Some((p_ix, low)); diff --git a/crates/gpui-flowtext/src/rich_text/layout/paragraph_layout.rs b/crates/gpui-flowtext/src/rich_text/layout/paragraph_layout.rs index b1137d5a..9c18ca1c 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/paragraph_layout.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/paragraph_layout.rs @@ -1,6 +1,6 @@ #[hotpath::measure] pub(super) fn layout_paragraph_at( - document: &Document, + document: &DocumentProjection, paragraph_ix: usize, width: Pixels, previous_bottom: Pixels, @@ -64,9 +64,14 @@ pub(super) fn layout_paragraph_at( ( LaidOutParagraph { index: paragraph_ix, + // PARAGRAPH-LOCAL byte range (matches `offset.byte`, which is paragraph-local + // everywhere in the editor). A prior change set this to the ABSOLUTE document + // range (`paragraph_byte_range`), so `contains_byte` compared a local caret + // byte against an absolute range and the caret vanished for every byte below + // the paragraph's document offset (the "invisible caret" bug). + byte_range: 0..paragraph_text.len(), cache_key, len: paragraph_text.len(), - byte_range: 0..paragraph_text.len(), top: y, bottom, lines: laid_out_lines, @@ -90,7 +95,7 @@ pub(super) struct ParagraphChunkBuildResult { #[allow(clippy::too_many_arguments, reason = "Paragraph layout needs several independent shaping and theme inputs.")] #[hotpath::measure] pub(super) fn build_paragraph_chunk_layout_with_visibility( - document: &Document, + document: &DocumentProjection, paragraph_ix: usize, width: Pixels, start_byte: usize, @@ -110,7 +115,6 @@ pub(super) fn build_paragraph_chunk_layout_with_visibility( } let paragraph = Paragraph { style: prep.layout_style, - byte_range: 0..prep.paragraph_text.len(), runs: prep.layout_runs.as_ref().to_vec(), version: prep.layout_version, }; @@ -170,7 +174,7 @@ pub(super) fn build_paragraph_chunk_layout_with_visibility( #[allow(clippy::too_many_arguments, reason = "Paragraph layout needs several independent shaping and theme inputs.")] #[hotpath::measure] fn layout_paragraph_chunk_at( - document: &Document, + document: &DocumentProjection, layout_paragraph_ix: usize, display_paragraph_ix: usize, width: Pixels, @@ -204,7 +208,7 @@ fn layout_paragraph_chunk_at( #[allow(clippy::too_many_arguments, reason = "Paragraph layout needs several independent shaping and theme inputs.")] #[hotpath::measure] fn layout_prepared_paragraph_chunk_at( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, display_paragraph_ix: usize, width: Pixels, @@ -301,9 +305,12 @@ fn layout_prepared_paragraph_chunk_at( let paragraph = LaidOutParagraph { index: display_paragraph_ix, + // PARAGRAPH-LOCAL byte range of THIS chunk (matches the paragraph-local + // `offset.byte`). Was the ABSOLUTE document range, which broke `contains_byte` + // and hid the caret for local bytes below the paragraph's document offset. + byte_range: start_byte..byte_range_end, cache_key, len, - byte_range: start_byte..byte_range_end, top: paragraph_top, bottom: row_bottom, lines: laid_out_lines, diff --git a/crates/gpui-flowtext/src/rich_text/layout/prep.rs b/crates/gpui-flowtext/src/rich_text/layout/prep.rs index 90f270c7..c9b74e24 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/prep.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/prep.rs @@ -2,13 +2,22 @@ use std::sync::Arc; use crop::Rope; +// §act-nine A9.3: prep validity is CONTENT-keyed — (style, version) via +// `ParagraphCacheKey` plus the invisibility mode. There is deliberately NO +// global `edit_generation` here: an edit to one paragraph must not invalidate +// every other paragraph's prep. Correctness rests on version discipline — +// every content change bumps `paragraph.version`, and structural rebuilds / +// canonical installs carry surviving versions FORWARD (never reset to 0), so +// a (style, version) pair is never reused for possibly-different content. #[derive(Clone, Copy, Debug, PartialEq)] pub(super) struct ParagraphPrepKey { pub(super) paragraph_key: ParagraphCacheKey, pub(super) invisibility_mode: bool, - pub(super) edit_generation: u64, } +// Shaping/work validity inherits the content-keyed `prep_key` (generation-free +// per §act-nine A9.3) and KEEPS `layout_generation`: theme/zoom/width clears +// bump it deliberately, and those inputs are invisible to the per-paragraph key. #[derive(Clone, Copy, Debug, PartialEq)] pub(super) struct ParagraphLayoutWorkKey { pub(super) prep_key: ParagraphPrepKey, @@ -19,6 +28,11 @@ pub(super) struct ParagraphLayoutWorkKey { #[derive(Clone, Debug)] pub(super) struct ParagraphPrep { pub(super) key: ParagraphPrepKey, + /// Stable identity of the paragraph this prep was built from — the install + /// gate and validity checks pin the prep to it, so a positional shift can + /// never install or serve one paragraph's prep for another that happens to + /// share a (style, version) pair. + pub(super) paragraph_id: ParagraphId, pub(super) paragraph_ix: usize, pub(super) paragraph_text: Arc, pub(super) layout_runs: Arc<[TextRun]>, @@ -32,7 +46,6 @@ pub(super) struct ParagraphPrep { pub(super) struct ParagraphPrepBatchRequest { pub(super) text: Rope, pub(super) theme: DocumentTheme, - pub(super) edit_generation: u64, pub(super) invisibility_mode: bool, pub(super) requested: usize, pub(super) paragraphs: Vec, @@ -41,13 +54,13 @@ pub(super) struct ParagraphPrepBatchRequest { } pub(super) struct ParagraphPrepSource { + paragraph_id: ParagraphId, paragraph_ix: usize, paragraph: Paragraph, byte_range: Range, } pub(super) struct ParagraphPrepBatchResult { - pub(super) edit_generation: u64, pub(super) invisibility_mode: bool, pub(super) requested: usize, pub(super) completed: usize, @@ -71,10 +84,10 @@ pub(super) fn build_paragraph_prep_batch(request: ParagraphPrepBatchRequest) -> let Some(prep) = build_paragraph_prep_from_parts( &request.text, &request.theme, + source.paragraph_id, source.paragraph_ix, &source.paragraph, source.byte_range.clone(), - request.edit_generation, request.invisibility_mode, ) else { continue; @@ -93,7 +106,6 @@ pub(super) fn build_paragraph_prep_batch(request: ParagraphPrepBatchRequest) -> .collect::>(); ParagraphPrepBatchResult { - edit_generation: request.edit_generation, invisibility_mode: request.invisibility_mode, requested: request.requested, completed: preps.len(), @@ -105,8 +117,7 @@ pub(super) fn build_paragraph_prep_batch(request: ParagraphPrepBatchRequest) -> #[hotpath::measure] pub(super) fn paragraph_prep_batch_request( - document: &Document, - edit_generation: u64, + document: &DocumentProjection, invisibility_mode: bool, paragraphs: Vec, max_paragraphs: usize, @@ -116,11 +127,13 @@ pub(super) fn paragraph_prep_batch_request( let paragraphs = paragraphs .into_iter() .filter_map(|paragraph_ix| { + let paragraph_id = document.ids.paragraph_ids.get(paragraph_ix).copied()?; document .paragraphs .get(paragraph_ix) .cloned() .map(|paragraph| ParagraphPrepSource { + paragraph_id, paragraph_ix, byte_range: paragraph_byte_range(document, paragraph_ix), paragraph, @@ -130,7 +143,6 @@ pub(super) fn paragraph_prep_batch_request( ParagraphPrepBatchRequest { text: document.text.clone(), theme: document.theme.clone(), - edit_generation, invisibility_mode, requested, paragraphs, @@ -139,21 +151,42 @@ pub(super) fn paragraph_prep_batch_request( } } +thread_local! { + // §act-eleven C4 tripwire: counts full prep BUILDS since the last reset. + // Equality tests prove the content-keyed caches are CORRECT; this counter + // proves they are EFFECTIVE — a regression that quietly reverts to + // whole-viewport re-prep per keystroke (the T8.12 "landed but inert" class) + // fails the bounded-recompute test instead of surfacing as a perf mystery. + static PREP_BUILD_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Prep builds since the last [`reset_prep_build_count`]. Test observability. +#[allow(dead_code, reason = "consumed by the in-crate C4 tripwire tests; unused in non-test builds")] +#[must_use] +pub fn prep_build_count() -> u64 { + PREP_BUILD_COUNT.with(std::cell::Cell::get) +} + +#[allow(dead_code, reason = "consumed by the in-crate C4 tripwire tests; unused in non-test builds")] +pub fn reset_prep_build_count() { + PREP_BUILD_COUNT.with(|count| count.set(0)); +} + #[hotpath::measure] -fn build_paragraph_prep_from_parts( +pub(super) fn build_paragraph_prep_from_parts( text: &Rope, theme: &DocumentTheme, + paragraph_id: ParagraphId, paragraph_ix: usize, paragraph: &Paragraph, paragraph_byte_range: Range, - edit_generation: u64, invisibility_mode: bool, ) -> Option { + PREP_BUILD_COUNT.with(|count| count.set(count.get() + 1)); let source_len = paragraph_text_len(paragraph); let key = ParagraphPrepKey { paragraph_key: paragraph_cache_key_for_paragraph(paragraph), invisibility_mode, - edit_generation, }; if invisibility_mode && matches!(paragraph.style, ParagraphStyle::Normal) { @@ -162,6 +195,7 @@ fn build_paragraph_prep_from_parts( // project, so it is hidden rather than rendered as a blank visible line. return Some(ParagraphPrep { key, + paragraph_id, paragraph_ix, paragraph_text: Arc::from(""), layout_runs: Arc::from(Vec::::new().into_boxed_slice()), @@ -176,6 +210,7 @@ fn build_paragraph_prep_from_parts( let wrap_break_ends = wrap_break_ends(&text); return Some(ParagraphPrep { key, + paragraph_id, paragraph_ix, paragraph_text: Arc::from(text), layout_runs: Arc::from(runs.into_boxed_slice()), @@ -190,6 +225,7 @@ fn build_paragraph_prep_from_parts( if invisibility_mode && !paragraph_is_visible_for_theme(theme, paragraph) { return Some(ParagraphPrep { key, + paragraph_id, paragraph_ix, paragraph_text: Arc::from(""), layout_runs: Arc::from(Vec::::new().into_boxed_slice()), @@ -205,6 +241,7 @@ fn build_paragraph_prep_from_parts( let wrap_break_ends = wrap_break_ends(&text); Some(ParagraphPrep { key, + paragraph_id, paragraph_ix, paragraph_text: Arc::from(text), layout_runs: Arc::from(paragraph.runs.clone().into_boxed_slice()), @@ -218,20 +255,20 @@ fn build_paragraph_prep_from_parts( #[hotpath::measure] pub(super) fn build_paragraph_prep( - document: &Document, + document: &DocumentProjection, paragraph_ix: usize, - edit_generation: u64, invisibility_mode: bool, ) -> Option { let paragraph = document.paragraphs.get(paragraph_ix)?; + let paragraph_id = document.ids.paragraph_ids.get(paragraph_ix).copied()?; let paragraph_byte_range = paragraph_byte_range(document, paragraph_ix); build_paragraph_prep_from_parts( &document.text, &document.theme, + paragraph_id, paragraph_ix, paragraph, paragraph_byte_range, - edit_generation, invisibility_mode, ) } @@ -329,9 +366,10 @@ mod prep_tests { }], ); - let prep = build_paragraph_prep(&document, 0, 7, false).expect("paragraph prep"); + let prep = build_paragraph_prep(&document, 0, false).expect("paragraph prep"); - assert_eq!(prep.key.edit_generation, 7); + assert!(!prep.key.invisibility_mode); + assert_eq!(prep.paragraph_id, document.ids.paragraph_ids[0]); assert_eq!(prep.paragraph_text.as_ref(), "alpha beta/gamma"); assert_eq!(prep.layout_runs.len(), 1); assert!(prep.visible); @@ -365,7 +403,7 @@ mod prep_tests { }], ); - let prep = build_paragraph_prep(&document, 0, 3, true).expect("paragraph prep"); + let prep = build_paragraph_prep(&document, 0, true).expect("paragraph prep"); assert!(prep.visible); assert_eq!(prep.paragraph_text.as_ref(), "cite spoken"); @@ -384,7 +422,7 @@ mod prep_tests { }], ); - let prep = build_paragraph_prep(&document, 0, 1, true).expect("paragraph prep"); + let prep = build_paragraph_prep(&document, 0, true).expect("paragraph prep"); assert!(!prep.visible); assert_eq!(prep.paragraph_text.as_ref(), ""); @@ -411,7 +449,7 @@ mod prep_tests { ], ); - let result = build_paragraph_prep_batch(paragraph_prep_batch_request(&document, 9, false, vec![0, 1, 2], 16, 1)); + let result = build_paragraph_prep_batch(paragraph_prep_batch_request(&document, false, vec![0, 1, 2], 16, 1)); assert_eq!(result.completed, 1); assert_eq!(result.deferred_paragraphs, vec![1, 2]); @@ -420,7 +458,7 @@ mod prep_tests { #[test] #[hotpath::measure] fn prep_uses_offset_index_range_instead_of_stale_paragraph_field() { - let mut document = document_from_input( + let document = document_from_input( DocumentTheme::default(), vec![ InputParagraph { @@ -433,9 +471,8 @@ mod prep_tests { }, ], ); - paragraphs_mut(&mut document)[1].byte_range = 6..19; - let prep = build_paragraph_prep(&document, 1, 1, false).expect("paragraph prep"); + let prep = build_paragraph_prep(&document, 1, false).expect("paragraph prep"); assert_eq!(prep.paragraph_text.as_ref(), "Kepe et al. ‘23"); } diff --git a/crates/gpui-flowtext/src/rich_text/layout/shaping.rs b/crates/gpui-flowtext/src/rich_text/layout/shaping.rs index 1c40bd83..a9a2630d 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/shaping.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/shaping.rs @@ -1,6 +1,6 @@ #[hotpath::measure] pub(super) fn measure_line_width( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: &EffectiveParagraphFormat, paragraph_text: &str, @@ -70,7 +70,7 @@ pub(super) fn measure_line_width( #[hotpath::measure] pub(super) fn shape_line( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: EffectiveParagraphFormat, line_text: &str, @@ -91,8 +91,6 @@ pub(super) fn shape_line( ); let mut x = px(0.0); let mut segments = Vec::with_capacity(fragments.len().max(1)); - let mut ascent = px(0.0); - let mut descent = px(0.0); for (fragment_ix, fragment) in fragments.iter().enumerate() { let text = &line_text[fragment.fragment.line_range.clone()]; @@ -105,8 +103,6 @@ pub(super) fn shape_line( let (box_pad_left, box_pad_right) = boxed_fragment_padding(&fragments, fragment_ix, document.theme.box_padding_left, document.theme.box_padding_right); let segment_ascent = shaped.ascent; let segment_descent = shaped.descent; - ascent = ascent.max(segment_ascent); - descent = descent.max(segment_descent); x += box_pad_left; segments.push(LaidOutSegment { shaped, @@ -125,7 +121,7 @@ pub(super) fn shape_line( if segments.is_empty() { let format = run_format(document, &p_format, RunStyles::default()); - let shaped = shape_fragment(window, "", &format); + let shaped = std::sync::Arc::new(shape_fragment(window, "", &format)); #[cfg(target_os = "linux")] let (segment_ascent, segment_descent) = { let (font_ascent, font_descent) = font_metrics_for_format(&format, cx); @@ -147,11 +143,13 @@ pub(super) fn shape_line( }); } - ascent = segments + // §perf: ascent/descent derive solely from the final segment set, so compute them + // once here instead of also max-accumulating them per segment inside the loop above. + let ascent = segments .iter() .map(|segment| segment.ascent) .fold(px(0.0), Pixels::max); - descent = segments + let descent = segments .iter() .map(|segment| segment.descent) .fold(px(0.0), Pixels::max); @@ -185,7 +183,10 @@ pub(super) fn shape_line( #[derive(Default)] pub(super) struct FragmentShapeCache { - shapes: FxHashMap, + // Arc'd values: a cache HIT previously deep-cloned the `ShapedLine` + // (glyph runs + decorations, ~1.4KB) — ~1.3GB of pure hit-path churn per + // profiled session. An Arc bump serves the same layout. + shapes: FxHashMap>, line_widths: FxHashMap, fragment_scratch: Vec, run_formats: FxHashMap, @@ -212,17 +213,17 @@ pub(super) fn shape_fragment_cached( source_start: usize, styles: RunStyles, cache: &mut FragmentShapeCache, -) -> ShapedLine { +) -> std::sync::Arc { let key = FragmentShapeCacheKey { source_start, len: text.len(), styles, }; if let Some(shaped) = cache.shapes.get(&key) { - return shaped.clone(); + return std::sync::Arc::clone(shaped); } - let shaped = shape_fragment(window, text, format); - cache.shapes.insert(key, shaped.clone()); + let shaped = std::sync::Arc::new(shape_fragment(window, text, format)); + cache.shapes.insert(key, std::sync::Arc::clone(&shaped)); shaped } @@ -253,7 +254,7 @@ pub(super) struct FormattedFragment { #[cfg(test)] #[hotpath::measure] pub(super) fn formatted_fragments_for_range( - document: &Document, + document: &DocumentProjection, p_format: &EffectiveParagraphFormat, paragraph: &Paragraph, range: &Range, @@ -275,7 +276,7 @@ pub(super) fn formatted_fragments_for_range( #[hotpath::measure] pub(super) fn formatted_fragments_for_range_into( - document: &Document, + document: &DocumentProjection, p_format: &EffectiveParagraphFormat, paragraph: &Paragraph, range: &Range, diff --git a/crates/gpui-flowtext/src/rich_text/layout/table_layout.rs b/crates/gpui-flowtext/src/rich_text/layout/table_layout.rs index e8a97129..ecbab1ba 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/table_layout.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/table_layout.rs @@ -1,6 +1,6 @@ #[hotpath::measure] fn layout_table_block( - document: &Document, + document: &DocumentProjection, block_ix: usize, table: &TableBlock, width: Pixels, @@ -11,7 +11,7 @@ fn layout_table_block( let table_left = document.theme.pageless_inset_x; let table_width = (width - document.theme.pageless_inset_x * 2.0).max(px(1.0)); let column_count = table - .column_widths + .columns .len() .max( table @@ -60,7 +60,7 @@ fn layout_table_block( } #[hotpath::measure] -fn table_row_height(document: &Document, row: &TableRow, column_widths: &[Pixels], window: &mut Window, cx: &mut App) -> Pixels { +fn table_row_height(document: &DocumentProjection, row: &TableRow, column_widths: &[Pixels], window: &mut Window, cx: &mut App) -> Pixels { let mut column_ix = 0; row .cells @@ -81,8 +81,9 @@ fn resolved_table_column_widths(table: &TableBlock, table_width: Pixels, column_ let mut auto_count = 0usize; for ix in 0..column_count { match table - .column_widths + .columns .get(ix) + .map(|column| &column.width) .unwrap_or(&TableColumnWidth::Fraction(1)) { TableColumnWidth::FixedPx(width) => fixed_total += px(*width as f32), @@ -95,8 +96,9 @@ fn resolved_table_column_widths(table: &TableBlock, table_width: Pixels, column_ (0..column_count) .map(|ix| { match table - .column_widths + .columns .get(ix) + .map(|column| &column.width) .unwrap_or(&TableColumnWidth::Fraction(1)) { TableColumnWidth::FixedPx(width) => px(*width as f32).max(px(8.0)), @@ -120,7 +122,7 @@ fn spanned_column_width(column_widths: &[Pixels], column_ix: usize, span: usize) } #[hotpath::measure] -fn table_cell_height(document: &Document, cell: &TableCell, width: Pixels, window: &mut Window, cx: &mut App) -> Pixels { +fn table_cell_height(document: &DocumentProjection, cell: &TableCell, width: Pixels, window: &mut Window, cx: &mut App) -> Pixels { let padding = table_cell_padding(); let content_width = (width - padding * 2.0).max(px(1.0)); let mut y = padding; @@ -144,7 +146,7 @@ fn table_cell_height(document: &Document, cell: &TableCell, width: Pixels, windo #[hotpath::measure] fn layout_table_cell_blocks( - document: &Document, + document: &DocumentProjection, cell: &TableCell, bounds: Bounds, window: &mut Window, @@ -173,7 +175,7 @@ fn layout_table_cell_blocks( #[hotpath::measure] fn layout_table_cell_paragraph( - document: &Document, + document: &DocumentProjection, cell_paragraph: &TableCellParagraph, index: usize, width: Pixels, diff --git a/crates/gpui-flowtext/src/rich_text/layout/types.rs b/crates/gpui-flowtext/src/rich_text/layout/types.rs index 7fd83bfa..44919be5 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/types.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/types.rs @@ -135,17 +135,25 @@ pub(super) struct ParagraphCacheKey { #[derive(Clone, Copy, PartialEq)] pub(super) struct ParagraphHeightCacheEntry { pub(super) key: ParagraphCacheKey, + /// §act-nine A9.3: stable identity replaces the global `edit_generation` — + /// the cache is positional, and (style, version) alone can collide across + /// different paragraphs after a row shift. + pub(super) paragraph_id: ParagraphId, pub(super) width: Pixels, pub(super) invisibility_mode: bool, - pub(super) edit_generation: u64, pub(super) height: Pixels, } #[hotpath::measure] -pub(super) fn paragraph_cache_key(_document: &Document, paragraph: &Paragraph) -> ParagraphCacheKey { +pub(super) fn paragraph_cache_key(_document: &DocumentProjection, paragraph: &Paragraph) -> ParagraphCacheKey { paragraph_cache_key_for_paragraph(paragraph) } +// NOT a content hash: hashes (style, version) only. Its correctness as a +// layout-cache validity key rests entirely on version discipline (§act-nine +// A9.3) — every content mutation bumps `paragraph.version`, and structural +// rebuilds / canonical installs carry surviving versions forward (never back +// to 0), so a (style, version) pair is never reused for different content. #[hotpath::measure] pub(super) fn paragraph_cache_key_for_paragraph(paragraph: &Paragraph) -> ParagraphCacheKey { let mut hasher = FxHasher::default(); @@ -227,7 +235,7 @@ impl LaidOutLine { #[derive(Clone)] pub(super) struct LaidOutSegment { - pub(super) shaped: ShapedLine, + pub(super) shaped: std::sync::Arc, pub(super) format: EffectiveRunFormat, pub(super) x: Pixels, pub(super) width: Pixels, diff --git a/crates/gpui-flowtext/src/rich_text/layout/wrapping.rs b/crates/gpui-flowtext/src/rich_text/layout/wrapping.rs index e9188410..8c521926 100644 --- a/crates/gpui-flowtext/src/rich_text/layout/wrapping.rs +++ b/crates/gpui-flowtext/src/rich_text/layout/wrapping.rs @@ -1,6 +1,6 @@ #[hotpath::measure] pub(super) fn wrap_lines( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: EffectiveParagraphFormat, text: &str, @@ -60,7 +60,7 @@ pub(super) fn wrap_lines( #[hotpath::measure] fn wrap_lines_limited( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: EffectiveParagraphFormat, text: &str, @@ -142,7 +142,7 @@ fn wrap_lines_limited( #[hotpath::measure] fn wrap_text_segment_limited( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: EffectiveParagraphFormat, text: &str, @@ -311,7 +311,7 @@ fn wrap_text_segment_limited( #[hotpath::measure] fn push_wrapped_soft_segment( lines: &mut Vec, - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: EffectiveParagraphFormat, text: &str, @@ -340,7 +340,7 @@ fn push_wrapped_soft_segment( #[hotpath::measure] fn wrap_text_segment( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: EffectiveParagraphFormat, text: &str, @@ -502,7 +502,7 @@ fn first_break_after(break_ends: &[usize], byte: usize) -> usize { #[allow(clippy::too_many_arguments, reason = "Wrapping needs independent font, width, style, and cache inputs.")] #[hotpath::measure] fn first_break_over_width( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: &EffectiveParagraphFormat, text: &str, @@ -581,22 +581,30 @@ pub(super) fn wrap_break_ends(text: &str) -> Vec { breaks } -#[hotpath::measure] +// §perf: NOT hotpath-measured — a two-instruction predicate called tens of +// millions of times per layout pass; the measurement hooks cost orders of +// magnitude more than the function and dominated profiler-build profiles +// (11.3% of total CPU attributed to a char compare). +#[inline] pub(super) fn is_wrap_break(ch: char) -> bool { ch.is_whitespace() || matches!(ch, '-' | '/' | ',' | ';' | ':') } -#[hotpath::measure] +#[inline] pub(super) fn skip_leading_whitespace(text: &str, mut byte: usize) -> usize { - while byte < text.len() && text[byte..].chars().next().is_some_and(char::is_whitespace) { - byte += text[byte..].chars().next().unwrap().len_utf8(); + // §perf: decode each char once per iteration instead of twice (peek + re-decode). + while let Some(ch) = text[byte..].chars().next() { + if !ch.is_whitespace() { + break; + } + byte += ch.len_utf8(); } byte } #[hotpath::measure] pub(super) fn first_overflow_line_end( - document: &Document, + document: &DocumentProjection, paragraph: &Paragraph, p_format: &EffectiveParagraphFormat, text: &str, diff --git a/crates/gpui-flowtext/src/rich_text/mod.rs b/crates/gpui-flowtext/src/rich_text/mod.rs index 4c4f9272..8f07cc9c 100644 --- a/crates/gpui-flowtext/src/rich_text/mod.rs +++ b/crates/gpui-flowtext/src/rich_text/mod.rs @@ -1,4 +1,4 @@ -// Submodules. Document model, edit helpers, persistence, demo builders, and +// Submodules. DocumentProjection model, edit helpers, persistence, demo builders, and // collaboration IDs live at the crate root; this module exposes the reusable // GPUI rich-text editor and rendering internals. mod benchmarks; @@ -55,7 +55,7 @@ pub(crate) fn log_timing_lazy(label: &str, start: Instant, detail: impl FnOnce() } #[cfg(test)] -use editor::{EditOperation, adjust_drop_after_source_delete}; +use editor::adjust_drop_after_source_delete; #[cfg(test)] mod tests; diff --git a/crates/gpui-flowtext/src/rich_text/paint.rs b/crates/gpui-flowtext/src/rich_text/paint.rs index 3a295249..7ce83f3a 100644 --- a/crates/gpui-flowtext/src/rich_text/paint.rs +++ b/crates/gpui-flowtext/src/rich_text/paint.rs @@ -1,6 +1,8 @@ use std::ops::Range; -use gpui::{App, Background, Bounds, Pixels, Point, ScrollHandle, Window, black, fill, hsla, point, px, rgb, size}; +use gpui::{App, Background, Bounds, Hsla, Pixels, Point, ScrollHandle, Window, black, fill, hsla, point, px, rgb, size}; + +use flowstate_fidelity::{self as fidelity, FidelityClass}; use super::*; @@ -12,7 +14,15 @@ pub(super) fn paint_layout( drag_selection: Option<&EditorSelection>, show_caret: bool, caret_width: Pixels, + caret_color_rgb: Option, + // Caret color to use when `caret_color_rgb` is `None` (solo editing, no + // collaboration-caret color). MUST be the theme's default text color so the + // caret contrasts with the background — a hardcoded black caret is invisible + // on a dark theme (the "invisible caret" bug). + default_caret_color: Hsla, external_carets: &[ExternalCaret], + external_selections: &[ExternalSelection], + annotation_selections: &[ExternalSelection], search_highlights: &[Range], active_search_highlight: Option, window: &mut Window, @@ -22,6 +32,23 @@ pub(super) fn paint_layout( let content_mask = window.content_mask().bounds; let visible_range = visible_paragraph_range(layout, bounds.origin, content_mask); let visible_count = visible_range.end.saturating_sub(visible_range.start); + // Fidelity: record the rendered paragraph set for this paint so a layout that + // trails the document (missing paragraphs, wrong index range) is visible in + // the firehose. The document/layout generation counters are not threaded into + // paint, so the caret-vs-laid-out check below is the staleness assertion. + if fidelity::enabled() { + let first_index = layout.paragraphs.first().map(|paragraph| paragraph.index); + let last_index = layout.paragraphs.last().map(|paragraph| paragraph.index); + fidelity::event(FidelityClass::Structure, "layout-paragraph-set", || { + format!( + "laid_out={} index_range={first_index:?}..={last_index:?} visible={}..{} blocks={}", + layout.paragraphs.len(), + visible_range.start, + visible_range.end, + layout.block_count(), + ) + }); + } for paragraph in &layout.paragraphs[visible_range.clone()] { if !paragraph_intersects_mask(paragraph, bounds.origin, content_mask) { continue; @@ -62,6 +89,32 @@ pub(super) fn paint_layout( if let Some(selection) = drag_selection { paint_selection(layout, selection, bounds.origin, content_mask, visible_range.clone(), window); } + // Remote peers' selection spans, each in that peer's presence color (the same + // hue as their caret, softened for a behind-the-glyphs highlight). + for external in external_selections { + paint_text_range_fill( + layout, + &external.selection, + bounds.origin, + content_mask, + visible_range.clone(), + remote_selection_color(external.color_rgb), + window, + ); + } + // Durable annotations use their own overlay layer so presence refreshes do + // not make comment anchors blink or disappear. + for annotation in annotation_selections { + paint_text_range_fill( + layout, + &annotation.selection, + bounds.origin, + content_mask, + visible_range.clone(), + annotation_selection_color(annotation.color_rgb), + window, + ); + } for paragraph in &layout.paragraphs[visible_range.clone()] { if !paragraph_intersects_mask(paragraph, bounds.origin, content_mask) { continue; @@ -96,20 +149,89 @@ pub(super) fn paint_layout( } } } + // Fidelity: before painting the local caret, verify it resolves against a + // layout that still matches the model caret. `show_caret` is only set for the + // chunk that owns the caret (see element.rs `caret_offset_belongs_to_chunk`), + // so the located paragraph's document index MUST equal the model caret's + // paragraph; a mismatch (or the model caret pointing past the last laid-out + // paragraph) means the painted caret is being resolved against a stale layout + // — render lag where the model caret is correct but the paint trails. + if fidelity::enabled() + && let Some(selection) = selection + && selection.is_caret() + && show_caret + { + let painted_rect = caret_bounds(layout, selection.head, selection.head_gravity, bounds.origin); + let painted_paragraph = locate_line(layout, selection.head, selection.head_gravity).map(|(p_ix, _)| layout.paragraphs[p_ix].index); + let max_paragraph = layout.paragraphs.last().map(|paragraph| paragraph.index); + fidelity::event(FidelityClass::Caret, "paint", || { + format!( + "model_caret={:?} gravity={:?} painted_rect={painted_rect:?} painted_paragraph={painted_paragraph:?} laid_out={} max_paragraph={max_paragraph:?}", + selection.head, + selection.head_gravity, + layout.paragraphs.len(), + ) + }); + fidelity::check( + painted_paragraph.is_none_or(|index| index == selection.head.paragraph), + FidelityClass::Caret, + "caret-render-stale", + || { + format!( + "model_paragraph={} painted_paragraph={painted_paragraph:?} model_byte={} laid_out={}", + selection.head.paragraph, + selection.head.byte, + layout.paragraphs.len(), + ) + }, + ); + fidelity::check( + max_paragraph.is_none_or(|max| selection.head.paragraph <= max), + FidelityClass::Structure, + "layout-generation-lag", + || { + format!( + "model_paragraph={} max_laid_out_paragraph={max_paragraph:?} laid_out={}", + selection.head.paragraph, + layout.paragraphs.len(), + ) + }, + ); + } if let Some(selection) = selection && selection.is_caret() && show_caret - && let Some(mut caret) = caret_bounds(layout, selection.head, bounds.origin) + && let Some(mut caret) = caret_bounds(layout, selection.head, selection.head_gravity, bounds.origin) && caret.intersects(&content_mask) { caret.size.width = caret_width; - window.paint_quad(fill(snap_vertical_rule_to_device_pixels(caret, window), black())); + let caret_color = caret_color_rgb.map_or_else(|| Background::from(default_caret_color), |color_rgb| Background::from(rgb(color_rgb))); + window.paint_quad(fill(snap_vertical_rule_to_device_pixels(caret, window), caret_color)); } for external_caret in external_carets { - if let Some(mut caret) = caret_bounds(layout, external_caret.offset, bounds.origin) + // Fidelity: remote carets are pre-filtered to this chunk (element.rs), so a + // located paragraph that differs from the remote offset's paragraph means + // the remote caret is painted against a stale layout. + if fidelity::enabled() { + let painted_paragraph = + locate_line(layout, external_caret.offset, external_caret.visual_gravity).map(|(p_ix, _)| layout.paragraphs[p_ix].index); + fidelity::event(FidelityClass::Caret, "paint-external", || { + format!( + "offset={:?} gravity={:?} painted_paragraph={painted_paragraph:?} color=#{:06x}", + external_caret.offset, external_caret.visual_gravity, external_caret.color_rgb, + ) + }); + fidelity::check( + painted_paragraph.is_none_or(|index| index == external_caret.offset.paragraph), + FidelityClass::Caret, + "caret-render-stale", + || format!("external offset={:?} painted_paragraph={painted_paragraph:?}", external_caret.offset), + ); + } + if let Some(mut caret) = caret_bounds(layout, external_caret.offset, external_caret.visual_gravity, bounds.origin) && caret.intersects(&content_mask) { - caret.size.width = px(2.0); + caret.size.width = caret_width; window.paint_quad(fill( snap_vertical_rule_to_device_pixels(caret, window), Background::from(rgb(external_caret.color_rgb)), @@ -199,7 +321,7 @@ fn paint_table_block( } let cell_selected = matches!( selected_block, - Some(BlockSelection::TableCell { block_ix, row_ix: selected_row, cell_ix: selected_cell }) + Some(BlockSelection::TableCell { block_ix, row_ix: selected_row, cell_ix: selected_cell, .. }) if block_ix == table.block_ix && selected_row == row_ix && selected_cell == cell_ix ); window.paint_quad(fill( @@ -222,6 +344,30 @@ fn paint_table_block( paint_table_text_selection(paragraph, caret.anchor, caret.byte, origin, content_mask, window); } paint_table_paragraph(paragraph, origin, content_mask, window, cx); + // Fidelity: a table-cell caret whose byte exceeds the laid-out + // paragraph length is being resolved against a stale cell layout. + if fidelity::enabled() + && let Some(caret) = table_cell_caret + && caret.block_ix == table.block_ix + && caret.row_ix == row_ix + && caret.cell_ix == cell_ix + && caret.paragraph_block_ix == paragraph.index + && caret.caret_visible + { + let resolved = caret_bounds_in_paragraph(paragraph, caret.byte, origin); + fidelity::event(FidelityClass::Caret, "paint-table-cell", || { + format!( + "block={} row={} cell={} paragraph={} byte={} paragraph_len={} rect={resolved:?}", + caret.block_ix, caret.row_ix, caret.cell_ix, paragraph.index, caret.byte, paragraph.len, + ) + }); + fidelity::check(caret.byte <= paragraph.len, FidelityClass::Caret, "caret-render-stale", || { + format!( + "table caret byte={} exceeds paragraph_len={} (block={} paragraph={})", + caret.byte, paragraph.len, caret.block_ix, paragraph.index, + ) + }); + } if let Some(caret) = table_cell_caret && caret.block_ix == table.block_ix && caret.row_ix == row_ix @@ -527,10 +673,7 @@ fn paint_search_highlights( if highlight.end.paragraph < visible_start || highlight.start.paragraph > visible_end { continue; } - let selection = EditorSelection { - anchor: highlight.start, - head: highlight.end, - }; + let selection = EditorSelection::range(highlight.start, highlight.end); paint_text_range_fill( layout, &selection, @@ -558,6 +701,17 @@ pub(super) fn paint_selection( paint_text_range_fill(layout, selection, origin, content_mask, visible_range, hsla(0.0, 0.0, 0.0, 0.22), window); } +/// A peer's presence color softened for use as a selection highlight: the caret +/// paints at full opacity, but a span behind the glyphs must stay readable, so +/// drop it to a translucent fill. +fn remote_selection_color(color_rgb: u32) -> Hsla { + Hsla::from(rgb(color_rgb)).opacity(0.30) +} + +fn annotation_selection_color(color_rgb: u32) -> Hsla { + gpui::Hsla::from(rgb(color_rgb)).opacity(0.22) +} + fn paint_text_range_fill( layout: &LayoutState, selection: &EditorSelection, @@ -617,14 +771,17 @@ pub(super) fn paint_line_text(line: &LaidOutLine, origin: Point, content } for segment in &line.segments { let segment_origin = origin + point(segment.x, baseline); + // §perf: run_bounds is a function of the segment only (not the run), so compute + // it and the mask test once per segment rather than once per shaped run; an + // off-mask segment now skips all its runs in a single test. + let run_bounds = Bounds::new( + point(segment_origin.x, origin.y + baseline - segment.ascent), + size(segment.width.max(px(1.0)), segment.ascent + segment.descent), + ); + if !run_bounds.intersects(&content_mask) { + continue; + } for run in &segment.shaped.runs { - let run_bounds = Bounds::new( - point(segment_origin.x, origin.y + baseline - segment.ascent), - size(segment.width.max(px(1.0)), segment.ascent + segment.descent), - ); - if !run_bounds.intersects(&content_mask) { - continue; - } for glyph in &run.glyphs { let glyph_origin = segment_origin + point(glyph.position.x, px(0.0)); let result = if glyph.is_emoji { @@ -633,7 +790,7 @@ pub(super) fn paint_line_text(line: &LaidOutLine, origin: Point, content window.paint_glyph(glyph_origin, run.font_id, glyph.id, segment.font_size, segment.format.color) }; if let Err(error) = result { - eprintln!("failed to paint glyph: {error}"); + tracing::warn!(%error, "failed to paint glyph"); } } } diff --git a/crates/gpui-flowtext/src/rich_text/selection.rs b/crates/gpui-flowtext/src/rich_text/selection.rs index 2d97db8b..a355f239 100644 --- a/crates/gpui-flowtext/src/rich_text/selection.rs +++ b/crates/gpui-flowtext/src/rich_text/selection.rs @@ -1,3 +1,5 @@ +use flowstate_fidelity::{self as fidelity, FidelityClass}; + use super::*; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -8,17 +10,33 @@ pub(super) struct MouseSelectionOptions { #[hotpath::measure] pub(super) fn expand_mouse_selection( - document: &Document, + document: &DocumentProjection, anchor: DocumentOffset, head: DocumentOffset, granularity: SelectionGranularity, options: MouseSelectionOptions, ) -> EditorSelection { + // Fidelity: anchor/head arrive here from hit-testing a screen point. Record + // the resolution and assert both endpoints land inside the document's + // paragraph range — an out-of-range paragraph means the point resolved + // against a stale/mismatched projection. + if fidelity::enabled() { + let paragraphs = document.paragraphs.len(); + fidelity::event(FidelityClass::Caret, "mouse-select", || { + format!("anchor={anchor:?} head={head:?} granularity={granularity:?} options={options:?} paragraphs={paragraphs}") + }); + fidelity::check( + paragraphs == 0 || (anchor.paragraph < paragraphs && head.paragraph < paragraphs), + FidelityClass::Structure, + "hit-test-offset-range", + || format!("anchor={anchor:?} head={head:?} paragraphs={paragraphs}"), + ); + } if granularity != SelectionGranularity::Character { return expand_drag_selection(document, anchor, head, granularity); } - let exact = EditorSelection { anchor, head }; + let exact = EditorSelection::range(anchor, head); if options.exact || !options.smart_word_selection || same_word_fragment(document, anchor, head) { return exact; } @@ -27,22 +45,16 @@ pub(super) fn expand_mouse_selection( } #[hotpath::measure] -fn smart_word_selection(document: &Document, anchor: DocumentOffset, head: DocumentOffset) -> EditorSelection { +fn smart_word_selection(document: &DocumentProjection, anchor: DocumentOffset, head: DocumentOffset) -> EditorSelection { if head < anchor { - EditorSelection { - anchor: smart_word_selection_end(document, anchor), - head: smart_word_selection_start(document, head), - } + EditorSelection::range(smart_word_selection_end(document, anchor), smart_word_selection_start(document, head)) } else { - EditorSelection { - anchor: smart_word_selection_start(document, anchor), - head: smart_word_selection_end(document, head), - } + EditorSelection::range(smart_word_selection_start(document, anchor), smart_word_selection_end(document, head)) } } #[hotpath::measure] -fn smart_word_selection_start(document: &Document, offset: DocumentOffset) -> DocumentOffset { +fn smart_word_selection_start(document: &DocumentProjection, offset: DocumentOffset) -> DocumentOffset { let Some(text) = paragraph_text_for_offset(document, offset) else { return DocumentOffset::default(); }; @@ -60,7 +72,7 @@ fn smart_word_selection_start(document: &Document, offset: DocumentOffset) -> Do } #[hotpath::measure] -fn smart_word_selection_end(document: &Document, offset: DocumentOffset) -> DocumentOffset { +fn smart_word_selection_end(document: &DocumentProjection, offset: DocumentOffset) -> DocumentOffset { let Some(text) = paragraph_text_for_offset(document, offset) else { return DocumentOffset::default(); }; @@ -78,7 +90,7 @@ fn smart_word_selection_end(document: &Document, offset: DocumentOffset) -> Docu } #[hotpath::measure] -fn paragraph_text_for_offset(document: &Document, offset: DocumentOffset) -> Option { +fn paragraph_text_for_offset(document: &DocumentProjection, offset: DocumentOffset) -> Option { if document.paragraphs.is_empty() { return None; } @@ -112,7 +124,7 @@ fn previous_word_end_at_or_before(text: &str, mut byte: usize) -> usize { } #[hotpath::measure] -pub(super) fn offset_is_in_same_word_as(document: &Document, anchor: DocumentOffset, offset: DocumentOffset) -> bool { +pub(super) fn offset_is_in_same_word_as(document: &DocumentProjection, anchor: DocumentOffset, offset: DocumentOffset) -> bool { if anchor.paragraph != offset.paragraph || document.paragraphs.is_empty() { return false; } @@ -126,7 +138,7 @@ pub(super) fn offset_is_in_same_word_as(document: &Document, anchor: DocumentOff } #[hotpath::measure] -fn same_word_fragment(document: &Document, anchor: DocumentOffset, head: DocumentOffset) -> bool { +fn same_word_fragment(document: &DocumentProjection, anchor: DocumentOffset, head: DocumentOffset) -> bool { if anchor.paragraph != head.paragraph || anchor == head { return false; } diff --git a/crates/gpui-flowtext/src/rich_text/tests/decorations_drag_search.rs b/crates/gpui-flowtext/src/rich_text/tests/decorations_drag_search.rs index b34e42ac..8d51a4eb 100644 --- a/crates/gpui-flowtext/src/rich_text/tests/decorations_drag_search.rs +++ b/crates/gpui-flowtext/src/rich_text/tests/decorations_drag_search.rs @@ -159,101 +159,6 @@ fn dragged_text_drop_offset_adjusts_after_source_deletion() { ); } -#[test] -#[hotpath::measure] -fn move_rich_text_operation_undo_redo_restores_source_and_drop() { - let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("abc "), run("MOVE", emphasized), plain(" def")], - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("target")], - }, - ], - ); - let source = DocumentOffset { - paragraph: 0, - byte: "abc ".len(), - }..DocumentOffset { - paragraph: 0, - byte: "abc MOVE".len(), - }; - let fragment = selected_rich_fragment(&document, source.clone()); - let drop = DocumentOffset { - paragraph: 1, - byte: "tar".len(), - }; - let adjusted_drop = adjust_drop_after_source_delete(drop, source.clone()); - delete_cross_paragraph_range(&mut document, source.clone()); - let inserted_end = insert_rich_fragment_at(&mut document, adjusted_drop, &fragment); - let operation = EditOperation::MoveRichText { - source_range: source, - adjusted_drop, - inserted_range: adjusted_drop..inserted_end, - fragment, - }; - - assert_eq!(paragraph_text(&document, 0), "abc def"); - assert_eq!(paragraph_text(&document, 1), "tarMOVEget"); - assert!( - document.paragraphs[1] - .runs - .iter() - .any(|run| run.styles.semantic == RunSemanticStyle::Custom(2)) - ); - - operation.undo(&mut document); - assert_eq!(paragraph_text(&document, 0), "abc MOVE def"); - assert_eq!(paragraph_text(&document, 1), "target"); - assert!( - document.paragraphs[0] - .runs - .iter() - .any(|run| run.styles.semantic == RunSemanticStyle::Custom(2)) - ); - - operation.redo(&mut document); - assert_eq!(paragraph_text(&document, 0), "abc def"); - assert_eq!(paragraph_text(&document, 1), "tarMOVEget"); - assert!( - document.paragraphs[1] - .runs - .iter() - .any(|run| run.styles.semantic == RunSemanticStyle::Custom(2)) - ); -} - -#[test] -#[hotpath::measure] -fn soft_line_break_stays_inside_paragraph_and_copies_as_newline() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("alphaomega")], - }], - ); - insert_text_at(&mut document, 0, "alpha".len(), SOFT_LINE_BREAK_STR, RunStyles::default()); - - assert_eq!(document.paragraphs.len(), 1); - assert_eq!(paragraph_text(&document, 0), format!("alpha{SOFT_LINE_BREAK_STR}omega")); - assert_eq!( - selected_plain_text( - &document, - DocumentOffset { paragraph: 0, byte: 0 }..DocumentOffset { - paragraph: 0, - byte: paragraph_text_len(&document.paragraphs[0]), - }, - ), - "alpha\nomega" - ); -} - #[test] #[hotpath::measure] fn find_text_ranges_returns_document_offsets_across_paragraphs() { @@ -296,32 +201,3 @@ fn find_text_ranges_returns_document_offsets_across_paragraphs() { ); } -#[test] -#[hotpath::measure] -fn cross_paragraph_style_mutation_keeps_runs_and_unselected_text_intact() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("abc")], - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("def")], - }, - ], - ); - mutate_runs_in_range( - &mut document, - DocumentOffset { paragraph: 0, byte: 1 }..DocumentOffset { paragraph: 1, byte: 2 }, - |styles| styles.semantic = RunSemanticStyle::Custom(1), - ); - - assert_eq!(paragraph_text(&document, 0), "abc"); - assert_eq!(paragraph_text(&document, 1), "def"); - assert_ne!(document.paragraphs[0].runs[0].styles.semantic, RunSemanticStyle::Custom(1)); - assert_eq!(document.paragraphs[0].runs[1].styles.semantic, RunSemanticStyle::Custom(1)); - assert_eq!(document.paragraphs[1].runs[0].styles.semantic, RunSemanticStyle::Custom(1)); - assert_ne!(document.paragraphs[1].runs[1].styles.semantic, RunSemanticStyle::Custom(1)); -} diff --git a/crates/gpui-flowtext/src/rich_text/tests/edit_layout.rs b/crates/gpui-flowtext/src/rich_text/tests/edit_layout.rs index ba997a0e..1765a2b4 100644 --- a/crates/gpui-flowtext/src/rich_text/tests/edit_layout.rs +++ b/crates/gpui-flowtext/src/rich_text/tests/edit_layout.rs @@ -1,115 +1,5 @@ -use super::*; -use gpui::{Bounds, black, point, px, size}; -use std::{ - collections::hash_map::DefaultHasher, - hash::{Hash, Hasher}, -}; - -#[test] -#[hotpath::measure] -fn paragraph_edit_helpers_preserve_text_and_styles() { - let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![run("hello", RunStyles::default())], - }], - ); - - insert_text_at(&mut document, 0, "he".len(), "y", RunStyles::default()); - assert_eq!(paragraph_text(&document, 0), "heyllo"); - assert_eq!(document.paragraphs[0].runs.len(), 1); - - apply_style_to_paragraph_range(&mut document, 0, "hey".len().."heyll".len(), RunStyle::Semantic(2)); - assert_eq!(paragraph_text(&document, 0), "heyllo"); - assert_eq!(document.paragraphs[0].runs.len(), 3); - assert_eq!(document.paragraphs[0].runs[1].styles, emphasized); - - delete_range_in_paragraph(&mut document, 0, "he".len().."heyll".len()); - assert_eq!(paragraph_text(&document, 0), "heo"); - assert_eq!(document.paragraphs[0].runs.len(), 1); - assert_eq!(document.paragraphs[0].runs[0].styles, RunStyles::default()); -} - -#[test] -#[hotpath::measure] -fn document_rope_edits_keep_utf8_byte_offsets() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![run("abé🚀cd", RunStyles::default())], - }], - ); - insert_text_at(&mut document, 0, "abé".len(), "Z", RunStyles::default()); - assert_eq!(paragraph_text(&document, 0), "abéZ🚀cd"); - - let delete_start = "abé".len(); - let delete_end = "abéZ🚀".len(); - delete_range_in_paragraph(&mut document, 0, delete_start..delete_end); - assert_eq!(paragraph_text(&document, 0), "abécd"); -} - -#[test] -#[hotpath::measure] -fn multi_paragraph_rich_paste_preserves_first_pasted_paragraph_style() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: Vec::new(), - }], - ); - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: vec![ - InputParagraph { - style: ParagraphStyle::Custom(1), - runs: vec![plain("Heading")], - }, - InputParagraph { - style: ParagraphStyle::Custom(3), - runs: vec![plain("Tag")], - }, - ], - blocks: Vec::new(), - assets: Vec::new(), - }; - - insert_rich_fragment_at(&mut document, DocumentOffset::default(), &fragment); - - assert_eq!(paragraph_text(&document, 0), "Heading"); - assert_eq!(document.paragraphs[0].style, ParagraphStyle::Custom(1)); - assert_eq!(paragraph_text(&document, 1), "Tag"); - assert_eq!(document.paragraphs[1].style, ParagraphStyle::Custom(3)); -} - -#[test] -#[hotpath::measure] -fn single_paragraph_rich_paste_preserves_pasted_paragraph_style() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: Vec::new(), - }], - ); - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: vec![InputParagraph { - style: ParagraphStyle::Custom(2), - runs: vec![plain("Block")], - }], - blocks: Vec::new(), - assets: Vec::new(), - }; - - insert_rich_fragment_at(&mut document, DocumentOffset::default(), &fragment); - - assert_eq!(paragraph_text(&document, 0), "Block"); - assert_eq!(document.paragraphs[0].style, ParagraphStyle::Custom(2)); -} +// NOTE: include!()-spliced into tests/mod.rs — imports and the shared +// authority helpers live there. #[test] #[hotpath::measure] @@ -119,7 +9,6 @@ fn layout_fragments_preserve_text_when_run_boundary_splits_utf8_character() { let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); let paragraph = Paragraph { style: ParagraphStyle::Normal, - byte_range: 0..text.len(), runs: vec![ TextRun { len: split_inside_apostrophe, diff --git a/crates/gpui-flowtext/src/rich_text/tests/history_fragments.rs b/crates/gpui-flowtext/src/rich_text/tests/history_fragments.rs deleted file mode 100644 index 09a86d8e..00000000 --- a/crates/gpui-flowtext/src/rich_text/tests/history_fragments.rs +++ /dev/null @@ -1,123 +0,0 @@ - -#[test] -#[hotpath::measure] -fn history_operation_round_trip_for_text_and_paragraph_split() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("alpha beta")], - }], - ); - let before = capture_document_span(&document, 0..1); - split_paragraph_at(&mut document, 0, "alpha".len()); - insert_text_at(&mut document, 1, 0, "NEW ", RunStyles::default().with(RunStyle::Semantic(2))); - let after = capture_document_span(&document, 0..2); - assert_eq!(document.paragraphs.len(), 2); - - let operation = EditOperation::ReplaceParagraphSpan { before, after }; - operation.undo(&mut document); - assert_eq!(document.paragraphs.len(), 1); - assert_eq!(paragraph_text(&document, 0), "alpha beta"); - - operation.redo(&mut document); - assert_eq!(document.paragraphs.len(), 2); - assert_eq!(paragraph_text(&document, 0), "alpha"); - assert_eq!(paragraph_text(&document, 1), "NEW beta"); - assert_eq!(document.paragraphs[1].runs[0].styles.semantic, RunSemanticStyle::Custom(2)); -} - -#[test] -#[hotpath::measure] -fn rich_fragment_insert_bulk_preserves_multiline_paste_shape() { - let cite = RunStyles::default().with(RunStyle::Semantic(1)); - let emphasis = RunStyles::default().with(RunStyle::Semantic(2)); - let underline = RunStyles::default().with(RunStyle::Semantic(3)); - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("hello "), run("world", cite)], - }], - ); - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: vec![ - InputParagraph { - style: ParagraphStyle::Custom(3), - runs: vec![run("A", emphasis), run("B", emphasis)], - }, - InputParagraph { - style: ParagraphStyle::Custom(4), - runs: vec![run("C", underline)], - }, - ], - blocks: Vec::new(), - assets: Vec::new(), - }; - - let caret = insert_rich_fragment_at( - &mut document, - DocumentOffset { - paragraph: 0, - byte: "hello ".len(), - }, - &fragment, - ); - - assert_eq!(caret, DocumentOffset { paragraph: 1, byte: 1 }); - assert_eq!(document.paragraphs.len(), 2); - assert_eq!(paragraph_text(&document, 0), "hello AB"); - assert_eq!(paragraph_text(&document, 1), "Cworld"); - assert_eq!(document.paragraphs[0].style, ParagraphStyle::Normal); - assert_eq!(document.paragraphs[1].style, ParagraphStyle::Custom(4)); - assert_eq!(document.paragraphs[0].runs.last().unwrap().styles, emphasis); - assert_eq!(document.paragraphs[1].runs.first().unwrap().styles, underline); - assert_eq!(document.paragraphs[1].runs.last().unwrap().styles, cite); - assert!(matches!(&document.blocks[0], Block::Paragraph(paragraph) if paragraph.byte_range == document.paragraphs[0].byte_range)); - assert!(matches!(&document.blocks[1], Block::Paragraph(paragraph) if paragraph.byte_range == document.paragraphs[1].byte_range)); -} - -#[test] -#[hotpath::measure] -fn insert_rich_fragment_history_operation_round_trips_without_paragraph_snapshots() { - let emphasis = RunStyles::default().with(RunStyle::Semantic(2)); - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("alpha omega")], - }], - ); - let fragment = RichClipboardFragment { - format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), - paragraphs: vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![run(" BETA", emphasis)], - }], - blocks: Vec::new(), - assets: Vec::new(), - }; - let offset = DocumentOffset { - paragraph: 0, - byte: "alpha".len(), - }; - let inserted_end = insert_rich_fragment_at(&mut document, offset, &fragment); - let operation = EditOperation::InsertRichFragment { - offset, - inserted_end, - fragment, - }; - - assert_eq!(paragraph_text(&document, 0), "alpha BETA omega"); - operation.undo(&mut document); - assert_eq!(paragraph_text(&document, 0), "alpha omega"); - operation.redo(&mut document); - assert_eq!(paragraph_text(&document, 0), "alpha BETA omega"); - assert!( - document.paragraphs[0] - .runs - .iter() - .any(|run| run.styles == emphasis) - ); -} diff --git a/crates/gpui-flowtext/src/rich_text/tests/history_tables.rs b/crates/gpui-flowtext/src/rich_text/tests/history_tables.rs deleted file mode 100644 index 7d1f3fcf..00000000 --- a/crates/gpui-flowtext/src/rich_text/tests/history_tables.rs +++ /dev/null @@ -1,460 +0,0 @@ - -#[test] -#[hotpath::measure] -fn block_delete_operation_undo_redo_preserves_non_text_block() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - let equation = Block::Equation(EquationBlock { - source: "a^2+b^2=c^2".into(), - syntax: EquationSyntax::Latex, - display: EquationDisplay::Display, - version: 0, - }); - document.blocks = std::sync::Arc::new(vec![Block::Paragraph(document.paragraphs[0].clone()), equation.clone()]); - - let op = EditOperation::DeleteBlock { - block_ix: 1, - block: equation, - }; - op.redo(&mut document); - assert_eq!(document.blocks.len(), 1); - op.undo(&mut document); - assert_eq!(document.blocks.len(), 2); - assert!(matches!(document.blocks[1], Block::Equation(_))); -} - -#[test] -#[hotpath::measure] -fn insert_blocks_operation_undo_redo_preserves_inserted_table_and_equation() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - let blocks = vec![ - Block::Table(TableBlock { - rows: vec![TableRow { - cells: vec![TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }, - text: String::new(), - })], - row_span: 1, - col_span: 1, - }], - }], - column_widths: vec![TableColumnWidth::Fraction(1)], - style: TableStyle { header_row: false }, - version: 0, - }), - Block::Equation(EquationBlock { - source: "x=1".into(), - syntax: EquationSyntax::Latex, - display: EquationDisplay::Display, - version: 0, - }), - ]; - - let op = EditOperation::InsertBlocks { - block_ix: 1, - blocks: blocks.clone(), - }; - op.redo(&mut document); - assert_eq!(document.blocks.len(), 3); - assert!(matches!(document.blocks[1], Block::Table(_))); - assert!(matches!(document.blocks[2], Block::Equation(_))); - op.undo(&mut document); - assert_eq!(document.blocks.len(), 1); - op.redo(&mut document); - assert_eq!(document.blocks.len(), 3); -} - -#[test] -#[hotpath::measure] -fn replace_block_operation_undo_redo_preserves_table_shape_changes() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - let before = Block::Table(TableBlock { - rows: vec![TableRow { - cells: vec![TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }, - text: String::new(), - })], - row_span: 1, - col_span: 1, - }], - }], - column_widths: vec![TableColumnWidth::Fraction(1)], - style: TableStyle { header_row: false }, - version: 0, - }); - let mut after = before.clone(); - let Block::Table(table) = &mut after else { - unreachable!(); - }; - table.rows.push(table.rows[0].clone()); - table.version = 1; - document.blocks = std::sync::Arc::new(vec![Block::Paragraph(document.paragraphs[0].clone()), before.clone()]); - - let op = EditOperation::ReplaceBlock { block_ix: 1, before, after }; - op.redo(&mut document); - let Block::Table(table) = &document.blocks[1] else { - panic!("expected table"); - }; - assert_eq!(table.rows.len(), 2); - op.undo(&mut document); - let Block::Table(table) = &document.blocks[1] else { - panic!("expected table"); - }; - assert_eq!(table.rows.len(), 1); -} - -#[test] -#[hotpath::measure] -fn table_cell_text_edit_is_a_replace_block_history_operation() { - let before = Block::Table(TableBlock { - rows: vec![TableRow { - cells: vec![TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }, - text: String::new(), - })], - row_span: 1, - col_span: 1, - }], - }], - column_widths: vec![TableColumnWidth::Fraction(1)], - style: TableStyle { header_row: false }, - version: 0, - }); - let mut after = before.clone(); - let Block::Table(table) = &mut after else { - unreachable!(); - }; - let TableCellBlock::Paragraph(paragraph) = &mut table.rows[0].cells[0].blocks[0] else { - unreachable!(); - }; - paragraph.text = "cell".to_string(); - paragraph.paragraph.byte_range = 0.."cell".len(); - paragraph.paragraph.runs = vec![TextRun { - len: "cell".len(), - styles: RunStyles::default(), - }]; - table.version = 1; - - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - document.blocks = std::sync::Arc::new(vec![Block::Paragraph(document.paragraphs[0].clone()), before.clone()]); - let op = EditOperation::ReplaceBlock { block_ix: 1, before, after }; - op.redo(&mut document); - let Block::Table(table) = &document.blocks[1] else { - panic!("expected table"); - }; - let TableCellBlock::Paragraph(paragraph) = &table.rows[0].cells[0].blocks[0] else { - panic!("expected paragraph"); - }; - assert_eq!(paragraph.text, "cell"); - op.undo(&mut document); - let Block::Table(table) = &document.blocks[1] else { - panic!("expected table"); - }; - let TableCellBlock::Paragraph(paragraph) = &table.rows[0].cells[0].blocks[0] else { - panic!("expected paragraph"); - }; - assert!(paragraph.text.is_empty()); -} - -#[test] -#[hotpath::measure] -fn replace_block_operation_undo_redo_preserves_equation_source_changes() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - let before = Block::Equation(EquationBlock { - source: "x".into(), - syntax: EquationSyntax::Latex, - display: EquationDisplay::Display, - version: 0, - }); - let after = Block::Equation(EquationBlock { - source: "x+1".into(), - syntax: EquationSyntax::Latex, - display: EquationDisplay::Display, - version: 1, - }); - document.blocks = std::sync::Arc::new(vec![Block::Paragraph(document.paragraphs[0].clone()), before.clone()]); - let op = EditOperation::ReplaceBlock { block_ix: 1, before, after }; - op.redo(&mut document); - let Block::Equation(equation) = &document.blocks[1] else { - panic!("expected equation"); - }; - assert_eq!(equation.source.as_ref(), "x+1"); - op.undo(&mut document); - let Block::Equation(equation) = &document.blocks[1] else { - panic!("expected equation"); - }; - assert_eq!(equation.source.as_ref(), "x"); -} - -#[test] -#[hotpath::measure] -fn default_inserted_table_shape_round_trips_through_document() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - let table = Block::Table(TableBlock { - rows: (0..2) - .map(|_| TableRow { - cells: (0..2) - .map(|_| TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0..0, - runs: Vec::new(), - version: 0, - }, - text: String::new(), - })], - row_span: 1, - col_span: 1, - }) - .collect(), - }) - .collect(), - column_widths: vec![TableColumnWidth::Fraction(1), TableColumnWidth::Fraction(1)], - style: TableStyle { header_row: false }, - version: 0, - }); - document.blocks = std::sync::Arc::new(vec![Block::Paragraph(document.paragraphs[0].clone()), table]); - - let path = std::env::temp_dir().join(format!("flowtext-default-table-{}.document", uuid::Uuid::new_v4())); - write_document(&path, &document).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - let Block::Table(table) = &loaded.blocks[1] else { - panic!("expected table block"); - }; - assert_eq!(table.rows.len(), 2); - assert!(table.rows.iter().all(|row| row.cells.len() == 2)); - assert_eq!(table.column_widths.len(), 2); -} - -#[test] -#[hotpath::measure] -fn table_cell_paragraph_clipboard_conversion_preserves_text_and_styles() { - let styles = RunStyles::default().with(RunStyle::Semantic(2)); - let paragraph = InputParagraph { - style: ParagraphStyle::Custom(3), - runs: vec![InputRun { - text: "cell text".to_string(), - styles, - }], - }; - let cell = table_cell_paragraph_from_input_paragraph(¶graph); - assert_eq!(cell.text, "cell text"); - assert_eq!(cell.paragraph.style, ParagraphStyle::Custom(3)); - assert_eq!(cell.paragraph.runs[0].styles, styles); - - let restored = input_paragraph_from_table_cell_paragraph(&cell); - assert_eq!(input_paragraph_text(&restored), "cell text"); - assert_eq!(restored.style, ParagraphStyle::Custom(3)); - assert_eq!(restored.runs[0].styles, styles); -} - -#[test] -#[hotpath::measure] -fn splitting_table_cell_paragraph_preserves_text_and_run_styles() { - let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); - let mut cell = TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0.."alpha beta".len(), - runs: vec![ - TextRun { - len: "alpha ".len(), - styles: RunStyles::default(), - }, - TextRun { - len: "beta".len(), - styles: emphasized, - }, - ], - version: 0, - }, - text: "alpha beta".to_string(), - })], - row_span: 1, - col_span: 1, - }; - - let new_ix = split_table_cell_paragraph_at(&mut cell, 0, "alpha ".len()).unwrap(); - assert_eq!(new_ix, 1); - - let TableCellBlock::Paragraph(left) = &cell.blocks[0] else { - panic!("expected left paragraph"); - }; - let TableCellBlock::Paragraph(right) = &cell.blocks[1] else { - panic!("expected right paragraph"); - }; - - assert_eq!(left.text, "alpha "); - assert_eq!(right.text, "beta"); - assert_eq!(left.paragraph.runs[0].styles, RunStyles::default()); - assert_eq!(right.paragraph.runs[0].styles, emphasized); - assert_eq!(left.paragraph.byte_range, 0.."alpha ".len()); - assert_eq!(right.paragraph.byte_range, 0.."beta".len()); -} - -#[test] -#[hotpath::measure] -fn merging_table_cell_paragraphs_preserves_boundary_caret_and_styles() { - let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); - let mut cell = TableCell { - blocks: vec![ - TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0.."left".len(), - runs: vec![TextRun { - len: "left".len(), - styles: RunStyles::default(), - }], - version: 0, - }, - text: "left".to_string(), - }), - TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0.."right".len(), - runs: vec![TextRun { - len: "right".len(), - styles: emphasized, - }], - version: 0, - }, - text: "right".to_string(), - }), - ], - row_span: 1, - col_span: 1, - }; - - let (paragraph_ix, caret) = merge_table_cell_paragraph_with_previous(&mut cell, 1).unwrap(); - assert_eq!((paragraph_ix, caret), (0, "left".len())); - assert_eq!(cell.blocks.len(), 1); - - let TableCellBlock::Paragraph(merged) = &cell.blocks[0] else { - panic!("expected merged paragraph"); - }; - assert_eq!(merged.text, "leftright"); - assert_eq!(merged.paragraph.runs.len(), 2); - assert_eq!(merged.paragraph.runs[0].styles, RunStyles::default()); - assert_eq!(merged.paragraph.runs[1].styles, emphasized); -} - -#[test] -#[hotpath::measure] -fn inserting_rich_paragraphs_into_table_cell_preserves_tail_and_styles() { - let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); - let cite = RunStyles::default().with(RunStyle::Semantic(1)); - let mut cell = TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0.."alpha omega".len(), - runs: vec![ - TextRun { - len: "alpha ".len(), - styles: RunStyles::default(), - }, - TextRun { - len: "omega".len(), - styles: emphasized, - }, - ], - version: 0, - }, - text: "alpha omega".to_string(), - })], - row_span: 1, - col_span: 1, - }; - - let inserted = vec![ - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![InputRun { - text: "B".to_string(), - styles: cite, - }], - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![InputRun { - text: "C".to_string(), - styles: cite, - }], - }, - ]; - - let caret = insert_table_cell_paragraphs_at(&mut cell, 0, "alpha ".len(), &inserted).unwrap(); - assert_eq!(caret, (1, "C".len())); - assert_eq!(cell.blocks.len(), 2); - - let TableCellBlock::Paragraph(first) = &cell.blocks[0] else { - panic!("expected first paragraph"); - }; - let TableCellBlock::Paragraph(second) = &cell.blocks[1] else { - panic!("expected second paragraph"); - }; - assert_eq!(first.text, "alpha B"); - assert_eq!(second.text, "Comega"); - assert_eq!(first.paragraph.runs.last().unwrap().styles, cite); - assert_eq!(second.paragraph.runs[0].styles, cite); - assert_eq!(second.paragraph.runs[1].styles, emphasized); -} diff --git a/crates/gpui-flowtext/src/rich_text/tests/layout_scaling.rs b/crates/gpui-flowtext/src/rich_text/tests/layout_scaling.rs new file mode 100644 index 00000000..6b06c7f6 --- /dev/null +++ b/crates/gpui-flowtext/src/rich_text/tests/layout_scaling.rs @@ -0,0 +1,203 @@ +// Class 3 — the render/layout pipeline must stay VIRTUALIZED: layout work per pass is +// bounded by the viewport, not the document size. +// +// The impact-doc field logs showed `render-layout-pass` stalling with `layout_gen` many +// generations behind `edit_gen`, re-laying-out the visible window every frame on a +// ~6000-paragraph document. If virtualization breaks (the editor exact-lays-out every +// paragraph), layout cost grows with document size and the render thread can never catch +// up. These tests drive real layout via `benchmark_paragraph_item_sizes` in a sized window +// and assert the exact-height / prep work is bounded and does not scale with the document. + +fn big_document(paragraphs: usize) -> DocumentProjection { + let paras = (0..paragraphs) + .map(|ix| InputParagraph { + style: if ix % 40 == 0 { ParagraphStyle::Custom(2) } else { ParagraphStyle::Normal }, + runs: vec![plain( + "Body paragraph carrying several words so the layout has real text to measure and reflow.", + )], + }) + .collect(); + document_from_input(DocumentTheme::default(), paras) +} + +/// §perf-heaven T7.16: content the ASCII `big_document` never exercises — a CJK +/// paragraph (multibyte: byte length ≫ char count), a very-long-word paragraph +/// (word wrap can't break inside it), a code-like line, and an empty paragraph. +/// This is what guards the T7.17 byte-vs-char estimate fix: a multibyte estimate +/// measured in bytes would over-shoot the exact height ~3–4×, tripping the net's +/// over-shoot bound. +fn varied_document(repeats: usize) -> DocumentProjection { + // The first sample is a LONG pure-CJK paragraph (every char is 3 UTF-8 bytes, + // and it wraps over many lines) so a byte-length estimate over-shoots the exact + // height by ~3× — decisively past the net's 2× over-shoot bound. The shorter + // samples alone stay under it (the base +1 line dampens a one-liner), so this + // long paragraph is what makes the net actually TRIP on a T7.17 regression. + let long_cjk = "日本語の長い段落を用意して、文字数とバイト数の差が高さ推定に与える影響を確かめます。".repeat(6); + let samples: [String; 6] = [ + long_cjk, + "Supercalifragilisticexpialidocious-antidisestablishmentarianism-pneumonoultramicroscopicsilicovolcanoconiosis".to_string(), + "let chars_per_line = ((content_width / avg_char_width * WORD_WRAP_FILL).floor() as usize).max(1);".to_string(), + "Ordinary body paragraph carrying several words so the layout has real text to measure and reflow across lines.".to_string(), + "Mixed 混合 content with ASCII and 日本語 together on one line to stress the char-vs-byte estimate boundary.".to_string(), + String::new(), + ]; + let paras = (0..repeats) + .flat_map(|_| { + samples.iter().map(|text| InputParagraph { + style: ParagraphStyle::Normal, + runs: if text.is_empty() { Vec::new() } else { vec![plain(text)] }, + }) + }) + .collect(); + document_from_input(DocumentTheme::default(), paras) +} + +/// Build an editor in a sized window, force a cold layout, and return the pass metrics. +fn cold_layout_metrics(cx: &mut gpui::TestAppContext, paragraphs: usize) -> ItemSizeBenchmarkResult { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(big_document(paragraphs), None, cx)); + handle + .update(cx, |editor, window, cx| { + editor.benchmark_invalidate_document_layout_caches(); + editor.benchmark_paragraph_item_sizes(px(760.0), window, cx) + }) + .expect("windowed layout pass") +} + +/// A cold layout of a 3000-paragraph document must exact-lay-out far fewer paragraphs than +/// the document holds — the rest are virtualized with estimated heights. +#[gpui::test] +fn cold_layout_exact_heights_are_far_fewer_than_the_document(cx: &mut gpui::TestAppContext) { + let paragraphs = 3000; + let metrics = cold_layout_metrics(cx, paragraphs); + assert!( + metrics.exact_height_count < paragraphs / 3, + "cold layout exact-laid-out {} of {paragraphs} paragraphs — virtualization is not bounding layout to the viewport", + metrics.exact_height_count, + ); + assert!( + metrics.prep_installed < paragraphs / 3, + "cold layout prepped {} of {paragraphs} paragraph chunks — layout prep is not virtualized", + metrics.prep_installed, + ); +} + +/// Tripling the document size with the SAME viewport must not multiply the exact-height / +/// prep work — layout stays O(viewport), not O(document). A virtualization regression that +/// lays out every paragraph would blow the exact-height count up ~3×. +#[gpui::test] +fn layout_exact_height_work_does_not_scale_with_document_size(cx: &mut gpui::TestAppContext) { + let small = cold_layout_metrics(cx, 2000); + let large = cold_layout_metrics(cx, 6000); + + let exact_ceiling = small.exact_height_count.max(64) * 2; + assert!( + large.exact_height_count <= exact_ceiling, + "exact-height layout scaled with document size (virtualization regression): 2000-para={} vs 6000-para={} (ceiling {exact_ceiling})", + small.exact_height_count, + large.exact_height_count, + ); + let prep_ceiling = small.prep_installed.max(64) * 2; + assert!( + large.prep_installed <= prep_ceiling, + "layout prep scaled with document size: 2000-para={} vs 6000-para={} (ceiling {prep_ceiling})", + small.prep_installed, + large.prep_installed, + ); +} + +/// §perf-heaven T5 NET (the layout-fidelity oracle): the per-paragraph height +/// ESTIMATE must track the exact height within a bounded band. Under-shooting is +/// the dangerous case — scroll jumps UP as real heights land below the caret — so +/// it is bounded tightly; over-shooting only wastes space. Layout heights are not +/// covered by the CRDT convergence fuzz or the corpus sweep, so THIS is the net +/// that guards the estimate heuristic and any future persisted-estimate work +/// (a persisted estimate that diverges from a fresh one trips here). +#[gpui::test] +fn estimate_tracks_exact_height_within_bounds(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(big_document(400), None, cx)); + let acc = handle + .update(cx, |editor, window, cx| { + editor.benchmark_invalidate_document_layout_caches(); + editor.benchmark_estimate_accuracy(px(760.0), window, cx) + }) + .expect("windowed layout pass"); + eprintln!( + "T5 estimate-accuracy net: compared={} max_over={:.3} max_under={:.3}", + acc.compared, acc.max_over_ratio, acc.max_under_ratio, + ); + assert!(acc.compared >= 8, "net must compare real laid-out paragraphs (else it's vacuous), got {}", acc.compared); + assert!( + acc.max_under_ratio < 0.2, + "estimate UNDER-shot exact height by >20% — scroll would jump up (T5 heuristic/word-wrap regression): {:.3}", + acc.max_under_ratio, + ); + assert!( + acc.max_over_ratio < 2.0, + "estimate OVER-shot exact height by >2x (T5 heuristic regression): {:.3}", + acc.max_over_ratio, + ); +} + +/// §perf-heaven T7.16/T7.17 NET: the estimate must stay within the same bounds +/// on VARIED content — crucially multibyte (CJK) paragraphs, where measuring the +/// text in bytes instead of characters over-shoots the exact height ~3–4×. With +/// the byte-length estimate this trips the over-shoot bound; with the char-count +/// fix it holds. +#[gpui::test] +fn estimate_tracks_exact_height_for_varied_content(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(varied_document(70), None, cx)); + let acc = handle + .update(cx, |editor, window, cx| { + editor.benchmark_invalidate_document_layout_caches(); + editor.benchmark_estimate_accuracy(px(760.0), window, cx) + }) + .expect("windowed layout pass"); + eprintln!( + "T7.16 varied-content estimate-accuracy net: compared={} max_over={:.3} max_under={:.3}", + acc.compared, acc.max_over_ratio, acc.max_under_ratio, + ); + assert!(acc.compared >= 8, "net must compare real laid-out paragraphs (else it's vacuous), got {}", acc.compared); + assert!( + acc.max_under_ratio < 0.2, + "varied-content estimate UNDER-shot exact height by >20% (scroll would jump up): {:.3}", + acc.max_under_ratio, + ); + // Tighter than the general 2.0 bound: on this multibyte-heavy fixture the + // char-count estimate (T7.17) holds max_over ≈ 0, while the byte-length + // regression pushes it to ≈1.27 (the Latin-calibrated `avg_char_width` only + // partially absorbs the 3-bytes-per-CJK-char inflation, so it never reaches + // 2×). A 1.0 bound cleanly separates the two — this is the assertion that + // actually TRIPS on a T7.17 regression, proven by reverting `chars().count()` + // to `.len()` (→ max_over ≈ 1.27, fails). + assert!( + acc.max_over_ratio < 1.0, + "varied-content estimate OVER-shot exact height by >1x — T7.17 byte-vs-char regression on multibyte text: {:.3}", + acc.max_over_ratio, + ); +} + +/// G22 measurement — time the cold layout + shape pass (the CPU side of scroll: +/// `paint_line_text`/`shape_fragment`) at increasing document sizes. Run with +/// `--nocapture`. Because layout is viewport-virtualized, the pass time should stay +/// roughly FLAT as the document grows; a rising number is a virtualization regression. +#[gpui::test] +fn g22_cold_layout_pass_timing(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + for paragraphs in [2000_usize, 6000, 12000] { + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(big_document(paragraphs), None, cx)); + let (ms, exact, prep) = handle + .update(cx, |editor, window, cx| { + editor.benchmark_invalidate_document_layout_caches(); + let started = std::time::Instant::now(); + let metrics = editor.benchmark_paragraph_item_sizes(px(760.0), window, cx); + (started.elapsed().as_secs_f64() * 1000.0, metrics.exact_height_count, metrics.prep_installed) + }) + .expect("windowed layout pass"); + eprintln!("G22 cold-layout {paragraphs:>6} paras: {ms:8.2} ms (exact_heights={exact}, prep={prep})"); + assert!(exact < paragraphs, "layout must stay virtualized (exact-laid {exact} of {paragraphs})"); + } +} + diff --git a/crates/gpui-flowtext/src/rich_text/tests/mod.rs b/crates/gpui-flowtext/src/rich_text/tests/mod.rs index cebe5a31..81ed8bc8 100644 --- a/crates/gpui-flowtext/src/rich_text/tests/mod.rs +++ b/crates/gpui-flowtext/src/rich_text/tests/mod.rs @@ -1,7 +1,19 @@ +// Shared scaffolding for the include!-spliced suites below. The split files +// are textually part of THIS module, so all imports live here — no `use` +// statements inside the included files. +// +// Loro-first invariant 5: an editor WITHOUT a write authority is read-only, so +// every MUTATING behavior test lives in tests/editor_behavior.rs (integration +// target), where the real flowstate-collab write authority attaches without +// the unit-test dual-crate-instance problem. Suites here are pure layout / +// hit-testing / selection geometry. + +use super::*; +use gpui::{Bounds, black, point, px, size}; + include!("edit_layout.rs"); include!("selection.rs"); -include!("persistence_blocks.rs"); -include!("history_tables.rs"); include!("positions_formatting.rs"); -include!("history_fragments.rs"); include!("decorations_drag_search.rs"); +include!("layout_scaling.rs"); +include!("prep_equivalence.rs"); diff --git a/crates/gpui-flowtext/src/rich_text/tests/persistence_blocks.rs b/crates/gpui-flowtext/src/rich_text/tests/persistence_blocks.rs deleted file mode 100644 index eacb1739..00000000 --- a/crates/gpui-flowtext/src/rich_text/tests/persistence_blocks.rs +++ /dev/null @@ -1,385 +0,0 @@ - -#[test] -#[hotpath::measure] -fn single_paragraph_edits_keep_following_derived_byte_ranges_current() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("first")], - }, - InputParagraph { - style: ParagraphStyle::Custom(0), - runs: vec![plain("second")], - }, - ], - ); - - insert_text_at(&mut document, 0, "first".len(), " extended", RunStyles::default()); - - let range = paragraph_byte_range(&document, 1); - assert_eq!(document_text_slice(&document, range.clone()), "second"); - assert!(range.end <= document.text.byte_len()); -} - -#[test] -#[hotpath::measure] -fn document_round_trip_preserves_text_structure_and_styles() { - let document = demo_document(); - let dir = std::env::temp_dir(); - let path = dir.join(format!("flowtext-test-{}.document", std::process::id())); - write_document(&path, &document).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - assert_eq!( - document_text_slice(&document, 0..document.text.byte_len()), - document_text_slice(&loaded, 0..loaded.text.byte_len()) - ); - assert_eq!(document.paragraphs.len(), loaded.paragraphs.len()); - // Verify styles and run structure for every paragraph, not just the first. - for (ix, (orig, loaded_para)) in document - .paragraphs - .iter() - .zip(loaded.paragraphs.iter()) - .enumerate() - { - assert_eq!(orig.style, loaded_para.style, "paragraph {ix} style mismatch"); - assert_eq!(orig.runs, loaded_para.runs, "paragraph {ix} runs mismatch"); - } -} - -#[test] -#[hotpath::measure] -fn split_and_merge_preserve_empty_styled_paragraphs() { - let spoken = RunStyles::default().with(RunStyle::Highlight(1)); - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Custom(0), - runs: vec![run("Custom heading", spoken)], - }], - ); - - let first_len = paragraph_text_len(&document.paragraphs[0]); - split_paragraph_at(&mut document, 0, first_len); - assert_eq!(document.paragraphs.len(), 2); - assert_eq!(document.paragraphs[1].style, ParagraphStyle::Custom(0)); - assert_eq!(paragraph_text_len(&document.paragraphs[1]), 0); - assert!(document.paragraphs[1].runs.is_empty()); - - let join_byte = paragraph_text_len(&document.paragraphs[0]); - delete_cross_paragraph_range( - &mut document, - DocumentOffset { - paragraph: 0, - byte: join_byte, - }..DocumentOffset { paragraph: 1, byte: 0 }, - ); - assert_eq!(document.paragraphs.len(), 1); - assert_eq!(paragraph_text(&document, 0), "Custom heading"); - assert_eq!( - document.paragraphs[0].runs, - vec![TextRun { - len: "Custom heading".len(), - styles: spoken - }] - ); -} - -#[test] -#[hotpath::measure] -fn document_round_trip_preserves_empty_styled_paragraphs() { - let document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Custom(0), - runs: Vec::new(), - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }, - ], - ); - let path = std::env::temp_dir().join(format!("flowtext-empty-{}.document", std::process::id())); - write_document(&path, &document).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - assert_eq!(loaded.paragraphs[0].style, ParagraphStyle::Custom(0)); - assert_eq!(paragraph_text_len(&loaded.paragraphs[0]), 0); - assert!(loaded.paragraphs[0].runs.is_empty()); - assert_eq!(paragraph_text(&loaded, 1), "body"); -} - -#[test] -#[hotpath::measure] -fn document_v4_round_trip_preserves_mixed_block_order_and_assets() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Custom(0), - runs: vec![plain("Heading")], - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("After image")], - }, - ], - ); - let asset_id = AssetId(42); - let asset_bytes = vec![1, 2, 3, 4]; - let mut hasher = DefaultHasher::new(); - asset_bytes.hash(&mut hasher); - document.assets.assets.insert( - asset_id, - AssetRecord { - id: asset_id, - mime_type: "image/png".into(), - original_name: Some("figure.png".into()), - content_hash: hasher.finish(), - bytes: std::sync::Arc::new(asset_bytes), - }, - ); - document.blocks = std::sync::Arc::new(vec![ - Block::Paragraph(document.paragraphs[0].clone()), - Block::Image(ImageBlock { - asset_id, - alt_text: "figure".into(), - caption: None, - sizing: ImageSizing::FitWidth, - alignment: BlockAlignment::Center, - version: 0, - }), - Block::Paragraph(document.paragraphs[1].clone()), - Block::Equation(EquationBlock { - source: "x^2 + y^2 = z^2".into(), - syntax: EquationSyntax::Latex, - display: EquationDisplay::Display, - version: 0, - }), - ]); - - let path = std::env::temp_dir().join(format!("flowtext-blocks-{}.document", uuid::Uuid::new_v4())); - write_document(&path, &document).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - assert_eq!(loaded.blocks.len(), 4); - assert!(matches!(loaded.blocks[0], Block::Paragraph(_))); - assert!(matches!(loaded.blocks[1], Block::Image(_))); - assert!(matches!(loaded.blocks[2], Block::Paragraph(_))); - assert!(matches!(loaded.blocks[3], Block::Equation(_))); - assert_eq!(loaded.assets.assets[&asset_id].bytes.as_slice(), &[1, 2, 3, 4]); -} - -#[test] -#[hotpath::measure] -fn image_fit_width_layout_uses_asset_aspect_ratio() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - let asset_id = AssetId(7); - document.assets.assets.insert( - asset_id, - AssetRecord { - id: asset_id, - mime_type: "image/png".into(), - original_name: None, - content_hash: 0, - bytes: std::sync::Arc::new(test_png_2x1()), - }, - ); - let image = ImageBlock { - asset_id, - alt_text: "".into(), - caption: None, - sizing: ImageSizing::FitWidth, - alignment: BlockAlignment::Left, - version: 0, - }; - - let width = document.theme.pageless_inset_x * 2.0 + px(200.0); - assert_eq!(image_layout_height_for_test(&document, &image, width), px(100.0)); -} - -#[test] -#[hotpath::measure] -fn paragraph_sync_preserves_non_text_blocks_when_paragraphs_are_removed() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("before")], - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("after")], - }, - ], - ); - let image = Block::Image(ImageBlock { - asset_id: AssetId(99), - alt_text: "image".into(), - caption: None, - sizing: ImageSizing::FitWidth, - alignment: BlockAlignment::Center, - version: 0, - }); - document.blocks = std::sync::Arc::new(vec![ - Block::Paragraph(document.paragraphs[0].clone()), - image.clone(), - Block::Paragraph(document.paragraphs[1].clone()), - ]); - - let current = capture_document_span(&document, 0..2); - apply_document_span_replacement( - &mut document, - ¤t, - &DocumentSpan { - start_paragraph: 0, - text: "after".to_string(), - paragraphs: vec![Paragraph { - style: ParagraphStyle::Normal, - byte_range: 0.."after".len(), - runs: vec![TextRun { - len: "after".len(), - styles: RunStyles::default(), - }], - version: 0, - }], - }, - ); - - assert_eq!(document.paragraphs.len(), 1); - assert!( - document - .blocks - .iter() - .any(|block| matches!(block, Block::Image(_))) - ); -} - -#[test] -#[hotpath::measure] -fn deleting_empty_paragraph_above_image_keeps_image_before_next_paragraph() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("before")], - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: Vec::new(), - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("after")], - }, - ], - ); - let image = Block::Image(ImageBlock { - asset_id: AssetId(100), - alt_text: "image".into(), - caption: None, - sizing: ImageSizing::FitWidth, - alignment: BlockAlignment::Center, - version: 0, - }); - document.blocks = std::sync::Arc::new(vec![ - Block::Paragraph(document.paragraphs[0].clone()), - Block::Paragraph(document.paragraphs[1].clone()), - image, - Block::Paragraph(document.paragraphs[2].clone()), - ]); - - delete_cross_paragraph_range( - &mut document, - DocumentOffset { - paragraph: 0, - byte: "before".len(), - }..DocumentOffset { paragraph: 1, byte: 0 }, - ); - - assert_eq!(document.paragraphs.len(), 2); - assert!(matches!(document.blocks[0], Block::Paragraph(_))); - assert!(matches!(document.blocks[1], Block::Image(_))); - assert!(matches!(document.blocks[2], Block::Paragraph(_))); -} - -#[hotpath::measure] -fn test_png_2x1() -> Vec { - vec![ - 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 2, 0, 0, 0, 1, 8, 6, 0, 0, 0, 244, 34, 127, 138, 0, 0, 0, 12, 73, 68, - 65, 84, 8, 29, 99, 248, 15, 4, 0, 9, 251, 3, 253, 167, 170, 43, 113, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, - ] -} - -#[test] -#[hotpath::measure] -fn document_v4_round_trip_preserves_table_cell_paragraph_and_run_styles() { - let emphasized = RunStyles::default() - .with(RunStyle::Semantic(2)) - .with(RunStyle::Highlight(1)); - let cell_paragraph = Paragraph { - style: ParagraphStyle::Custom(3), - byte_range: 0.."cell".len(), - runs: vec![TextRun { - len: "cell".len(), - styles: emphasized, - }], - version: 0, - }; - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("before")], - }], - ); - document.blocks = std::sync::Arc::new(vec![ - Block::Paragraph(document.paragraphs[0].clone()), - Block::Table(TableBlock { - rows: vec![TableRow { - cells: vec![TableCell { - blocks: vec![TableCellBlock::Paragraph(TableCellParagraph { - paragraph: cell_paragraph.clone(), - text: "cell".to_string(), - })], - row_span: 1, - col_span: 1, - }], - }], - column_widths: vec![TableColumnWidth::Fraction(1)], - style: TableStyle { header_row: true }, - version: 0, - }), - ]); - - let path = std::env::temp_dir().join(format!("flowtext-table-{}.document", uuid::Uuid::new_v4())); - write_document(&path, &document).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - let Block::Table(table) = &loaded.blocks[1] else { - panic!("expected table block"); - }; - assert!(table.style.header_row); - let TableCellBlock::Paragraph(loaded_paragraph) = &table.rows[0].cells[0].blocks[0] else { - panic!("expected table-cell paragraph"); - }; - assert_eq!(loaded_paragraph.paragraph.style, ParagraphStyle::Custom(3)); - assert_eq!(loaded_paragraph.paragraph.runs, cell_paragraph.runs); - assert_eq!(loaded_paragraph.text, "cell"); -} diff --git a/crates/gpui-flowtext/src/rich_text/tests/positions_formatting.rs b/crates/gpui-flowtext/src/rich_text/tests/positions_formatting.rs index a207de5c..3e7b022b 100644 --- a/crates/gpui-flowtext/src/rich_text/tests/positions_formatting.rs +++ b/crates/gpui-flowtext/src/rich_text/tests/positions_formatting.rs @@ -15,7 +15,7 @@ fn document_position_round_trips_top_level_text_blocks() { }, ], ); - document.blocks = std::sync::Arc::new(vec![ + document.blocks = crate::BlockSeq::from_vec(vec![ Block::Paragraph(document.paragraphs[0].clone()), Block::Image(ImageBlock { asset_id: AssetId(42), @@ -23,6 +23,7 @@ fn document_position_round_trips_top_level_text_blocks() { caption: None, sizing: ImageSizing::Intrinsic, alignment: BlockAlignment::Center, + external_url: None, version: 0, }), Block::Paragraph(document.paragraphs[1].clone()), @@ -44,37 +45,6 @@ fn document_position_round_trips_top_level_text_blocks() { ); } -#[test] -#[hotpath::measure] -fn document_validation_rejects_zero_sized_fixed_images() { - let mut document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("body")], - }], - ); - document.blocks = std::sync::Arc::new(vec![ - Block::Paragraph(document.paragraphs[0].clone()), - Block::Image(ImageBlock { - asset_id: AssetId(99), - alt_text: "invalid".into(), - caption: None, - sizing: ImageSizing::Fixed { - width_px: 0, - height_px: None, - }, - alignment: BlockAlignment::Left, - version: 0, - }), - ]); - - let path = std::env::temp_dir().join(format!("flowtext-invalid-image-{}.document", uuid::Uuid::new_v4())); - let error = write_document(&path, &document).unwrap_err(); - assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); - let _ = std::fs::remove_file(path); -} - #[test] #[hotpath::measure] fn double_click_at_text_paragraph_end_selects_only_that_paragraph() { @@ -102,13 +72,13 @@ fn double_click_at_text_paragraph_end_selects_only_that_paragraph() { assert_eq!( selection, - EditorSelection { - anchor: DocumentOffset { paragraph: 0, byte: 0 }, - head: DocumentOffset { + EditorSelection::range( + DocumentOffset { paragraph: 0, byte: 0 }, + DocumentOffset { paragraph: 0, byte: "first paragraph".len(), }, - } + ) ); } @@ -137,52 +107,10 @@ fn double_click_empty_paragraph_selects_only_empty_paragraph() { assert_eq!( selection, - EditorSelection { - anchor: DocumentOffset { paragraph: 1, byte: 0 }, - head: DocumentOffset { paragraph: 1, byte: 0 }, - } + EditorSelection::collapsed(DocumentOffset { paragraph: 1, byte: 0 }) ); } -#[test] -#[hotpath::measure] -fn selection_across_empty_paragraphs_and_clear_formatting_policy() { - let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); - let mut document = document_from_input( - DocumentTheme::default(), - vec![ - InputParagraph { - style: ParagraphStyle::Custom(3), - runs: vec![run("tag", emphasized)], - }, - InputParagraph { - style: ParagraphStyle::Custom(0), - runs: Vec::new(), - }, - InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![run("body", emphasized)], - }, - ], - ); - let selection = DocumentOffset { paragraph: 0, byte: 1 }..DocumentOffset { paragraph: 2, byte: 1 }; - assert!(selection_contains_whole_paragraph(&document, selection.clone())); - - for paragraph_ix in selection.start.paragraph..=selection.end.paragraph { - clear_whole_paragraph_formatting(&mut document, paragraph_ix); - } - - for paragraph in document.paragraphs.iter() { - assert_eq!(paragraph.style, ParagraphStyle::Normal); - assert!( - paragraph - .runs - .iter() - .all(|run| run.styles == RunStyles::default()) - ); - } -} - #[test] #[hotpath::measure] fn run_style_full_selection_toggle_policy() { @@ -222,58 +150,3 @@ fn semantic_run_styles_are_mutually_exclusive() { styles.apply(RunStyle::Semantic(5)); assert_eq!(styles.semantic, RunSemanticStyle::Custom(5)); } - -#[test] -#[hotpath::measure] -fn document_round_trip_preserves_condensed_semantic_styles() { - let path = std::env::temp_dir().join(format!("flowtext-semantic-{}.document", uuid::Uuid::new_v4())); - let document = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![ - run("condensed", RunStyles::default().with(RunStyle::Semantic(4))), - run( - " ultra", - RunStyles::default() - .with(RunStyle::Semantic(5)) - .with(RunStyle::Highlight(1)), - ), - ], - }], - ); - write_document(&path, &document).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - assert_eq!(loaded.paragraphs[0].runs[0].styles.semantic, RunSemanticStyle::Custom(4)); - assert_eq!(loaded.paragraphs[0].runs[1].styles.semantic, RunSemanticStyle::Custom(5)); - assert_eq!(loaded.paragraphs[0].runs[1].styles.highlight, Some(HighlightStyle::Custom(1))); -} - -#[test] -#[hotpath::measure] -fn document_save_can_replace_existing_file() { - let path = std::env::temp_dir().join(format!("flowtext-replace-{}.document", uuid::Uuid::new_v4())); - let first = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("first")], - }], - ); - let second = document_from_input( - DocumentTheme::default(), - vec![InputParagraph { - style: ParagraphStyle::Normal, - runs: vec![plain("second")], - }], - ); - - write_document(&path, &first).unwrap(); - write_document(&path, &second).unwrap(); - let loaded = read_document(&path).unwrap(); - let _ = std::fs::remove_file(path); - - assert_eq!(paragraph_text(&loaded, 0), "second"); -} diff --git a/crates/gpui-flowtext/src/rich_text/tests/prep_equivalence.rs b/crates/gpui-flowtext/src/rich_text/tests/prep_equivalence.rs new file mode 100644 index 00000000..82a61be7 --- /dev/null +++ b/crates/gpui-flowtext/src/rich_text/tests/prep_equivalence.rs @@ -0,0 +1,656 @@ +// NOTE: include!()-spliced into tests/mod.rs — imports live there. +// +// §act-nine A9.3 NET — content-keyed layout caches. The prep / shaping / +// chunk / height caches key on paragraph CONTENT (style, version) + stable +// identity, with no global edit generation: an edit to one paragraph must +// keep every other paragraph's cached layout, and a changed paragraph must +// NEVER be served stale. These tests are the oracle for that law: +// +// - cache-served prep must equal a freshly computed prep (built via the same +// `build_paragraph_prep_from_parts` the pipeline uses) for every untouched +// paragraph after an unrelated edit — if a future change misses a prep +// input, fresh and cached diverge and this fails; +// - the edited paragraph's stale prep/chunks/height must not be served; +// - version RESETS (structural rebuilds, canonical installs) must not recreate +// an old (style, version) pair for different content — the version- +// discipline prerequisite, tested through the public install/patch paths; +// - a theme update must invalidate invisibility-mode prep (the theme feeds +// `run_is_visible_for_theme`, which is not part of the prep key); +// - a positional slot reused by a DIFFERENT paragraph that shares +// (style, version) must be a miss, not a stale hit (identity check). + +fn a93_varied_document() -> DocumentProjection { + let cite = RunStyles::default().with(RunStyle::Semantic(1)); + let spoken = RunStyles::default().with(RunStyle::Highlight(1)); + let mut theme = DocumentTheme::default(); + theme.set_invisibility_visible_semantic_style(1); + theme.set_invisibility_visible_highlight_style(1); + document_from_input(theme, vec![ + InputParagraph { + style: ParagraphStyle::Custom(2), + runs: vec![plain("A heading-styled paragraph")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("Plain body text that will be edited in place by the test")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("hidden lead-in "), InputRun { text: "cited evidence".into(), styles: cite }, InputRun { + text: " spoken tail".into(), + styles: spoken, + }], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("混合 multibyte 内容 with soft\u{2028}break and wrap candidates")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + }, + ]) +} + +/// In-place single-paragraph content edit, shaped exactly like the write +/// path's `replace_paragraph_content`: rope splice + run replacement + a +/// version bump + offset/block mirror refresh. Followed by an editor +/// generation bump to prove cache validity does not depend on it. +fn a93_edit_paragraph_in_place(editor: &mut RichTextEditor, paragraph_ix: usize, new_text: &str, cx: &mut gpui::Context) { + let old = editor.document.paragraphs.get(paragraph_ix).expect("edit target").clone(); + let byte_range = crate::edit_ops::paragraph_byte_range(&editor.document, paragraph_ix); + editor.document.text.delete(byte_range.clone()); + editor.document.text.insert(byte_range.start, new_text); + paragraphs_mut(&mut editor.document).set(paragraph_ix, Paragraph { + style: old.style, + runs: vec![TextRun { + len: new_text.len(), + styles: RunStyles::default(), + }], + version: old.version.wrapping_add(1), + }); + update_paragraph_offsets_after_len_change(&mut editor.document, paragraph_ix); + // The old keys bundled `edit_generation`; advancing it here asserts the + // content-keyed caches no longer care. + editor.mark_as_unsaved_branch(cx); +} + +fn a93_fresh_prep(editor: &RichTextEditor, paragraph_ix: usize) -> ParagraphPrep { + let paragraph = editor.document.paragraphs.get(paragraph_ix).expect("paragraph").clone(); + let paragraph_id = editor.document.ids.paragraph_ids[paragraph_ix]; + let byte_range = crate::edit_ops::paragraph_byte_range(&editor.document, paragraph_ix); + build_paragraph_prep_from_parts( + &editor.document.text, + &editor.document.theme, + paragraph_id, + paragraph_ix, + ¶graph, + byte_range, + editor.invisibility_mode(), + ) + .expect("fresh prep") +} + +fn a93_assert_prep_eq(served: &ParagraphPrep, fresh: &ParagraphPrep, paragraph_ix: usize) { + assert_eq!(served.key, fresh.key, "prep key mismatch at paragraph {paragraph_ix}"); + assert_eq!(served.paragraph_id, fresh.paragraph_id, "prep id mismatch at paragraph {paragraph_ix}"); + assert_eq!(served.paragraph_ix, fresh.paragraph_ix, "prep ix mismatch at paragraph {paragraph_ix}"); + assert_eq!(served.paragraph_text, fresh.paragraph_text, "prep text mismatch at paragraph {paragraph_ix}"); + assert_eq!( + served.layout_runs.as_ref(), + fresh.layout_runs.as_ref(), + "prep runs mismatch at paragraph {paragraph_ix}" + ); + assert_eq!(served.layout_style, fresh.layout_style, "prep style mismatch at paragraph {paragraph_ix}"); + assert_eq!(served.layout_version, fresh.layout_version, "prep version mismatch at paragraph {paragraph_ix}"); + assert_eq!(served.source_len, fresh.source_len, "prep source_len mismatch at paragraph {paragraph_ix}"); + assert_eq!( + served.wrap_break_ends.as_ref(), + fresh.wrap_break_ends.as_ref(), + "prep wrap breaks mismatch at paragraph {paragraph_ix}" + ); + assert_eq!(served.visible, fresh.visible, "prep visibility mismatch at paragraph {paragraph_ix}"); +} + +/// Regression ("invisible caret"): a paragraph below the first has a NON-ZERO +/// document byte offset, but its laid-out `byte_range` must be PARAGRAPH-LOCAL +/// (chunk-0 starts at 0). If it's the absolute document range, `contains_byte` +/// compares a paragraph-local caret byte against an absolute range and the caret +/// vanishes for every byte below the paragraph's document offset. +#[gpui::test] +fn later_paragraph_byte_range_is_paragraph_local(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, window, cx| { + let width = px(760.0); + let count = editor.document.paragraphs.len(); + assert!(count >= 2, "fixture must have a paragraph below the first"); + let last = count - 1; + let layout = editor.layout_paragraph_chunk_for_element(last, 0, width, window, cx).expect("chunk layout"); + let para = layout.paragraphs.iter().find(|p| p.index == last).expect("laid-out paragraph"); + assert_eq!( + para.byte_range.start, 0, + "chunk-0 byte_range must be paragraph-local (start 0), not the absolute document offset" + ); + // The caret's containment check must accept the paragraph's interior local + // bytes — a non-empty paragraph must contain byte 0. + assert!(para.contains_byte(0), "interior local byte 0 must be contained (the caret must be able to render there)"); + }) + .expect("window update"); +} + +/// §act-eleven C4 tripwire: an unrelated single-paragraph edit must rebuild a +/// BOUNDED number of preps on the next full relayout — not the whole viewport. +/// The equality tests prove correctness; this proves EFFECTIVENESS (the T8.12 +/// "landed but inert" failure class trips this immediately). +#[gpui::test] +fn single_edit_rebuilds_bounded_prep_count(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, window, cx| { + let width = px(760.0); + let count = editor.document.paragraphs.len(); + for paragraph_ix in 0..count { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + } + // Warm: every paragraph has prep. Now one in-place edit... + crate::rich_text::layout::reset_prep_build_count(); + a93_edit_paragraph_in_place(editor, 1, "tripwire body — bounded rebuilds only", cx); + for paragraph_ix in 0..count { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + } + let rebuilds = crate::rich_text::layout::prep_build_count(); + // The edited paragraph rebuilds (possibly once per visibility variant); + // everything else must be served from the content-keyed cache. 4 is a + // generous bound for ONE edited paragraph; whole-viewport regression on + // this fixture would be >= `count` (all paragraphs). + assert!( + rebuilds <= 4, + "single in-place edit triggered {rebuilds} prep builds (bound 4) — the content-keyed cache has gone inert" + ); + assert!(rebuilds >= 1, "edited paragraph never re-prepped — vacuous tripwire"); + }) + .expect("window update"); +} + +/// Unrelated edits must keep every OTHER paragraph's prep, shaping key, chunk +/// layout, and exact height — and must never serve the edited paragraph stale. +#[gpui::test] +fn unrelated_edit_preserves_other_paragraphs_layout_caches(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, window, cx| { + let width = px(760.0); + let count = editor.document.paragraphs.len(); + // Populate chunk + height + prep + shaping caches through the real + // layout machinery. + for paragraph_ix in 0..count { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + } + let mut before = Vec::new(); + for paragraph_ix in 0..count { + let prep = editor.ensure_paragraph_prep_sync(paragraph_ix).expect("prep populated"); + let work_key = editor.paragraph_work_key(prep.as_ref(), width); + let chunk_cached = editor.valid_chunk_cache_entry(paragraph_ix, width).is_some(); + let height = editor.valid_paragraph_height(paragraph_ix, width); + before.push((prep, work_key, chunk_cached, height)); + } + // Sanity: the visible short paragraphs completed, so heights are cached. + assert!(before.iter().any(|(_, _, _, height)| height.is_some()), "height cache never populated — vacuous test"); + + let edited = 1usize; + a93_edit_paragraph_in_place(editor, edited, "a completely different body after the in-place edit", cx); + + for (paragraph_ix, entry) in before.iter().enumerate().take(count) { + let (prep_before, work_key_before, chunk_before, height_before) = entry; + if paragraph_ix == edited { + assert!( + editor.valid_paragraph_prep(paragraph_ix).is_none(), + "edited paragraph served STALE prep" + ); + assert!( + editor.valid_chunk_cache_entry(paragraph_ix, width).is_none(), + "edited paragraph served STALE chunk layout" + ); + assert!( + editor.valid_paragraph_height(paragraph_ix, width).is_none(), + "edited paragraph served STALE height" + ); + let fresh = editor.ensure_paragraph_prep_sync(paragraph_ix).expect("rebuilt prep"); + assert_eq!(fresh.paragraph_text.as_ref(), "a completely different body after the in-place edit"); + assert_ne!( + editor.paragraph_work_key(fresh.as_ref(), width), + *work_key_before, + "edited paragraph's shaping work key must change" + ); + continue; + } + let served = editor + .valid_paragraph_prep(paragraph_ix) + .unwrap_or_else(|| panic!("unchanged paragraph {paragraph_ix} lost its prep across an unrelated edit")); + assert!( + std::sync::Arc::ptr_eq(&served, prep_before), + "unchanged paragraph {paragraph_ix} was re-prepped instead of cache-served" + ); + // The cache-served prep must equal a prep computed FRESH from the + // current document through the same pipeline function. + let fresh = a93_fresh_prep(editor, paragraph_ix); + a93_assert_prep_eq(&served, &fresh, paragraph_ix); + // Shaping-cache validity decision: the work key (content key + width + + // layout generation) is unchanged, so shaped fragments stay valid. + assert_eq!( + editor.paragraph_work_key(served.as_ref(), width), + *work_key_before, + "unchanged paragraph {paragraph_ix}'s shaping work key changed" + ); + assert_eq!( + editor.valid_chunk_cache_entry(paragraph_ix, width).is_some(), + *chunk_before, + "unchanged paragraph {paragraph_ix} lost its chunk layout" + ); + assert_eq!( + editor.valid_paragraph_height(paragraph_ix, width), + *height_before, + "unchanged paragraph {paragraph_ix} lost its cached height" + ); + } + }) + .expect("windowed layout pass"); +} + +/// The async install gate: a background prep batch completed AFTER an +/// unrelated edit must still install for the untouched paragraphs (content +/// still current) and must discard only the edited paragraph's stale prep. +#[gpui::test] +fn background_prep_batch_installs_for_still_current_content(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, _window, cx| { + let width = px(760.0); + let count = editor.document.paragraphs.len(); + let batch = paragraph_prep_batch_request(&editor.document, false, (0..count).collect(), count.max(1), usize::MAX); + let edited = 1usize; + a93_edit_paragraph_in_place(editor, edited, "edited while the batch was in flight", cx); + let result = build_paragraph_prep_batch(batch); + editor.install_layout_prep_batch(width, result, cx); + for paragraph_ix in 0..count { + if paragraph_ix == edited { + assert!( + editor.valid_paragraph_prep(paragraph_ix).is_none(), + "stale background prep installed for the edited paragraph" + ); + } else { + let served = editor + .valid_paragraph_prep(paragraph_ix) + .unwrap_or_else(|| panic!("background prep for still-current paragraph {paragraph_ix} was discarded")); + a93_assert_prep_eq(&served, &a93_fresh_prep(editor, paragraph_ix), paragraph_ix); + } + } + }) + .expect("windowed pass"); +} + +/// Version-discipline prerequisite (b): a canonical install replaces the +/// projection with all-version-0 materializer output. A surviving paragraph id +/// with DIFFERENT content must not collide with its old (style, version) cache +/// entries — versions carry forward, and no stale prep is ever served. +#[gpui::test] +fn canonical_install_with_reset_versions_never_serves_stale_prep(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, _window, cx| { + let count = editor.document.paragraphs.len(); + for paragraph_ix in 0..count { + let _ = editor.ensure_paragraph_prep_sync(paragraph_ix); + } + let changed = 1usize; + let old_text = editor + .ensure_paragraph_prep_sync(changed) + .expect("prep") + .paragraph_text + .clone(); + + // Materializer-shaped replacement: same ids, same styles, all versions + // RESET to 0, different content for one surviving paragraph. + let cite = RunStyles::default().with(RunStyle::Semantic(1)); + let spoken = RunStyles::default().with(RunStyle::Highlight(1)); + let mut replacement = document_from_input(editor.document.theme.clone(), vec![ + InputParagraph { + style: ParagraphStyle::Custom(2), + runs: vec![plain("A heading-styled paragraph")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("replaced content arriving at version zero")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("hidden lead-in "), InputRun { text: "cited evidence".into(), styles: cite }, InputRun { + text: " spoken tail".into(), + styles: spoken, + }], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("混合 multibyte 内容 with soft\u{2028}break and wrap candidates")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + }, + ]); + assert!(replacement.paragraphs.iter().all(|paragraph| paragraph.version == 0)); + replacement.ids = editor.document.ids.clone(); + + editor.install_canonical_projection(replacement, cx); + + // Surviving ids must have ADVANCED versions — never a reused 0. + for paragraph_ix in 0..count { + assert!( + editor.document.paragraphs.get(paragraph_ix).expect("paragraph").version > 0, + "canonical install reset a surviving paragraph's version to 0 (paragraph {paragraph_ix})" + ); + } + // And whatever the cache now serves must match the CURRENT content. + if let Some(served) = editor.valid_paragraph_prep(changed) { + assert_ne!(served.paragraph_text, old_text, "stale prep served after a canonical install"); + } + let fresh = editor.ensure_paragraph_prep_sync(changed).expect("prep"); + assert_eq!(fresh.paragraph_text.as_ref(), "replaced content arriving at version zero"); + }) + .expect("windowed pass"); +} + +/// Version-discipline prerequisite (a): the structural-batch rebuild path +/// materializes surviving paragraphs through `paragraph_from_input_paragraph` +/// (version 0). Driving a real non-simple patch batch through the public apply +/// path must ADVANCE the surviving paragraph's version and drop its old prep. +#[gpui::test] +fn structural_rebuild_advances_surviving_paragraph_versions(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, _window, cx| { + let target = 2usize; + for paragraph_ix in 0..editor.document.paragraphs.len() { + let _ = editor.ensure_paragraph_prep_sync(paragraph_ix); + } + let old_version = editor.document.paragraphs.get(target).expect("target").version; + let old_text = editor.ensure_paragraph_prep_sync(target).expect("prep").paragraph_text.clone(); + let target_paragraph_id = editor.document.ids.paragraph_ids[target]; + // Paragraph-only document: block row == paragraph index. + let target_block_id = editor.document.ids.block_ids[target]; + // A leading structural insert forces the legacy full-rebuild path (the + // in-place fast path only accepts content patches with one TRAILING + // structural patch), so the trailing content patch flows through the + // rebuilt-blocks arm — the exact `version: 0` hazard site. + let batch = ProjectionPatchBatch { + transaction_id: 1, + base_frontier: editor.document.frontier.clone(), + new_frontier: vec![1], + patches: vec![ + ProjectionPatch::InsertBlocks { + before: None, + row_hint: editor.document.blocks.len(), + blocks: vec![ProjectionStructuralBlock { + block_id: new_block_id(), + paragraph_id: Some(new_paragraph_id()), + block: InputBlock::Paragraph(InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("appended tail paragraph")], + }), + }], + }, + ProjectionPatch::ParagraphText { + block_id: target_block_id, + paragraph_id: target_paragraph_id, + row_hint: target, + new: InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("rebuilt with different content")], + }, + delta_utf8: Vec::new(), + }, + ] + .into(), + }; + editor.apply_remote_patch_batch(&batch, cx).expect("patch batch applies"); + + let rebuilt = editor.document.paragraphs.get(target).expect("target").clone(); + assert!( + rebuilt.version > old_version, + "structural rebuild reset the surviving paragraph's version ({} -> {})", + old_version, + rebuilt.version + ); + if let Some(served) = editor.valid_paragraph_prep(target) { + assert_ne!(served.paragraph_text, old_text, "stale prep served after a structural rebuild"); + } + let fresh = editor.ensure_paragraph_prep_sync(target).expect("prep"); + assert_eq!(fresh.paragraph_text.as_ref(), "rebuilt with different content"); + }) + .expect("windowed pass"); +} + +/// The theme feeds invisibility-mode prep (`run_is_visible_for_theme`) but is +/// not part of the prep key — a theme update must clear the prep map (the +/// pre-existing stale-prep hazard fixed with A9.3). +#[gpui::test] +fn theme_update_invalidates_invisibility_prep(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let cite = RunStyles::default().with(RunStyle::Semantic(1)); + let document = document_from_input(DocumentTheme::default(), vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("hidden "), InputRun { text: "cited".into(), styles: cite }], + }]); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(document, None, cx)); + handle + .update(cx, |editor, _window, cx| { + editor.set_invisibility_mode(true, cx); + let stale = editor.ensure_paragraph_prep_sync(0).expect("prep"); + assert!(!stale.visible, "semantic slot 1 is not visible yet"); + // The theme change flips the same paragraph's projected visibility; + // (style, version) is unchanged, so only a prep-map clear prevents a + // stale hidden prep from being served. + editor.update_document_theme(|theme| theme.set_invisibility_visible_semantic_style(1), cx); + assert!( + editor.valid_paragraph_prep(0).is_none(), + "theme update left stale invisibility prep in the cache" + ); + let fresh = editor.ensure_paragraph_prep_sync(0).expect("prep"); + assert!(fresh.visible, "rebuilt prep must reflect the new theme"); + assert_eq!(fresh.paragraph_text.as_ref(), "cited"); + }) + .expect("windowed pass"); +} + +/// A POSITIONAL cache slot (chunk/height) reused by a DIFFERENT paragraph +/// that happens to share (style, version) must be a MISS — the stable-id +/// check is what makes dropping the global edit generation safe. The +/// ID-KEYED prep cache has the opposite obligation (§act-eleven A11.7): a +/// prep must FOLLOW its paragraph identity across the swap. +#[gpui::test] +fn positional_caches_reject_slot_reuse_by_a_different_paragraph(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let document = document_from_input(DocumentTheme::default(), vec![ + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("first paragraph body")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("second paragraph carrying different words")], + }, + ]); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(document, None, cx)); + handle + .update(cx, |editor, window, cx| { + let width = px(760.0); + for paragraph_ix in 0..2 { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + assert!(editor.valid_chunk_cache_entry(paragraph_ix, width).is_some()); + } + // Both paragraphs share (Normal, version 0) — identical cache keys. + // Swap them (paragraphs AND ids AND text), leaving the positional + // chunk/height slots holding the OTHER paragraph's layout. + let first = editor.document.paragraphs.get(0).expect("first").clone(); + let second = editor.document.paragraphs.get(1).expect("second").clone(); + assert_eq!( + paragraph_cache_key(&editor.document, &first), + paragraph_cache_key(&editor.document, &second), + "fixture must collide on (style, version) for the test to bite" + ); + editor.document.text = crop::Rope::from("second paragraph carrying different words\nfirst paragraph body"); + paragraphs_mut(&mut editor.document).set(0, second); + paragraphs_mut(&mut editor.document).set(1, first); + std::sync::Arc::make_mut(&mut editor.document.ids.paragraph_ids).swap(0, 1); + std::sync::Arc::make_mut(&mut editor.document.ids.block_ids).swap(0, 1); + update_paragraph_offsets_after_len_change(&mut editor.document, 0); + update_paragraph_offsets_after_len_change(&mut editor.document, 1); + + for paragraph_ix in 0..2 { + assert!( + editor.valid_chunk_cache_entry(paragraph_ix, width).is_none(), + "chunk slot {paragraph_ix} served a different paragraph's layout (identity check missing)" + ); + assert!( + editor.valid_paragraph_height(paragraph_ix, width).is_none(), + "height slot {paragraph_ix} served a different paragraph's height (identity check missing)" + ); + // §act-eleven A11.7: the prep cache is ID-KEYED and position-free — + // ids moved WITH their content, so each paragraph's own prep must + // FOLLOW it to the new index (this is the structural-shift win). + let prep = editor + .valid_paragraph_prep(paragraph_ix) + .expect("id-keyed prep must survive an identity swap (it moved with the paragraph)"); + assert_eq!( + Some(prep.paragraph_id), + editor.paragraph_id_at(paragraph_ix), + "prep at slot {paragraph_ix} belongs to a different paragraph identity" + ); + } + }) + .expect("windowed pass"); +} + +/// §act-eleven C9: hit-test geometry properties over the real layout — every +/// sampled pixel must resolve to a VALID document offset (in-bounds paragraph, +/// in-bounds byte, char boundary), paragraphs must be non-decreasing as y +/// grows at fixed x, and bytes non-decreasing as x grows within one paragraph +/// at fixed y. Paint/hit geometry previously had NO oracle at all. +#[gpui::test] +fn hit_test_resolves_valid_monotonic_offsets(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, window, cx| { + let width = px(760.0); + let count = editor.document.paragraphs.len(); + for paragraph_ix in 0..count { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + } + let mut previous_paragraph_at_x: std::collections::HashMap = std::collections::HashMap::new(); + for y_step in 0..40 { + let y = px(y_step as f32 * 6.0); + let mut previous: Option = None; + for x_step in 0..20 { + let x = px(x_step as f32 * 38.0); + let offset = editor.hit_test_document_position(gpui::point(x, y), window, cx); + // Validity. + assert!(offset.paragraph < count, "hit at ({x_step},{y_step}) resolved out-of-bounds paragraph {}", offset.paragraph); + let text = crate::paragraph_text(&editor.document, offset.paragraph); + assert!(offset.byte <= text.len(), "hit at ({x_step},{y_step}) resolved out-of-bounds byte"); + assert!(text.is_char_boundary(offset.byte), "hit at ({x_step},{y_step}) resolved a non-char-boundary byte"); + // x-monotonicity within a paragraph at fixed y. + if let Some(previous) = previous + && previous.paragraph == offset.paragraph + { + assert!( + offset.byte >= previous.byte, + "byte went backwards left-to-right at y_step {y_step}: {} -> {}", + previous.byte, + offset.byte + ); + } + previous = Some(offset); + // y-monotonicity at fixed x. + let column = x_step; + if let Some(&previous_paragraph) = previous_paragraph_at_x.get(&column) { + assert!( + offset.paragraph >= previous_paragraph, + "paragraph went backwards top-to-bottom at x_step {column}: {previous_paragraph} -> {}", + offset.paragraph + ); + } + previous_paragraph_at_x.insert(column, offset.paragraph); + } + } + }) + .expect("window update"); +} + +/// §act-eleven A11.7: a STRUCTURAL shift (row inserted above) must keep every +/// shifted paragraph's prep — content keys don't move with position. Bound the +/// rebuilds with the C4 tripwire: only the NEW paragraph preps. +#[gpui::test] +fn structural_shift_keeps_shifted_tail_preps(cx: &mut gpui::TestAppContext) { + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(a93_varied_document(), None, cx)); + handle + .update(cx, |editor, window, cx| { + let width = px(760.0); + let count = editor.document.paragraphs.len(); + for paragraph_ix in 0..count { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + } + let before: Vec<_> = (0..count) + .map(|paragraph_ix| editor.valid_paragraph_prep(paragraph_ix).expect("prep populated")) + .collect(); + + // Insert a NEW paragraph row at the very top (the Enter-above shape), + // mirroring the in-place splice the projection apply performs. + crate::rich_text::layout::reset_prep_build_count(); + let new_paragraph = crate::Paragraph { + style: crate::ParagraphStyle::Normal, + runs: vec![crate::TextRun { + len: "inserted-above".len(), + styles: crate::RunStyles::default(), + }], + version: 7, + }; + let document = &mut editor.document; + document.text.insert(0, "inserted-above\n"); + crate::paragraphs_mut(document).insert(0, new_paragraph.clone()); + std::sync::Arc::make_mut(&mut document.ids.paragraph_ids).insert(0, crate::new_paragraph_id()); + document.blocks.insert(0, crate::Block::Paragraph(new_paragraph)); + std::sync::Arc::make_mut(&mut document.ids.block_ids).insert(0, crate::new_block_id()); + crate::rebuild_document_sections(document); + + let count = editor.document.paragraphs.len(); + for paragraph_ix in 0..count { + let _ = editor.layout_paragraph_chunk_for_element(paragraph_ix, 0, width, window, cx); + } + // Every SHIFTED paragraph must be served its ORIGINAL prep (same Arc). + for (old_ix, old_prep) in before.iter().enumerate() { + let shifted_ix = old_ix + 1; + let served = editor + .valid_paragraph_prep(shifted_ix) + .unwrap_or_else(|| panic!("shifted paragraph {shifted_ix} lost its prep")); + assert!( + std::sync::Arc::ptr_eq(&served, old_prep), + "shifted paragraph {shifted_ix} re-prepped despite unchanged content (positional residual regressed)" + ); + } + let rebuilds = crate::rich_text::layout::prep_build_count(); + assert!( + rebuilds <= 3, + "structural shift triggered {rebuilds} prep builds (bound 3: the inserted paragraph only)" + ); + }) + .expect("window update"); +} diff --git a/crates/gpui-flowtext/src/rich_text/tools.rs b/crates/gpui-flowtext/src/rich_text/tools.rs index e07a1301..18372a9d 100644 --- a/crates/gpui-flowtext/src/rich_text/tools.rs +++ b/crates/gpui-flowtext/src/rich_text/tools.rs @@ -199,7 +199,10 @@ impl RichTextEditor { } fn force_apply_inline_tool_to_current_target(&mut self, tool: ArmedInlineTool, cx: &mut Context) { - if let Some(BlockSelection::TableCell { block_ix, row_ix, cell_ix }) = self.selected_block { + if let Some(BlockSelection::TableCell { + block_ix, row_ix, cell_ix, .. + }) = self.selected_block + { let Some(selection_range) = self.table_cell_selection_range() else { return; }; diff --git a/crates/gpui-flowtext/src/rich_text/word_boundary.rs b/crates/gpui-flowtext/src/rich_text/word_boundary.rs index 5486bede..e09eef3c 100644 --- a/crates/gpui-flowtext/src/rich_text/word_boundary.rs +++ b/crates/gpui-flowtext/src/rich_text/word_boundary.rs @@ -90,7 +90,7 @@ pub(super) fn next_debate_word_boundary_in_paragraph_text(text: &str, byte: usiz } #[hotpath::measure] -pub(super) fn previous_debate_word_boundary_in_document(document: &Document, offset: DocumentOffset) -> DocumentOffset { +pub(super) fn previous_debate_word_boundary_in_document(document: &DocumentProjection, offset: DocumentOffset) -> DocumentOffset { if document.paragraphs.is_empty() { return DocumentOffset::default(); } @@ -131,7 +131,7 @@ pub(super) fn previous_debate_word_boundary_in_document(document: &Document, off } #[hotpath::measure] -pub(super) fn next_debate_word_boundary_in_document(document: &Document, offset: DocumentOffset) -> DocumentOffset { +pub(super) fn next_debate_word_boundary_in_document(document: &DocumentProjection, offset: DocumentOffset) -> DocumentOffset { if document.paragraphs.is_empty() { return DocumentOffset::default(); } @@ -166,82 +166,73 @@ pub(super) fn next_debate_word_boundary_in_document(document: &Document, offset: } #[hotpath::measure] -pub(super) fn selection_for_word_at(document: &Document, offset: DocumentOffset) -> EditorSelection { +pub(super) fn selection_for_word_at(document: &DocumentProjection, offset: DocumentOffset) -> EditorSelection { let Some(paragraph) = document.paragraphs.get(offset.paragraph) else { - return EditorSelection { - anchor: DocumentOffset::default(), - head: DocumentOffset::default(), - }; + return EditorSelection::collapsed(DocumentOffset::default()); }; let paragraph_len = paragraph_text_len(paragraph); if paragraph_len == 0 || offset.byte >= paragraph_len { return selection_for_paragraph_at(document, offset.paragraph); } - EditorSelection { - anchor: previous_debate_word_boundary_in_document(document, offset), - head: next_debate_word_boundary_in_document(document, offset), - } + EditorSelection::range( + previous_debate_word_boundary_in_document(document, offset), + next_debate_word_boundary_in_document(document, offset), + ) } #[hotpath::measure] -pub(super) fn selection_for_paragraph_at(document: &Document, paragraph: usize) -> EditorSelection { +pub(super) fn selection_for_paragraph_at(document: &DocumentProjection, paragraph: usize) -> EditorSelection { let paragraph = paragraph.min(document.paragraphs.len().saturating_sub(1)); - EditorSelection { - anchor: DocumentOffset { paragraph, byte: 0 }, - head: DocumentOffset { + EditorSelection::range( + DocumentOffset { paragraph, byte: 0 }, + DocumentOffset { paragraph, byte: paragraph_text_len(&document.paragraphs[paragraph]), }, - } + ) } #[hotpath::measure] pub(super) fn expand_drag_selection( - document: &Document, + document: &DocumentProjection, anchor: DocumentOffset, head: DocumentOffset, granularity: SelectionGranularity, ) -> EditorSelection { match granularity { - SelectionGranularity::Character => EditorSelection { anchor, head }, + SelectionGranularity::Character => EditorSelection::range(anchor, head), SelectionGranularity::Word => { let anchor_range = selection_for_word_at(document, anchor).normalized(); let head_range = selection_for_word_at(document, head).normalized(); if head < anchor { - EditorSelection { - anchor: anchor_range.end, - head: head_range.start, - } + EditorSelection::range(anchor_range.end, head_range.start) } else { - EditorSelection { - anchor: anchor_range.start, - head: head_range.end, - } + EditorSelection::range(anchor_range.start, head_range.end) } }, SelectionGranularity::Paragraph => { if head < anchor { - EditorSelection { - anchor: DocumentOffset { + EditorSelection::range( + DocumentOffset { paragraph: anchor.paragraph, byte: paragraph_text_len(&document.paragraphs[anchor.paragraph]), }, - head: DocumentOffset { + DocumentOffset { paragraph: head.paragraph, byte: 0, }, - } + ) } else { - EditorSelection { - anchor: DocumentOffset { + EditorSelection::range( + DocumentOffset { paragraph: anchor.paragraph, byte: 0, }, - head: DocumentOffset { + DocumentOffset { paragraph: head.paragraph, byte: paragraph_text_len(&document.paragraphs[head.paragraph]), }, - } + ) } }, } @@ -250,7 +241,7 @@ pub(super) fn expand_drag_selection( // Grapheme-cluster-aware step backwards. Handles combining marks and // compound emoji correctly, so one keystroke deletes one visible character. #[hotpath::measure] -pub(super) fn prev_grapheme_boundary_in_paragraph(document: &Document, paragraph_ix: usize, byte: usize) -> usize { +pub(super) fn prev_grapheme_boundary_in_paragraph(document: &DocumentProjection, paragraph_ix: usize, byte: usize) -> usize { if byte == 0 { return 0; } @@ -265,7 +256,7 @@ pub(super) fn prev_grapheme_boundary_in_paragraph(document: &Document, paragraph } #[hotpath::measure] -pub(super) fn next_grapheme_boundary_in_paragraph(document: &Document, paragraph_ix: usize, byte: usize) -> usize { +pub(super) fn next_grapheme_boundary_in_paragraph(document: &DocumentProjection, paragraph_ix: usize, byte: usize) -> usize { let paragraph = &document.paragraphs[paragraph_ix]; let len = paragraph_text_len(paragraph); if byte >= len { @@ -279,7 +270,7 @@ pub(super) fn next_grapheme_boundary_in_paragraph(document: &Document, paragraph } #[hotpath::measure] -fn paragraph_byte_at(document: &Document, paragraph_ix: usize, byte: usize) -> Option { +fn paragraph_byte_at(document: &DocumentProjection, paragraph_ix: usize, byte: usize) -> Option { let paragraph = document.paragraphs.get(paragraph_ix)?; (byte < paragraph_text_len(paragraph)).then(|| { document diff --git a/crates/gpui-flowtext/tests/editor_behavior.rs b/crates/gpui-flowtext/tests/editor_behavior.rs new file mode 100644 index 00000000..d630bfa0 --- /dev/null +++ b/crates/gpui-flowtext/tests/editor_behavior.rs @@ -0,0 +1,915 @@ +//! Mutating editor-behavior tests (Loro-first spec §5/§13). +//! +//! These live in an INTEGRATION target (not the unit suites) so the real +//! flowstate-collab write authority and the editor share one gpui-flowtext +//! crate instance — invariant 5's "one write path" holds even in tests. An +//! editor without an authority is a read-only display surface; every test +//! here attaches `LocalDocHandle` over a canonical Loro core imported from +//! its fixture, exactly like production document open. + +#[cfg(test)] +mod tests { + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::{LocalDocHandle, LocalWriteConfig}; + use gpui::AppContext as _; + use gpui::px; + use gpui_flowtext::*; + + /// Import the editor's fixture projection into a canonical core and install + /// the REAL write authority (production wiring). + fn attach_test_authority(editor: &gpui::Entity, cx: &mut gpui::TestAppContext) { + let fixture = editor.read_with(cx, |editor, _| editor.document().clone()); + let core = + CrdtRuntime::from_document_projection(&fixture, "Test Document").expect("importing the fixture projection into a canonical Loro core"); + let (handle, _gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let projection = handle + .projection() + .expect("reading the authority's canonical projection"); + editor.update(cx, |editor, cx| { + editor.set_write_authority(std::sync::Arc::new(handle), projection, cx); + }); + } + + fn big_document(paragraphs: usize) -> DocumentProjection { + let paras = (0..paragraphs) + .map(|ix| InputParagraph { + style: if ix % 40 == 0 { + ParagraphStyle::Custom(2) + } else { + ParagraphStyle::Normal + }, + runs: vec![plain( + "Body paragraph carrying several words so the layout has real text to measure and reflow.", + )], + }) + .collect(); + document_from_input(DocumentTheme::default(), paras) + } + + fn editor_with_authority(document: DocumentProjection, cx: &mut gpui::TestAppContext) -> gpui::Entity { + let editor = cx.new(|cx| RichTextEditor::new_with_path(document, None, cx)); + attach_test_authority(&editor, cx); + editor + } + + /// §act-eleven C9: the editor interaction smoke-fuzz — a seeded random mix + /// over the PUBLIC command surface (typing, backspace/delete, word deletes, + /// paragraph breaks, selection moves, undo/redo) through the REAL write + /// authority. Invariants after EVERY op: id vectors parallel to their + /// sequences, selection within bounds on a char boundary, the rope's byte + /// length consistent with the paragraph lengths, and the editor's document + /// text equal to the authority's canonical projection text (no silent + /// editor/authority drift). The command layer had NO randomized coverage — + /// the collab fuzzes drive the model beneath it, not these entry points. + #[gpui::test] + fn interaction_smoke_fuzz_preserves_editor_invariants(cx: &mut gpui::TestAppContext) { + struct Rng(u64); + impl Rng { + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + fn below(&mut self, bound: usize) -> usize { + if bound == 0 { 0 } else { (self.next() % bound as u64) as usize } + } + } + + for seed in [11u64, 2026, 424242] { + let mut rng = Rng(seed); + // Attach the authority by hand so the test keeps the handle for the + // editor-vs-canonical drift assertion. + let editor = cx.new(|cx| RichTextEditor::new_with_path(big_document(12), None, cx)); + let fixture = editor.read_with(cx, |editor, _| editor.document().clone()); + let core = CrdtRuntime::from_document_projection(&fixture, "Smoke Fuzz").expect("canonical core"); + let (handle, _gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let authority = std::sync::Arc::new(handle); + let projection = authority.projection().expect("authority projection"); + editor.update(cx, |editor, cx| { + #[allow( + clippy::clone_on_ref_ptr, + reason = "method-syntax clone so the unsized coercion applies to the annotated binding" + )] + let authority_dyn: std::sync::Arc = authority.clone(); + editor.set_write_authority(authority_dyn, projection, cx); + }); + editor.update(cx, |editor, cx| { + for step in 0..160 { + let paragraph_count = editor.document().paragraphs.len(); + let paragraph_ix = rng.below(paragraph_count.max(1)); + let paragraph_len = flowstate_document::paragraph_text_len(&editor.document().paragraphs[paragraph_ix]); + let byte = clamp_paragraph_byte_to_char_boundary(editor.document(), paragraph_ix, rng.below(paragraph_len + 1)); + let offset = DocumentOffset { + paragraph: paragraph_ix, + byte, + }; + match rng.below(10) { + 0..=2 => { + editor.set_selection(EditorSelection::collapsed(offset), cx); + editor.insert_text_command(&format!("s{step} "), cx); + }, + 3 => { + editor.set_selection(EditorSelection::collapsed(offset), cx); + editor.backspace_command(cx); + }, + 4 => { + editor.set_selection(EditorSelection::collapsed(offset), cx); + editor.delete_forward_command(cx); + }, + 5 => { + editor.set_selection(EditorSelection::collapsed(offset), cx); + editor.delete_word_backward_command(cx); + }, + 6 => { + editor.set_selection(EditorSelection::collapsed(offset), cx); + editor.delete_word_forward_command(cx); + }, + 7 => { + editor.set_selection(EditorSelection::collapsed(offset), cx); + editor.insert_paragraph_break_command(cx); + }, + 8 => editor.undo(cx), + _ => editor.redo(cx), + } + + // ---- invariants, every step ---- + let document = editor.document(); + assert!(!document.paragraphs.is_empty(), "seed {seed} step {step}: document emptied"); + assert_eq!( + document.ids.paragraph_ids.len(), + document.paragraphs.len(), + "seed {seed} step {step}: paragraph id vector desynced" + ); + assert_eq!( + document.ids.block_ids.len(), + document.blocks.len(), + "seed {seed} step {step}: block id vector desynced" + ); + let expected_rope_len: usize = document + .paragraphs + .iter() + .map(flowstate_document::paragraph_text_len) + .sum::() + + document.paragraphs.len().saturating_sub(1); + assert_eq!( + document.text.byte_len(), + expected_rope_len, + "seed {seed} step {step}: rope length inconsistent with paragraph lengths" + ); + let head = editor.selection().head; + assert!( + head.paragraph < document.paragraphs.len(), + "seed {seed} step {step}: selection paragraph out of bounds" + ); + let head_paragraph_len = flowstate_document::paragraph_text_len(&document.paragraphs[head.paragraph]); + assert!(head.byte <= head_paragraph_len, "seed {seed} step {step}: selection byte out of bounds"); + let canonical = authority.projection().expect("authority projection"); + assert_eq!( + document.text.to_string(), + canonical.text.to_string(), + "seed {seed} step {step}: editor text drifted from the authority's canonical projection" + ); + } + }); + } + } + + /// Text edits through the REAL write path (editor command → typed intent → + /// write authority → canonical Loro commit → exact projection patches): + /// inserts, minimal marks over the changed range, and range deletes preserve + /// text and run structure. (Formerly a unit test of the retired optimistic + /// projection-edit helpers.) + #[gpui::test] + fn text_edits_preserve_text_and_styles_through_the_authority(cx: &mut gpui::TestAppContext) { + let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); + let editor = editor_with_authority( + document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![run("hello", RunStyles::default())], + }], + ), + cx, + ); + + editor.update(cx, |editor, cx| { + editor.set_selection( + EditorSelection::collapsed(DocumentOffset { + paragraph: 0, + byte: "he".len(), + }), + cx, + ); + editor.insert_text_command("y", cx); + assert_eq!(paragraph_text(editor.document(), 0), "heyllo"); + assert_eq!(editor.document().paragraphs[0].runs.len(), 1); + + editor.set_selection( + EditorSelection::range( + DocumentOffset { + paragraph: 0, + byte: "hey".len(), + }, + DocumentOffset { + paragraph: 0, + byte: "heyll".len(), + }, + ), + cx, + ); + editor.apply_run_style_to_selection(RunStyle::Semantic(2), cx); + assert_eq!(paragraph_text(editor.document(), 0), "heyllo"); + assert_eq!(editor.document().paragraphs[0].runs.len(), 3); + assert_eq!(editor.document().paragraphs[0].runs[1].styles, emphasized); + + editor.set_selection( + EditorSelection::range( + DocumentOffset { + paragraph: 0, + byte: "he".len(), + }, + DocumentOffset { + paragraph: 0, + byte: "heyll".len(), + }, + ), + cx, + ); + editor.backspace_command(cx); + assert_eq!(paragraph_text(editor.document(), 0), "heo"); + assert_eq!(editor.document().paragraphs[0].runs.len(), 1); + assert_eq!(editor.document().paragraphs[0].runs[0].styles, RunStyles::default()); + }); + } + + /// §act-three C (background open): a panel painted from a phase-V cached + /// projection is a READ-ONLY display surface (no authority) — edits are inert + /// — until phase G attaches the authority, after which editing commits + /// normally and the projection is the authority's canonical one. This is the + /// editor-level heart of the two-phase open (`create_pending_document_panel` + /// → `attach_runtime_to_pending_panel`). + #[gpui::test] + fn read_only_panel_becomes_editable_when_authority_attaches(cx: &mut gpui::TestAppContext) { + let projection = document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![run("cached view", RunStyles::default())], + }], + ); + // Phase V: editor built from the cached projection, NO authority. + let editor = cx.new(|cx| RichTextEditor::new_with_path(projection.clone(), None, cx)); + + editor.update(cx, |editor, cx| { + assert!(!editor.has_write_authority(), "phase-V panel must have no write authority"); + // An edit against a read-only surface is inert (no fallback editing). + editor.set_selection( + EditorSelection::collapsed(DocumentOffset { + paragraph: 0, + byte: "cached".len(), + }), + cx, + ); + editor.insert_text_command("X", cx); + assert_eq!(paragraph_text(editor.document(), 0), "cached view", "read-only surface must ignore edits"); + }); + + // Phase G: the authority runtime finishes loading and attaches. + attach_test_authority(&editor, cx); + + editor.update(cx, |editor, cx| { + assert!(editor.has_write_authority(), "phase-G attach must install the write authority"); + // The projection is now the authority's canonical one (same content). + assert_eq!(paragraph_text(editor.document(), 0), "cached view"); + // Editing now commits through the real write path. + editor.set_selection( + EditorSelection::collapsed(DocumentOffset { + paragraph: 0, + byte: "cached".len(), + }), + cx, + ); + editor.insert_text_command("X", cx); + assert_eq!(paragraph_text(editor.document(), 0), "cachedX view", "editing must work after attach"); + }); + } + + /// UTF-8 byte-offset law through the real write path: text anchors address + /// paragraph bytes while the canonical Loro body is unicode — inserts and + /// deletes around multi-byte characters must round-trip exactly. + #[gpui::test] + fn document_rope_edits_keep_utf8_byte_offsets(cx: &mut gpui::TestAppContext) { + let editor = editor_with_authority( + document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![run("abé🚀cd", RunStyles::default())], + }], + ), + cx, + ); + + editor.update(cx, |editor, cx| { + editor.set_selection( + EditorSelection::collapsed(DocumentOffset { + paragraph: 0, + byte: "abé".len(), + }), + cx, + ); + editor.insert_text_command("Z", cx); + assert_eq!(paragraph_text(editor.document(), 0), "abéZ🚀cd"); + + editor.set_selection( + EditorSelection::range( + DocumentOffset { + paragraph: 0, + byte: "abé".len(), + }, + DocumentOffset { + paragraph: 0, + byte: "abéZ🚀".len(), + }, + ), + cx, + ); + editor.backspace_command(cx); + assert_eq!(paragraph_text(editor.document(), 0), "abécd"); + }); + } + + /// Rich paste through the canonical `InsertRichFragment` intent: the FIRST + /// pasted paragraph flows into the caret paragraph, which keeps ITS OWN + /// paragraph style (the destination wins — the canonical fragment law; the + /// retired optimistic paste adopted the pasted style instead), while each + /// subsequent pasted paragraph carries its own style across the new boundary. + #[gpui::test] + fn rich_paste_keeps_destination_style_and_carries_subsequent_paragraph_styles(cx: &mut gpui::TestAppContext) { + let editor = editor_with_authority( + document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: Vec::new(), + }], + ), + cx, + ); + let fragment = RichClipboardFragment { + format: RICH_TEXT_CLIPBOARD_FORMAT.to_string(), + paragraphs: vec![ + InputParagraph { + style: ParagraphStyle::Custom(1), + runs: vec![plain("Heading")], + }, + InputParagraph { + style: ParagraphStyle::Custom(3), + runs: vec![plain("Tag")], + }, + ], + blocks: Vec::new(), + assets: Vec::new(), + }; + + editor.update(cx, |editor, cx| { + editor.set_selection(EditorSelection::collapsed(DocumentOffset::default()), cx); + assert!(editor.insert_rich_fragment_paste_at_caret(&fragment, cx)); + + assert_eq!(editor.document().paragraphs.len(), 2); + assert_eq!(paragraph_text(editor.document(), 0), "Heading"); + // Destination paragraph style wins for the first pasted paragraph. + assert_eq!(editor.document().paragraphs[0].style, ParagraphStyle::Normal); + assert_eq!(paragraph_text(editor.document(), 1), "Tag"); + assert_eq!(editor.document().paragraphs[1].style, ParagraphStyle::Custom(3)); + }); + } + + /// Clear-formatting over a selection spanning an empty paragraph, through the + /// REAL write path: whole-paragraph selections clear paragraph styles back to + /// Normal plus one run-style reset over the span (the editor's + /// `clear_formatting` policy, committed via the write authority). + #[gpui::test] + fn selection_across_empty_paragraphs_and_clear_formatting_policy(cx: &mut gpui::TestAppContext) { + let emphasized = RunStyles::default().with(RunStyle::Semantic(2)); + let editor = editor_with_authority( + document_from_input( + DocumentTheme::default(), + vec![ + InputParagraph { + style: ParagraphStyle::Custom(3), + runs: vec![run("tag", emphasized)], + }, + InputParagraph { + style: ParagraphStyle::Custom(0), + runs: Vec::new(), + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![run("body", emphasized)], + }, + ], + ), + cx, + ); + + editor.update(cx, |editor, cx| { + let selection = DocumentOffset { paragraph: 0, byte: 1 }..DocumentOffset { paragraph: 2, byte: 1 }; + assert!(selection_contains_whole_paragraph(editor.document(), selection.clone())); + + editor.set_selection(EditorSelection::range(selection.start, selection.end), cx); + editor.clear_formatting(cx); + + for paragraph in &editor.document().paragraphs { + assert_eq!(paragraph.style, ParagraphStyle::Normal); + assert!( + paragraph + .runs + .iter() + .all(|run| run.styles == RunStyles::default()) + ); + } + }); + } + + /// Soft line breaks through the REAL write path (one `InsertText` intent — + /// U+2028 is body text, not a paragraph boundary): the paragraph must not + /// split, and plain-text copy renders the break as '\n'. + #[gpui::test] + fn soft_line_break_stays_inside_paragraph_and_copies_as_newline(cx: &mut gpui::TestAppContext) { + let editor = editor_with_authority( + document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("alphaomega")], + }], + ), + cx, + ); + + editor.update(cx, |editor, cx| { + editor.set_selection( + EditorSelection::collapsed(DocumentOffset { + paragraph: 0, + byte: "alpha".len(), + }), + cx, + ); + editor.insert_text_command(SOFT_LINE_BREAK_STR, cx); + + assert_eq!(editor.document().paragraphs.len(), 1); + assert_eq!(paragraph_text(editor.document(), 0), format!("alpha{SOFT_LINE_BREAK_STR}omega")); + assert_eq!( + selected_plain_text( + editor.document(), + DocumentOffset { paragraph: 0, byte: 0 }..DocumentOffset { + paragraph: 0, + byte: paragraph_text_len(&editor.document().paragraphs[0]), + }, + ), + "alpha\nomega" + ); + }); + } + + /// Cross-paragraph run-style mutation through the REAL write path (one + /// `SetMarks` intent over the selection): only the selected span changes; + /// runs and unselected text on both sides stay intact. + #[gpui::test] + fn cross_paragraph_style_mutation_keeps_runs_and_unselected_text_intact(cx: &mut gpui::TestAppContext) { + let editor = editor_with_authority( + document_from_input( + DocumentTheme::default(), + vec![ + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("abc")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("def")], + }, + ], + ), + cx, + ); + + editor.update(cx, |editor, cx| { + editor.set_selection( + EditorSelection::range(DocumentOffset { paragraph: 0, byte: 1 }, DocumentOffset { paragraph: 1, byte: 2 }), + cx, + ); + editor.apply_run_style_to_selection(RunStyle::Semantic(1), cx); + + let document = editor.document(); + assert_eq!(paragraph_text(document, 0), "abc"); + assert_eq!(paragraph_text(document, 1), "def"); + assert_ne!(document.paragraphs[0].runs[0].styles.semantic, RunSemanticStyle::Custom(1)); + assert_eq!(document.paragraphs[0].runs[1].styles.semantic, RunSemanticStyle::Custom(1)); + assert_eq!(document.paragraphs[1].runs[0].styles.semantic, RunSemanticStyle::Custom(1)); + assert_ne!(document.paragraphs[1].runs[1].styles.semantic, RunSemanticStyle::Custom(1)); + }); + } + + /// After a burst of edits, a re-layout must remain incremental — the edit advances + /// `edit_generation`, and the following layout pass must not re-prep the whole document + /// (the field stall was the layout redoing work every frame while falling behind edits). + #[gpui::test] + fn relayout_after_edits_stays_incremental(cx: &mut gpui::TestAppContext) { + let paragraphs = 3000; + cx.update(gpui_component::init); + let handle = cx.add_window(|_window, cx| RichTextEditor::new_with_path(big_document(paragraphs), None, cx)); + // Edits only commit through a write authority (invariant 5) — attach the + // real one before typing. + let editor = handle.root(cx).expect("window root editor"); + attach_test_authority(&editor, cx); + handle + .update(cx, |editor, window, cx| { + // Warm the layout once. + let _ = editor.benchmark_paragraph_item_sizes(px(760.0), window, cx); + + // Type several graphemes near the top; each advances the edit generation. + editor.set_selection(EditorSelection::collapsed(DocumentOffset { paragraph: 1, byte: 0 }), cx); + let gen_before = editor.edit_generation(); + for _ in 0..8 { + editor.insert_text_command("x", cx); + } + assert!(editor.edit_generation() > gen_before, "edits must advance the edit generation"); + + // A re-layout after the edits must prep only a bounded number of chunks, not the doc. + let metrics = editor.benchmark_paragraph_item_sizes(px(760.0), window, cx); + assert!( + metrics.prep_installed < paragraphs / 3, + "re-layout after {} edits re-prepped {} of {paragraphs} chunks — not incremental", + 8, + metrics.prep_installed, + ); + }) + .expect("windowed edit + relayout"); + } + + /// Spec §11 keystroke-latency ratchet (initial ceilings: p50 < 16ms, + /// p95 < 50ms on the large fixture). Measures the FULL Loro-first path per + /// keystroke — intent build → write gate → identity resolution → Loro + /// commit → exact patch synthesis → patch apply into THE projection — on a + /// 5000-paragraph document. Latency is now a UI-thread input-latency + /// guarantee, not an actor round-trip. Run manually: + /// `cargo test -p gpui-flowtext --release --test editor_behavior -- --ignored` + #[gpui::test] + #[ignore = "latency ratchet; run manually with --release --ignored"] + fn keystroke_latency_through_the_authority(cx: &mut gpui::TestAppContext) { + let paragraphs = (0..5_000) + .map(|ix| InputParagraph { + style: if ix % 40 == 0 { + ParagraphStyle::Custom(2) + } else { + ParagraphStyle::Normal + }, + runs: vec![plain( + "Competitive policy debate rewards evidence organized for instant retrieval and flawless formatting.", + )], + }) + .collect(); + let editor = editor_with_authority(document_from_input(DocumentTheme::default(), paragraphs), cx); + let target = editor.read_with(cx, |editor, _| editor.document().paragraphs.len() / 2); + editor.update(cx, |editor, cx| { + editor.set_selection(EditorSelection::collapsed(DocumentOffset { paragraph: target, byte: 0 }), cx); + }); + + let mut samples = Vec::with_capacity(1_800); + editor.update(cx, |editor, cx| { + for step in 0..1_500 { + let started = std::time::Instant::now(); + editor.insert_text_command("x", cx); + samples.push(started.elapsed()); + if step % 5 == 4 { + let started = std::time::Instant::now(); + editor.backspace_command(cx); + samples.push(started.elapsed()); + } + } + }); + + samples.sort(); + let p50 = samples[samples.len() / 2]; + let p95 = samples[samples.len() * 95 / 100]; + let p99 = samples[samples.len() * 99 / 100]; + let max = *samples.last().expect("samples recorded"); + println!( + "Loro-first keystroke latency over {} samples on a 5000-paragraph document: p50={p50:?} p95={p95:?} p99={p99:?} max={max:?}", + samples.len(), + ); + // Spec §11 initial ratchet ceilings — tighten on improvement, never loosen + // without a human decision. Being under budget is not a quality claim. + assert!( + p50 < std::time::Duration::from_millis(16), + "keystroke p50 regressed past the ratchet: {p50:?}" + ); + assert!( + p95 < std::time::Duration::from_millis(50), + "keystroke p95 regressed past the ratchet: {p95:?}" + ); + } + + /// Replace-all must reach CANONICAL state (2026-07-07 field class: the old + /// implementation rewrote the editor's projection copy directly for + /// same-paragraph matches, so saves, peers, and undo silently lost every + /// replacement — the projection looked right until the next authority + /// emission clobbered it). + #[gpui::test] + fn replace_all_search_highlights_commits_canonically(cx: &mut gpui::TestAppContext) { + let editor = cx.new(|cx| { + RichTextEditor::new_with_path( + document_from_input( + DocumentTheme::default(), + vec![ + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("alpha foo beta foo")], + }, + InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("foo gamma")], + }, + ], + ), + None, + cx, + ) + }); + let fixture = editor.read_with(cx, |editor, _| editor.document().clone()); + let core = CrdtRuntime::from_document_projection(&fixture, "Replace").expect("canonical core"); + let (handle, _gate) = LocalDocHandle::new(core, LocalWriteConfig::default()); + let handle = std::sync::Arc::new(handle); + let projection = handle.projection().expect("canonical projection"); + editor.update(cx, |editor, cx| { + editor.set_write_authority(handle.clone(), projection, cx); + }); + + editor.update(cx, |editor, cx| { + let ranges = vec![ + DocumentOffset { paragraph: 0, byte: 6 }..DocumentOffset { paragraph: 0, byte: 9 }, + DocumentOffset { paragraph: 0, byte: 15 }..DocumentOffset { paragraph: 0, byte: 18 }, + DocumentOffset { paragraph: 1, byte: 0 }..DocumentOffset { paragraph: 1, byte: 3 }, + ]; + editor.set_search_highlights(ranges, Some(0), cx); + assert_eq!(editor.replace_all_search_highlights("bar", cx), 3); + }); + + // THE data-loss assertion: the write authority's canonical state carries + // every replacement. + let canonical = handle.projection().expect("canonical after replace"); + assert_eq!(paragraph_text(&canonical, 0), "alpha bar beta bar"); + assert_eq!(paragraph_text(&canonical, 1), "bar gamma"); + // The editor's projection agrees with canonical (no divergent local copy). + editor.read_with(cx, |editor, _| { + assert_eq!(paragraph_text(editor.document(), 0), "alpha bar beta bar"); + assert_eq!(paragraph_text(editor.document(), 1), "bar gamma"); + }); + + // One undo reverts the whole replace-all (the compound intent is one + // undo member), and the revert reaches canonical too. + editor.update(cx, |editor, cx| { + editor.undo(cx); + }); + let canonical = handle.projection().expect("canonical after undo"); + assert_eq!(paragraph_text(&canonical, 0), "alpha foo beta foo"); + assert_eq!(paragraph_text(&canonical, 1), "foo gamma"); + } + + /// FAIL-LOUD (concurrent-typing interleave): when a remote peer inserts text + /// BEFORE the local caret, the caret must stay pinned to its own content — not + /// merely clamp to document bounds. Today `apply_remote_patch_batch` / + /// `sync_projection_from_authority` only clamp `self.selection.head`, never + /// shifting it by the inserted length, so the local user's next keystroke + /// resolves to a stale projection offset and interleaves into the remote run. + /// + /// Deterministic (no Fugue tie-break dependence): peer B prepends before the + /// SHARED base character "X", so B's run is unambiguously ordered before A's + /// caret. The edit reaches A through the REAL authority + projection stream. + #[gpui::test] + fn remote_insert_before_caret_must_shift_not_interleave(cx: &mut gpui::TestAppContext) { + use flowstate_collab::crdt_runtime::CrdtRuntime; + use flowstate_collab::local_write::GateHolder; + + // Shared base: one paragraph "XY". Build A's canonical core, snapshot it so + // peer B can share A's exact history, then install the real authority. + let fixture = document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("XY")], + }], + ); + let core_a = CrdtRuntime::from_document_projection(&fixture, "peer-A").expect("core A"); + // Peer B is a fork of A's doc: it shares A's exact history but has a distinct + // peer id, so its edits are genuinely concurrent and merge cleanly. + let doc_b = core_a.doc().fork(); + doc_b + .set_peer_id(0x00B0_B0B0) + .expect("B gets a distinct peer id"); // fork keeps A's id + let base_vv = doc_b.state_vv(); + let body_b = flowstate_document::loro_schema::body_text(&doc_b); + + let (handle_a, gate_a) = LocalDocHandle::new(core_a, LocalWriteConfig::default()); + let projection_a = handle_a.projection().expect("A projection"); + let editor = cx.new(|cx| RichTextEditor::new_with_path(projection_a.clone(), None, cx)); + editor.update(cx, |editor, cx| { + editor.set_write_authority(std::sync::Arc::new(handle_a), projection_a, cx); + }); + + // B inserts "ABC" at the START of paragraph 0 (body position 1, just after the + // leading boundary "\n"), then exports ONLY its new ops (delta from the shared + // base version vector). + body_b.insert(1, "ABC").expect("B inserts"); + doc_b.commit(); + let update_b = doc_b + .export(loro::ExportMode::updates(&base_vv)) + .expect("B exports its delta"); + + // A parks its caret between the shared X and Y ("X|Y" → byte 1). + editor.update(cx, |editor, cx| { + editor.set_selection(EditorSelection::collapsed(DocumentOffset { paragraph: 0, byte: 1 }), cx); + }); + + // B's edit arrives at A through the production path: gate import → ordered + // projection stream → editor drain. Release the gate before the editor drains + // (its sync re-locks the same gate). + let mut guard = gate_a.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard.import_remote_update(&update_b).expect("A imports B"); + drop(guard); + editor.update(cx, |editor, cx| editor.sync_projection_from_authority(cx)); + + editor.read_with(cx, |editor, _| { + assert_eq!( + paragraph_text(editor.document(), 0), + "ABCXY", + "B's prepend must merge before the shared text" + ); + // Caret was at "X|Y" (byte 1). Three bytes inserted BEFORE it ⇒ the same + // logical spot is byte 4 ("ABCX|Y"). A bare clamp leaves it at byte 1 + // ("A|BCXY"), which is the interleave bug. + assert_eq!( + editor.selection().head, + DocumentOffset { paragraph: 0, byte: 4 }, + "caret must SHIFT past a remote insert-before-caret, not merely clamp" + ); + }); + + // The symptom itself: A types 'H'. It must land after its own X → "ABCXHY", + // never interleaved into B's run ("ABHCXY"). + editor.update(cx, |editor, cx| editor.insert_text_command("H", cx)); + editor.read_with(cx, |editor, _| { + assert_eq!( + paragraph_text(editor.document(), 0), + "ABCXHY", + "A's keystroke must not interleave into the remote run" + ); + }); + } + + /// FAST-path guard: once the caret has been captured at a synced moment (here by + /// a local edit), a remote insert-before-caret repositions it by RESOLVING the + /// stored CRDT cursors — it must NOT fall back to the O(doc) `fork_at` rebase. + /// Asserts both correctness (caret + no interleave) and that ZERO forks happened, + /// so a regression to always-fork on every remote batch fails loudly. + #[gpui::test] + fn armed_caret_uses_fast_cursor_path_not_fork(cx: &mut gpui::TestAppContext) { + use flowstate_collab::crdt_runtime::{CrdtRuntime, caret_rebase_fork_count}; + use flowstate_collab::local_write::GateHolder; + + let fixture = document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("XY")], + }], + ); + let core_a = CrdtRuntime::from_document_projection(&fixture, "peer-A").expect("core A"); + // B forks the ORIGINAL base (before A's local edit), so B's insert is concurrent. + let doc_b = core_a.doc().fork(); + doc_b.set_peer_id(0x00B0_B0B0).expect("distinct peer id"); + let base_vv = doc_b.state_vv(); + let body_b = flowstate_document::loro_schema::body_text(&doc_b); + + let (handle_a, gate_a) = LocalDocHandle::new(core_a, LocalWriteConfig::default()); + let projection_a = handle_a.projection().expect("A projection"); + let editor = cx.new(|cx| RichTextEditor::new_with_path(projection_a.clone(), None, cx)); + editor.update(cx, |editor, cx| { + editor.set_write_authority(std::sync::Arc::new(handle_a), projection_a, cx); + }); + + // A performs a LOCAL edit ("Z" between X and Y): "XY" → "XZY", caret after Z + // (byte 2). This ARMS the fast-path anchor for the post-write caret. + editor.update(cx, |editor, cx| { + editor.set_selection(EditorSelection::collapsed(DocumentOffset { paragraph: 0, byte: 1 }), cx); + editor.insert_text_command("Z", cx); + }); + editor.read_with(cx, |editor, _| { + assert_eq!(paragraph_text(editor.document(), 0), "XZY"); + assert_eq!(editor.selection().head, DocumentOffset { paragraph: 0, byte: 2 }); + }); + + // B (concurrent) prepends "ABC" before the shared X. + body_b.insert(1, "ABC").expect("B inserts"); + doc_b.commit(); + let update_b = doc_b + .export(loro::ExportMode::updates(&base_vv)) + .expect("B delta"); + + let forks_before = caret_rebase_fork_count(); + let mut guard = gate_a.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard.import_remote_update(&update_b).expect("A imports B"); + drop(guard); + editor.update(cx, |editor, cx| editor.sync_projection_from_authority(cx)); + let forks_after = caret_rebase_fork_count(); + + editor.read_with(cx, |editor, _| { + assert_eq!(paragraph_text(editor.document(), 0), "ABCXZY", "B's prepend merges before A's local edit"); + // Caret was after Z (byte 2 in "XZY"); after "ABC" prepended it is byte 5 in + // "ABCXZY" ("ABCXZ|Y"). + assert_eq!( + editor.selection().head, + DocumentOffset { paragraph: 0, byte: 5 }, + "armed caret repositions across the remote insert" + ); + }); + assert_eq!(forks_after - forks_before, 0, "an armed caret must use the fast cursor path, not fork_at"); + } + + /// FAST-path guard after a CARET MOVE (no write): moving the caret re-arms the + /// anchor (via `emit_selection_changed`), so a subsequent remote insert-before- + /// caret RESOLVES the stored cursors instead of the O(doc) `fork_at`. Regression + /// guard for the freeze that shipped when re-arm happened only after a + /// write/sync: the first edit — and any incoming remote edit — after every + /// click/arrow fell to the ~350ms fork on a large doc. + #[gpui::test] + fn moved_caret_uses_fast_cursor_path_not_fork(cx: &mut gpui::TestAppContext) { + use flowstate_collab::crdt_runtime::{CrdtRuntime, caret_rebase_fork_count}; + use flowstate_collab::local_write::GateHolder; + + let fixture = document_from_input( + DocumentTheme::default(), + vec![InputParagraph { + style: ParagraphStyle::Normal, + runs: vec![plain("XYZW")], + }], + ); + let core_a = CrdtRuntime::from_document_projection(&fixture, "peer-A").expect("core A"); + let doc_b = core_a.doc().fork(); + doc_b.set_peer_id(0x00B0_B0B0).expect("distinct peer id"); + let base_vv = doc_b.state_vv(); + let body_b = flowstate_document::loro_schema::body_text(&doc_b); + + let (handle_a, gate_a) = LocalDocHandle::new(core_a, LocalWriteConfig::default()); + let projection_a = handle_a.projection().expect("A projection"); + let editor = cx.new(|cx| RichTextEditor::new_with_path(projection_a.clone(), None, cx)); + editor.update(cx, |editor, cx| { + editor.set_write_authority(std::sync::Arc::new(handle_a), projection_a, cx); + }); + + // A MOVES the caret (no edit) to byte 2 ("XY|ZW"). The re-arm on selection + // change captures the fast-path anchor for this position — the crux of the fix. + editor.update(cx, |editor, cx| { + editor.set_selection(EditorSelection::collapsed(DocumentOffset { paragraph: 0, byte: 2 }), cx); + }); + + // B (concurrent) prepends "ABC" before the shared text. + body_b.insert(1, "ABC").expect("B inserts"); + doc_b.commit(); + let update_b = doc_b + .export(loro::ExportMode::updates(&base_vv)) + .expect("B delta"); + + let forks_before = caret_rebase_fork_count(); + let mut guard = gate_a.lock(GateHolder::ImportChunk).expect("gate healthy"); + guard.import_remote_update(&update_b).expect("A imports B"); + drop(guard); + editor.update(cx, |editor, cx| editor.sync_projection_from_authority(cx)); + let forks_after = caret_rebase_fork_count(); + + editor.read_with(cx, |editor, _| { + assert_eq!( + paragraph_text(editor.document(), 0), + "ABCXYZW", + "B's prepend merges before the shared text" + ); + // Caret at byte 2 ("XY|ZW") shifts past the 3-char prepend to byte 5 ("ABCXY|ZW"). + assert_eq!( + editor.selection().head, + DocumentOffset { paragraph: 0, byte: 5 }, + "moved caret repositions across the remote insert" + ); + }); + assert_eq!( + forks_after - forks_before, + 0, + "a caret moved (not written) must still use the fast cursor path, not fork_at" + ); + } +} diff --git a/deny.toml b/deny.toml index e06e1c53..a551eafb 100644 --- a/deny.toml +++ b/deny.toml @@ -28,7 +28,20 @@ unmaintained = "workspace" yanked = "warn" # Add RUSTSEC IDs here to acknowledge/skip a specific advisory, e.g.: # ignore = ["RUSTSEC-0000-0000"] -ignore = [] +# +# quick-xml < 0.41 DoS advisories (quadratic duplicate-attribute check / +# unbounded namespace allocation). Our OWN untrusted-input parse paths +# (flowstate-docx interpreter + fragment parse) are on quick-xml 0.41; the +# remaining old copies are pinned by the LATEST releases of third-party +# crates and cannot be bumped without vendoring them: +# 0.30 xcb (build-time X11 protocol defs, trusted input) +# 0.36 docx-rs 0.4.20 (latest; pins ^0.36) +# 0.37 rdocx-opc/rdocx-oxml 0.1.2 (latest; pin ^0.37) + our matching +# `quick-xml-oxml` rename in flowstate-docx (Reader types must unify) +# 0.38 citationberg (even 0.7.0 pins ^0.38; via typst-library/hayagriva) +# 0.39 wayland-scanner (proc-macro, build-time trusted input) +# Re-check on dependency upgrades and drop these when upstreams move. +ignore = ["RUSTSEC-2026-0194", "RUSTSEC-2026-0195"] # --------------------------------------------------------------------------- # License policy. Unlisted licenses are denied under version 2. @@ -53,6 +66,12 @@ allow = [ "0BSD", "bzip2-1.0.6", "NCSA", + # Public-domain-equivalent; pulled in by the iroh/collab websocket stack + # (async_io_stream, pharos, ws_stream_wasm). + "Unlicense", + # Community Data License Agreement (permissive) — the standard license for + # the Mozilla CA-bundle data crates (webpki-roots, webpki-root-certs). + "CDLA-Permissive-2.0", ] exceptions = [] diff --git a/docs/collab-coalescing-parity.md b/docs/collab-coalescing-parity.md new file mode 100644 index 00000000..c1db7278 --- /dev/null +++ b/docs/collab-coalescing-parity.md @@ -0,0 +1,104 @@ +# Decision: object-adjacent empty-paragraph coalescing parity + +**Status:** ✅ DONE — **Fork B** implemented, structural fuzz green (N=2–5, out-of-order delivery), zero regressions. +**Found by:** the N-peer convergence fuzz harness (`multi_peer_convergence_tests.rs`, commit d4b11ae). +**Acceptance test (now passing):** `cargo test -p flowstate-collab npeer_fuzz_structural_fixture_paragraph_ops`. + +## Resolution (three fixes) + +Fork B (preserve real empty paragraphs, drop only object-phantoms) needed parity on **both sides of an object** and in the **remote-import** path: + +1. **Coalesce phantom-only AFTER an object** — `push_flow_blocks` (`loro_projection.rs:279`) coalesces only the first `\n` after an object (`current_boundary == None`); a real recorded empty after an object survives. +2. **Flush real empties BEFORE an object** — the OBJECT handler (`loro_projection.rs:316`) now flushes an empty `current` with a live boundary that follows already-emitted content (`current_boundary.is_some() && !output.is_empty()`), so an empty line before an image survives. The leading sentinel empty before a doc-first object is still excluded (the object stays block 0). +3. **Remote imports on object docs take the full rebuild** — `remote_nonstructural_projection_patches` (`projection_patch.rs:493`) is object-unaware (`body_input_paragraph` folds `OBJECT_REPLACEMENT` into text and mis-indexes around coalesced empties), so any object in the doc now routes remote updates through `refresh_projection` (object-aware, O(N) since the batched-resolver perf fix). This was the multi-peer half of the bug: a remote merge left the incremental projection stale. + +Together these make the incremental projection and `document_from_loro` agree for every op family the fuzz drives, at any peer count, under out-of-order delivery. **Zero regressions** (105 collab + document + gpui-flowtext). + +The sections below are the original analysis that led to choosing Fork B, kept for history. + +--- + +This is the last convergence bug the harness surfaces after the fixes already landed this session. I'm isolating it here rather than guessing because the correct fix depends on a **product question** and touches the incremental-materializer rework you're actively editing. + +--- + +## TL;DR + +`document_from_loro` (the authoritative full rebuild, via `push_flow_blocks`) **coalesces** an empty paragraph that sits immediately after an object (image/table/equation) — it drops it from the projection. The **incremental** projection (the per-edit replay in `gpui-flowtext`) does **not** coalesce. So the moment an edit *creates or exposes* an object-adjacent empty paragraph, the two projections disagree on paragraph count/identity, the preflight assert (`crdt_runtime.rs:1290`) fires, and in a release build the incremental patch shipped to the editor/peers is wrong → "visual fuckery" / rolling-back edits. + +The two must use **one** segmentation rule. The fork is *which* rule, and that hinges on a product question. + +--- + +## Why it exists + +An object is its own block. In the body flow, `[paragraph, object, paragraph]` serializes as: + +``` +\n before OBJECT \n after +``` + +Without any coalescing, `push_flow_blocks` would emit a **record-less empty paragraph** between `OBJECT` and `after` (the `\n` after the object, with no preceding text, produces an empty paragraph). That's a *phantom* — an artifact of the object being a block, not something the user created. Commit `3d2cb82` added the coalescing branch (`loro_projection.rs:279`) to drop it. + +But that branch drops **every** empty paragraph whose previous block is an object — including **real** ones the user typed or an edit created (e.g. deleting the last char of a paragraph that happens to sit right after an image). The incremental replay keeps those. Hence the divergence. + +### Concrete repro (seed 0xB2, single peer) + +Fixture has `… alpha, IMAGE, "Two empties above me." …`. Edits split `"Two empties above me."` into `"T"` + `"wo empties above me."`, then delete the `"T"`: + +- **Incremental** (predicted): keeps the now-empty `""` paragraph → **9 paragraphs**. +- **Full** (`document_from_loro`): coalesces the object-adjacent `""` → **8 paragraphs**. + +`debug_assert_eq!` on `paragraph_ids` fires. + +--- + +## Why the "obvious" one-line fix (Fork B) is NOT enough + +I tried gating the coalescing on `current_boundary.is_none()` (coalesce only the *first* `\n` after an object = the phantom, keep subsequent empties). It gives the correct paragraph **count per body `\n`** and passed the 104 unit tests — but the fuzz still failed, because the phantom/real distinction is **position-ambiguous**: an edit-created **real** empty can itself be the first `\n` after an object, and a bare `[image, paragraph]` shares one `\n` that is simultaneously "the phantom slot" and "the next paragraph's boundary." Position alone can't tell them apart, and the incremental replay has no equivalent notion at all. Reverted (see git history of this doc's commit). + +The lesson: **any fix that changes only one of the two paths, or distinguishes phantom-vs-real by position, will not converge.** The rule must be shared and unambiguous. + +--- + +## The forks + +### Fork A — make the incremental replay coalesce, matching the full rebuild *(recommended: lowest risk)* +Teach the `gpui-flowtext` incremental path (`replay_semantic_command_on_projection` / `rebuild_document_from_projection_patch_blocks` / the object fast-paths in `projection_apply.rs`) to drop an object-adjacent empty paragraph exactly as `push_flow_blocks` does. `push_flow_blocks` stays the authority; the incremental path is made to agree. +- **Pro:** smallest blast radius on convergence-critical code; keeps the deliberate `3d2cb82` behavior; the full rebuild is untouched and remains the source of truth; provable by the fuzz. +- **Con:** preserves the (arguably surprising) product behavior that an empty line directly after an image is silently dropped. Requires care in the rework you're editing. +- **Where:** `crates/gpui-flowtext/src/rich_text/editor/projection_apply.rs` (+ `lifecycle.rs`). The rule to mirror: `loro_projection.rs:279-286`. + +### Fork B — record-aware coalescing in BOTH paths *(product-nicer, more work)* +Make the segmentation rule "an object-adjacent empty paragraph is coalesced **iff it has no durable paragraph metadata record**" (phantom = record-less; real = recorded), and apply that same rule in `push_flow_blocks` **and** the incremental path. Real empties (which always carry a record — from `ensure_paragraph_metadata`/split) survive everywhere; only true phantoms drop. +- **Pro:** correct product behavior — users can keep an empty line next to an image; matches intuition. +- **Con:** touches the convergence-critical full-rebuild segmentation *and* the incremental path; the record lookup at a boundary is subtle (the `\n` after an object is shared with the next paragraph's boundary — needs the record to be attributed to the empty, not the follower). Higher risk; more validation needed. + +### Fork C — prune coalesced empties from Loro on write +When the full rebuild would coalesce an empty, also delete its `\n`/record from Loro so the body matches the projection. +- **Con:** mutation-on-projection, races with concurrent edits, can fight the CRDT. Not recommended. + +--- + +## The product question you need to answer + +**Should an empty paragraph immediately after an image/table be preserved, or silently dropped?** + +- If **dropped is fine** (current behavior) → **Fork A**. Ship the safe convergence fix; done. +- If **it should be preserved** (my hunch for a word processor — people put blank lines around figures) → **Fork B**. More work, but the right end state. + +My recommendation: **Fork A now** (it makes the harness green and stops the corruption with minimal risk), then **Fork B as a follow-up** product improvement once the harness is a trusted gate. Shipping A first means every future change is protected by the passing fuzz while B is developed. + +--- + +## What's already fixed this session (so this doc is scoped) + +- `17a624e` perf (OOM), `b473b1f` docx soft-break import, `5d50099` FS-170 durable-cursor offsets (InsertText/DeleteRange/SplitParagraph), `99d00f0` caret round-trip guard. +- `d4b11ae` **the harness** + JoinParagraphs atomicity + Loro-space coordinates for SetParagraphStyle/SetRunStyles/JoinParagraphs (three more mutation arms the earlier coordinate fix missed). + +The harness (`npeer_fuzz_blank_paragraph_ops`, N=2–5) is **green**; the structural fuzz is `#[ignore]`d pointing here. Un-ignore it as the acceptance test for whichever fork you pick. + +## Also worth knowing (not blockers) + +- The generator currently covers paragraph/text ops (Insert incl. soft breaks, Delete, Split, Join, SetParagraphStyle). Block/table/object ops (InsertBlock, DeleteBlock, MoveBlock, ReplaceBlock, the 9 table ops, image/equation ops) are **not yet generated** — the construction cookbook for all 24 `SemanticEditCommand` variants is ready to wire in. Expect more bugs when object/table mutation is fuzzed; add it after the coalescing fork lands. +- All divergences found so far are **single-peer materializer-equivalence** (incremental vs a fresh `document_from_loro` on the same doc), i.e. they reproduce without concurrency — concurrency just reaches the triggering states faster. diff --git a/docs/collab-object-block-positioning.md b/docs/collab-object-block-positioning.md new file mode 100644 index 00000000..2714bbd2 --- /dev/null +++ b/docs/collab-object-block-positioning.md @@ -0,0 +1,51 @@ +# Object + table op convergence — RESOLVED + +**Status:** ✅ DONE. The full editor op surface (all 24 `SemanticEditCommand` variants) +converges across N peers under out-of-order delivery. Acceptance tests: +`npeer_fuzz_object_structural_ops`, `npeer_fuzz_table_ops`, +`npeer_fuzz_replace_paragraph_span`, `object_block_positioning_single_peer` (plus the +paragraph/coordinate suites). All green. + +## What broke and how it was fixed + +### 1. Object block positioning (off-by-N) — `e909924` +The Loro-side object insert/move placed the `OBJECT_REPLACEMENT` char via a raw body walk +that counted one block per `\n`/object, ignoring the projection's coalescing of +record-less phantom empties after an object — so a projection block index mapped to a body +position off by N. Replaced with `projection_block_lead_pos_in_loro`, which resolves the +lead position from the projection's own (coalesced) block list + each block's durable +cursor (object `anchor_cursor`, or paragraph boundary `\n`, sentinel-clamped). + +### 2. The coalescing "shared root" — `a1a5a4b` +The incremental replay (`replay_semantic_command_on_projection`) models plain paragraph +text/structure but NOT object coalescing (dropping the phantom empty after an object, +re-segmenting the sentinel-first region). So on any object-bearing doc its prediction can +diverge structurally from the authoritative `document_from_loro` rebuild — leading-object +insert/move, DeleteBlock-materialized phantom empties, and later text/split/join near them. + +Key reframe: the shipped projection patch is ALWAYS the canonical rebuild, verified, with a +full-projection fallback — so **cross-peer convergence was never at risk**; the divergences +only tripped the preflight identity assert and the local optimistic echo. Fixes: +- Object-bearing docs adopt the canonical rebuild as their prediction (the runtime already + materializes it every transaction — free; mirrors the remote-import path). Pure-paragraph + docs keep the exact incremental prediction. +- The identity `debug_assert` is split: STRUCTURE (block kinds + paragraph texts) is asserted + hard (catches every real positional/segmentation/content bug); exact ID equality became a + fidelity observation, because a record-less/fabricated id (a phantom empty pending the + repair pass) is derived from a boundary OpID and legitimately differs between the + incremental carry-forward and a fresh rebuild — the repair pass's domain, not the replay's. + +### 3. Table concurrent-delete column topology — `1cb95f1` +A concurrent `DeleteTableColumn` removes the column's map from `columns_by_id`, but the +ordered `column_order` list is a separate CRDT that can still reference it after an +out-of-order merge. The projector HARD-ERRORED ("missing table column"), sinking the whole +projection. Fixed to skip the stale order entry (deterministic across peers), matching the +malformed-id skip directly above and §P2b's "a single bad id can't sink the whole table"; +the topology normalization already drops cells left referencing the removed column. + +## Coverage (all fuzzed, N-peer, out-of-order) +Paragraph/text (insert incl. U+2028 soft breaks, delete, split, join, set-paragraph-style, +set-run-styles); object block insert/delete/move/replace at EVERY position incl. leading; +object properties (image alt/layout/caption, equation source); the 9 table ops; and +`ReplaceParagraphSpan`. Each asserts cross-peer projection equality + per-peer +incremental-vs-fresh materializer equivalence. diff --git a/heaven.sh b/heaven.sh new file mode 100755 index 00000000..12bf7e52 --- /dev/null +++ b/heaven.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env bash +# heaven.sh — the perf-heaven gate (Phase 0 of flowstate_perf_heaven_spec.md). +# +# One entry point for the three oracles that make aggressive optimization safe: +# SPEED — the flowstate-corpus perf benches (is it faster, where did the +# time/bytes go?). +# FIDELITY — the corpus sweep over every real .docx (did we break a real +# document?). +# CONVERGENCE — the intent/convergence fuzz suites (does it still converge +# N-peer under adversarial interleavings?). +# +# The doctrine (§2 of the spec): MEASURE -> CUT -> PROVE-RED -> RECOVER -> +# RE-MEASURE -> LOCK. This script is the MEASURE and PROVE ends; you run it +# before a cut to capture a baseline and after to prove the win survived and no +# oracle went red. +# +# Usage: +# ./heaven.sh # fidelity + convergence + a quick bench (the gate) +# ./heaven.sh fidelity # corpus sweep only (0 failures required) +# ./heaven.sh convergence # fuzz suites only +# ./heaven.sh bench # perf benches only, printed to stdout +# ./heaven.sh baseline # bench -> $HEAVEN_DIR/baseline.txt +# ./heaven.sh compare # bench -> $HEAVEN_DIR/current.txt, diff vs baseline +# ./heaven.sh watch # continuous corpus sweep as files are added +# ./heaven.sh verify # T1 richtext fast-path equivalence (snapshot->Src) + fuzz +# ./heaven.sh docx-roundtrip # docx exporter export->reimport audit (pre-existing losses) +# ./heaven.sh docx-fallback # docx unit suite under the FORCED old walk (escape-hatch guard) +# ./heaven.sh soak # long-horizon randomized fuzz sweep (FUZZ_SOAK_* env) +# ./heaven.sh all # everything +# +# Env: +# FLOWSTATE_CORPUS_DIR corpus root (default repo-local helpers/corpus/dropbox-office-flat) +# FLOWSTATE_BENCH_FIXTURE a large .docx for the benches (required for bench) +# HEAVEN_DIR where baselines live (default ~/.flowstate-corpus) +# FLOWSTATE_RICHTEXT_VERIFY build both the T1 Src fast path and the full state +# and assert equality (the fast-path recovery guard) +# FLOWSTATE_RICHTEXT_NO_FASTPATH disable the T1 Src fast path (escape hatch) +set -euo pipefail + +cd "$(dirname "$0")" + +HEAVEN_DIR="${HEAVEN_DIR:-$HOME/.flowstate-corpus}" +mkdir -p "$HEAVEN_DIR" + +bold() { printf '\033[1m%s\033[0m\n' "$*"; } +rule() { printf '%s\n' "----------------------------------------------------------------------"; } + +# --- FIDELITY: the corpus sweep ------------------------------------------------- +# The bar: every real .docx imports and projects with ZERO defects and no content +# loss. This is what catches a projection regression from a perf change. +# (NOTE: `docx-roundtrip` below is a SEPARATE, stricter export->reimport audit +# that fails on a large PRE-EXISTING fraction of the corpus — the docx exporter is +# lossy — so it is NOT part of the perf-regression gate.) +run_fidelity() { + bold "FIDELITY — corpus sweep (0 defects / no content loss)" + rule + cargo run --release --quiet -p flowstate-corpus --bin flowstate-corpus -- "$@" +} + +run_watch() { + bold "FIDELITY — continuous corpus sweep (--watch)" + rule + cargo run --release --quiet -p flowstate-corpus --bin flowstate-corpus -- --watch "$@" +} + +# The docx export->reimport round-trip — a separate audit of the docx EXPORTER +# (not the projection/perf paths). A large pre-existing fraction fails today. +# §act-eleven C3: the docx unit suite under the FORCED fallback walk — the old +# XmlNode walk is the typed walk's correctness escape hatch, and an escape +# hatch nothing exercises is broken exactly when needed. Separate cargo +# invocation = separate process, so the env-gated OnceLock cannot leak into +# the typed-path run above. +run_docx_fallback() { + bold "DOCX — unit suite under FLOWSTATE_DOCX_TYPED_WALK=0 (fallback walk)" + rule + FLOWSTATE_DOCX_TYPED_WALK=0 cargo test --release -p flowstate-docx +} + +run_docx_roundtrip() { + bold "DOCX-ROUNDTRIP — export -> reimport text survives (docx exporter audit)" + rule + cargo run --release --quiet -p flowstate-corpus --bin flowstate-corpus -- --roundtrip "$@" +} + +# --- T1 fast-path equivalence (the perf-heaven recovery guard) ------------------ +# The §perf-heaven T1 richtext Src fast path is ACTIVE by default. This proves it +# bit-identical to the full state build over every real corpus doc AND every fuzz +# interleaving. BOTH legs run release, so BOTH need FLOWSTATE_RICHTEXT_VERIFY to +# compile the assert in (run_convergence exports it below). NOTE: the corpus leg +# is the real T1 exerciser — fuzz docs are built by live ops (`Dst` state), so +# the Src fast path rarely engages there; the fuzz leg mainly guards the assert's +# own plumbing plus any Src states reached via snapshot exchange. +# A divergence panics with the offending document/interleaving — the doctrine's +# fuzz-guided recovery signal. Must be green before the fast path is trusted. +run_verify() { + bold "T1 VERIFY — richtext Src fast path == full state (corpus, snapshot->Src+verify)" + rule + # FLOWSTATE_SNAPSHOT_VERIFY forces each doc through a Loro snapshot -> fresh + # reimport -> project, so the projection lands on a `Src` state and the T1 fast + # path actually runs; FLOWSTATE_RICHTEXT_VERIFY makes it assert bit-identical to + # the full build. (A plain import leaves the state `Dst` and never runs T1.) + FLOWSTATE_RICHTEXT_VERIFY=1 FLOWSTATE_SNAPSHOT_VERIFY=1 \ + cargo run --release --quiet -p flowstate-corpus --bin flowstate-corpus -- --recheck "$@" + echo + bold "T1 VERIFY — richtext Src fast path == full state (fuzz, release + verify env)" + rule + run_convergence +} + +# --- CONVERGENCE: the fuzz suites ---------------------------------------------- +# The N-peer convergence + intent complexity guards. These must stay green; the +# T2 tripwire (mass_ops_do_not_scan_block_index_per_paragraph) lives here. +FUZZ_TESTS=( + object_table_convergence + intent_fuzz + intent_complexity + network_pathology + anti_entropy + swarm_loopback + field_regressions + vendored_patch_guards + doc_io_pump +) +run_convergence() { + bold "CONVERGENCE — fuzz + complexity suites" + rule + local args=() + for t in "${FUZZ_TESTS[@]}"; do args+=(--test "$t"); done + # FLOWSTATE_RICHTEXT_VERIFY: the vendored Src-equivalence assert is + # `cfg!(debug_assertions) || env` — these suites run RELEASE, so without the + # env the assert is compiled out and the "fuzz verifies T1" claim is theater. + FLOWSTATE_RICHTEXT_VERIFY=1 cargo test --release -p flowstate-collab "${args[@]}" + # §act-eleven C2: the A10.3 runtime-index oracle (incremental index == + # fresh-from-projection, every non-rebuild batch) is DEBUG-only. Without this + # leg the canonical gate never runs the oracle that caught four latent + # index-staleness bugs in the act-ten run. + bold "CONVERGENCE — intent suites in DEBUG (A10.3 index oracle armed)" + rule + cargo test -p flowstate-collab --test intent_fuzz --test intent_complexity --test object_table_convergence +} + +# --- §act-eleven C10: the long-horizon randomized SOAK -------------------------- +# A wide seed sweep at larger op counts than the deterministic CI seeds — the +# machine is idle-capable and disposable (corpus policy), so let it run for +# hours. Tune with FUZZ_SOAK_SEEDS / FUZZ_SOAK_ROUNDS / FUZZ_SOAK_OPS; +# FUZZ_PER_OP_CHECK=1 adds per-op tracing for shrink-on-failure. +run_soak() { + bold "SOAK — intent + object/table fuzz, ${FUZZ_SOAK_SEEDS:-500} seeds" + rule + FLOWSTATE_RICHTEXT_VERIFY=1 cargo test --release -p flowstate-collab \ + --test intent_fuzz --test object_table_convergence -- --ignored --nocapture "$@" +} + +# --- §act-eleven / A11.6: the RASTER screenshot net ------------------------------ +# Renders a deterministic editor fixture FULLSCREEN on the live compositor +# (blade needs real Vulkan presentation — Xvfb has no DRI3), captures via the +# COSMIC portal (non-interactive), and compares against the machine-local +# golden with a two-tier tolerance (mean drift + hot-pixel fraction). The +# golden lives in HEAVEN_DIR like the hotpath baselines — fonts/scale/driver +# are machine state. NOTE: the probe window flashes on screen for ~2.5s. +run_screenshot_capture() { + local out_dir="$1" + cargo build --release --quiet -p flowstate-corpus --bin screenshot_probe --bin screenshot_compare + mkdir -p "$out_dir" + rm -f "$out_dir"/*.png + ./target/release/screenshot_probe & + local probe_pid=$! + sleep 1.6 + cosmic-screenshot --interactive=false --modal=false --notify=false -s "$out_dir" >/dev/null 2>&1 || true + wait "$probe_pid" || true + ls -t "$out_dir"/*.png 2>/dev/null | head -1 +} +run_screenshot() { + bold "SCREENSHOT — raster compare vs machine-local golden" + rule + if [[ ! -f "$HEAVEN_DIR/screenshot-golden.png" ]]; then + echo "no golden yet — run './heaven.sh screenshot-baseline' first" >&2 + exit 2 + fi + local shot + shot=$(run_screenshot_capture "$HEAVEN_DIR/screenshot-current") + if [[ -z "$shot" ]]; then + echo "screenshot capture produced no image (portal denied?)" >&2 + exit 2 + fi + ./target/release/screenshot_compare "$HEAVEN_DIR/screenshot-golden.png" "$shot" +} +run_screenshot_baseline() { + bold "SCREENSHOT — capturing new golden" + rule + local shot + shot=$(run_screenshot_capture "$HEAVEN_DIR/screenshot-current") + if [[ -z "$shot" ]]; then + echo "screenshot capture produced no image (portal denied?)" >&2 + exit 2 + fi + cp "$shot" "$HEAVEN_DIR/screenshot-golden.png" + bold "golden saved -> $HEAVEN_DIR/screenshot-golden.png" +} + +# --- SPEED: the perf benches --------------------------------------------------- +# collab_bench: warm field ops (open/import/keystroke/mass-restyle/undo/redo). +# perf_probe: cold vs warm projection phase timing. +# The hotpath tables come from the --features hotpath* builds; kept optional. +require_fixture() { + if [[ -z "${FLOWSTATE_BENCH_FIXTURE:-}" ]]; then + echo "error: set FLOWSTATE_BENCH_FIXTURE=/path/to/large.docx for the benches" >&2 + exit 2 + fi +} +run_bench() { + require_fixture + bold "SPEED — perf_probe (cold/warm projection phases)" + rule + cargo run --release --quiet -p flowstate-corpus --bin perf_probe -- "$FLOWSTATE_BENCH_FIXTURE" || true + echo + bold "SPEED — collab_bench (warm field ops)" + rule + cargo run --release --quiet -p flowstate-corpus --bin collab_bench -- "$FLOWSTATE_BENCH_FIXTURE" || true +} + +run_baseline() { + run_bench | tee "$HEAVEN_DIR/baseline.txt" + bold "baseline saved -> $HEAVEN_DIR/baseline.txt" +} +run_compare() { + run_bench | tee "$HEAVEN_DIR/current.txt" + if [[ -f "$HEAVEN_DIR/baseline.txt" ]]; then + echo + bold "DELTA vs baseline" + rule + diff -u "$HEAVEN_DIR/baseline.txt" "$HEAVEN_DIR/current.txt" || true + else + echo "no baseline yet — run './heaven.sh baseline' first" >&2 + fi +} + +# --- T7.28: STABLE micro-measurement (core-pinned, min-of-N) -------------------- +# Wall-clock on this box swings ~2x from thermal/allocator noise, so a single +# bench run cannot resolve a sub-second cut. `stable` pins the bench to one core +# (taskset, if present — pick an isolated core via HEAVEN_STABLE_CORE) and runs +# it N times (HEAVEN_STABLE_RUNS). Read the MINIMUM across runs: the min is the +# least-contended, most-reproducible estimate. Pre-builds once so compilation +# never pollutes a timed run. Pair with baseline/compare for the delta. +STABLE_RUNS="${HEAVEN_STABLE_RUNS:-5}" +STABLE_CORE="${HEAVEN_STABLE_CORE:-3}" +pin() { + if command -v taskset >/dev/null 2>&1; then taskset -c "$STABLE_CORE" "$@"; else "$@"; fi +} +run_stable() { + require_fixture + bold "SPEED (STABLE) — core $STABLE_CORE, min of $STABLE_RUNS runs (read the MIN, not any single run)" + rule + # Pre-build so run 1 is not paying for compilation. + cargo build --release --quiet -p flowstate-corpus --bin collab_bench + cargo build --release --quiet -p flowstate-corpus --bin perf_probe + for i in $(seq 1 "$STABLE_RUNS"); do + bold "-- run $i/$STABLE_RUNS --" + pin cargo run --release --quiet -p flowstate-corpus --bin collab_bench -- "$FLOWSTATE_BENCH_FIXTURE" || true + done +} + +# --- T8.21: HOTPATH regression gate (top-N alloc/CPU functions) ----------------- +# Wall-clock is noisy (see `stable`), but the hotpath alloc/CPU TABLES are a rich, +# reproducible signal: a regression shows up as a NEW large allocator or a +# function jumping the top-N. `hotpath` runs collab_bench with the alloc + cpu +# hotpath features, extracts the top functions, and diffs them against a saved +# baseline — so a 100 MB allocator that sneaks in is caught by the gate, not by +# eyeball. `hotpath-baseline` saves the current tables as the reference. +run_hotpath() { + require_fixture + bold "HOTPATH — top alloc + CPU functions (collab_bench)" + rule + # §oom-leads #4: structural rounds are counter nets with a known cliff — + # exclude them from the share-gate profile until the remote-structural fix. + export FLOWSTATE_BENCH_SKIP_STRUCTURAL=1 + cargo build --release --features hotpath-alloc --quiet -p flowstate-corpus --bin collab_bench + bold "ALLOC top functions" + cargo run --release --features hotpath-alloc --quiet -p flowstate-corpus --bin collab_bench -- "$FLOWSTATE_BENCH_FIXTURE" 2>&1 \ + | sed -n '/alloc-bytes/,/threads/p' | grep -E "^\| [A-Za-z]" | head -15 | tee "$HEAVEN_DIR/hotpath-alloc.txt" + echo + cargo build --release --features hotpath-cpu --quiet -p flowstate-corpus --bin collab_bench + bold "CPU top functions" + cargo run --release --features hotpath-cpu --quiet -p flowstate-corpus --bin collab_bench -- "$FLOWSTATE_BENCH_FIXTURE" 2>&1 \ + | grep -E "^\| [A-Za-z]" | head -15 | tee "$HEAVEN_DIR/hotpath-cpu.txt" + local gate_failed=0 + for kind in alloc cpu; do + if [[ -f "$HEAVEN_DIR/hotpath-$kind.baseline.txt" ]]; then + echo; bold "GATE vs baseline ($kind top functions)"; rule + hotpath_gate "$kind" || gate_failed=1 + else + echo "(no $kind baseline — run './heaven.sh hotpath-baseline' to arm the gate)" + fi + done + echo "(save the current tables as the reference with: ./heaven.sh hotpath-baseline)" + if (( gate_failed )); then + echo "HOTPATH GATE: FAILED (see above). A new hot function entered the top-N or an existing one grew past tolerance." >&2 + return 1 + fi +} +# The actual gate (was `diff -u ... || true`, which could never fail and diffed +# run-varying raw numbers). Compares the normalized "% Total" column: FAIL if a +# function absent from the baseline top-N appears at >= HEAVEN_GATE_NEW_PCT, or +# a shared function's share grows by more than HEAVEN_GATE_GROWTH_ABS points AND +# HEAVEN_GATE_GROWTH_REL x its baseline share. `::main` rows (the whole-run root +# span) are skipped — their share is definitionally noisy. +hotpath_gate() { + local kind="$1" + awk -F'|' \ + -v new_pct="${HEAVEN_GATE_NEW_PCT:-3.0}" \ + -v grow_abs="${HEAVEN_GATE_GROWTH_ABS:-2.0}" \ + -v grow_rel="${HEAVEN_GATE_GROWTH_REL:-1.5}" ' + function trim(s) { gsub(/^[ \t]+|[ \t]+$/, "", s); return s } + function pct(s) { s = trim(s); sub(/%$/, "", s); return s + 0 } + FNR == 1 { file_ix++ } + NF < 4 { next } + { + name = trim($2); share = pct($(NF - 1)) + if (name == "" || name ~ /::main$/ || name == "Function") next + if (file_ix == 1) base[name] = share + else { cur[name] = share; order[++n] = name } + } + END { + bad = 0 + for (i = 1; i <= n; i++) { + name = order[i]; share = cur[name] + if (!(name in base)) { + if (share >= new_pct) { printf " FAIL new hot function: %s at %.2f%% (>= %.1f%%)\n", name, share, new_pct; bad = 1 } + else printf " note new entrant: %s at %.2f%% (below %.1f%% gate)\n", name, share, new_pct + } else if (share > base[name] * grow_rel && share - base[name] > grow_abs) { + printf " FAIL grew: %s %.2f%% -> %.2f%% (rel x%.2f, +%.2f pts)\n", name, base[name], share, share / (base[name] > 0 ? base[name] : 0.01), share - base[name]; bad = 1 + } + } + if (!bad) print " OK — no new top-N entrant, no function past growth tolerance" + exit bad + }' "$HEAVEN_DIR/hotpath-$kind.baseline.txt" "$HEAVEN_DIR/hotpath-$kind.txt" +} +run_hotpath_baseline() { + # Re-arming IS the answer to a failed gate (a big win shifts every share); + # don't let the gate's exit code abort the save. + run_hotpath || true + cp "$HEAVEN_DIR/hotpath-alloc.txt" "$HEAVEN_DIR/hotpath-alloc.baseline.txt" 2>/dev/null || true + cp "$HEAVEN_DIR/hotpath-cpu.txt" "$HEAVEN_DIR/hotpath-cpu.baseline.txt" 2>/dev/null || true + bold "hotpath baseline saved -> $HEAVEN_DIR/hotpath-{alloc,cpu}.baseline.txt" +} + +case "${1:-gate}" in + fidelity) shift; run_fidelity "$@" ;; + convergence) shift; run_convergence "$@" ;; + hotpath) shift; run_hotpath "$@" ;; + hotpath-baseline) shift; run_hotpath_baseline "$@" ;; + bench) shift; run_bench "$@" ;; + baseline) shift; run_baseline "$@" ;; + compare) shift; run_compare "$@" ;; + stable) shift; run_stable "$@" ;; + watch) shift; run_watch "$@" ;; + verify) shift; run_verify "$@" ;; + docx-roundtrip) shift; run_docx_roundtrip "$@" ;; + docx-fallback) shift; run_docx_fallback "$@" ;; + soak) shift; run_soak "$@" ;; + screenshot) shift; run_screenshot "$@" ;; + screenshot-baseline) shift; run_screenshot_baseline "$@" ;; + all) run_fidelity; echo; run_convergence; echo; run_bench ;; + gate) run_fidelity; echo; run_convergence ;; + *) echo "usage: $0 {gate|fidelity|convergence|bench|baseline|compare|stable|hotpath|hotpath-baseline|watch|verify|docx-roundtrip|all}" >&2; exit 2 ;; +esac diff --git a/helpers/docs/collab_qa.md b/helpers/docs/collab_qa.md new file mode 100644 index 00000000..427b9e93 --- /dev/null +++ b/helpers/docs/collab_qa.md @@ -0,0 +1,97 @@ +# Flowstate Collaboration QA + +Manual QA for P2P `.db8` collaboration. Run with two or three app instances using separate working directories. Use fresh documents unless a scenario says otherwise. + +## Setup + +1. Start instance A and open or create a `.db8` document. +2. Use `Share / Collaborate...` to start a session and copy an invite. +3. Start instance B, use `Join Collaboration Session...`, paste the invite, and verify a new shared tab opens. +4. For three-peer cases, repeat join from instance C using a fresh invite from either A or B. + +## Core Sync + +1. Type in A and verify B/C update live. +2. Type in B and verify A/C update live. +3. Paste multi-paragraph rich text and verify convergence. +4. Exercise multibyte text with the fixture `a\u{00e9}\u{1f30d}\u{2028}x`. +5. Apply paragraph styles, run styles, highlights, underline, and strikethrough from different peers. +6. Insert, move, replace, and delete image/equation/table blocks. +7. Paste an image on A; verify B/C show a loading placeholder then the image. +8. Paste a different image on B; verify A/C can serve it after B has it. + +## Presence + +1. Move the caret with keyboard arrows in A; verify B sees the remote caret move. +2. Select text in B; verify A sees the caret position update. +3. Leave C or kill it; verify presence ages out within 30 seconds. +4. Reopen the share dialog and verify participant count/status remains sensible. + +## Undo And Conflicts + +1. Type several edits on A, then undo and redo; verify only A's edits are undone/redone and all peers converge. +2. Interleave A and B edits in the same paragraph, then undo on A and B independently. +3. Simultaneously edit the same table cell from A and B; verify the block converges with last-writer-wins behavior. +4. Simultaneously split a paragraph on A while B types later in the paragraph; verify no data loss and convergence. + +## Network And Recovery + +1. Disconnect B's network or block traffic, keep editing on B, then reconnect; verify anti-entropy catches up. +2. Let A leave while B and C remain; verify B and C continue editing and converging. +3. Force a large paste above the gossip inline limit; verify blob pull or later anti-entropy convergence. +4. Quit or crash a pathless joined tab; verify a recovery file exists under the temp `flowstate-collab-recovery` directory and opens as a local document on recovery. +5. Test a stale/dead invite; verify join fails with actionable copy. + +## Prompts And Leave + +1. Click `Leave session`; cancel, then leave. Verify the tab remains open as a local document. +2. Close an attached clean path-backed tab; verify leave prompt, then close. +3. Close an attached dirty path-backed tab; verify leave prompt, save prompt, and cancel at each step keeps the session attached. +4. Close a pathless attached tab; verify leave prompt, `Save As...` / `Don't Save` / `Cancel` behavior. +5. Quit with multiple attached tabs; verify the combined leave prompt, then per-tab save prompts. Cancel anywhere must abort quit. + +## Compatibility And Error Cases + +1. Paste a malformed invite; verify inline parse error and no network dial. +2. Use an unsupported-version invite if available; verify clear rejection. +3. Try joining a session already open in the same app; verify it is rejected. +4. Kill the inviter before join; verify join fails without modifying existing documents. +5. Confirm saved files are plain `.db8` snapshots and do not contain CRDT history. + +## Failure Checklist + +| Row | Scenario | Expected Result | +|---|---|---| +| F1 | Garbage or truncated ticket | Inline parse error; no network dial. | +| F2 | Inviter unreachable at join | Join times out with actionable copy. | +| F3 | Inviter reachable but snapshot pull fails | Fallback candidates are tried; all failures end in `JoinFailed`. | +| F4 | Old dead-session ticket | Join times out and copy indicates the session may be over. | +| F5 | Protocol version skew | Join rejects fast or in-session gossip is ignored with a one-time notice. | +| F6 | Lineage violation | Join meta check fails or digest mismatch is ignored; no merge. | +| F7 | Peer crash or kill | Presence ages out within 30 seconds and roster updates. | +| F8 | Transient network loss | Offline pill appears; edits accumulate and resync after reconnect. | +| F9 | Long offline window | Reconnect merges local and remote edits without data loss. | +| F10 | All other peers leave | Session remains attached as `Only you`; no auto-detach. | +| F11 | Gossip receiver lag | Digest pull runs and peers converge. | +| F12 | Oversized blob pull fails | Later anti-entropy recovers the missing update. | +| F13 | Projection drift | Self-check rebuilds from projection and clamps selection. | +| F14 | Remote patch during drag or IME | Patch is deferred, then applied safely. | +| F15 | Asset fetch fails everywhere | Placeholder persists; text sync remains unaffected. | +| F16 | Self-join with own ticket | Friendly error; no dial. | +| F17 | Share on already-attached tab | Attached view opens and can mint a fresh invite. | +| F18 | Close tab or quit while attached | Transactional prompts appear; Cancel aborts the close or quit. | +| F19 | Crash with attached pathless tab | Recovery file opens as a local document on next launch. | +| F20 | Same table cell edited concurrently | Cell converges with documented last-writer-wins behavior. | +| F21 | Peer clock skew | Presence remains sensible; document sync does not depend on wall clock. | + +## Additional Soak Checklist + +1. Simultaneously type in the same paragraph from two peers. +2. Race Enter-split on one peer against typing later in the same paragraph on another peer. +3. Race edits in the same table cell and observe last-writer-wins convergence. +4. Paste an image on each peer and verify asset placeholders resolve. +5. Undo and redo interleaved edits from multiple peers. +6. Leave and rejoin using a fresh ticket. +7. Close a laptop lid or disconnect a peer for 5 minutes, then resync. +8. Cancel close-tab prompts at each step and verify the session stays attached. +9. Run a 30-minute soak with autosave enabled on a path-backed peer. diff --git a/helpers/docs/collaboration.md b/helpers/docs/collaboration.md new file mode 100644 index 00000000..24ef884f --- /dev/null +++ b/helpers/docs/collaboration.md @@ -0,0 +1,49 @@ +# Flowstate Collaboration + +Flowstate collaboration is a transport and presence layer around the same Loro-native document used for local editing and persistence. It does not introduce a second document model, operation log, or save format. + +## Canonical data flow + +Local editing: + +1. `RichTextEditor` resolves input against its current `DocumentProjection` and queues semantic command batches tagged with that projection frontier. +2. The document or session owner sends those commands to the document's single `CrdtRuntimeHandle`. +3. The runtime rejects stale-frontier commands, mutates its owned `LoroDoc`, commits the semantic undo group, and emits a frontier-scoped projection patch or snapshot. +4. The UI applies only that derived projection result. Loro update bytes are persisted and, for a live session, published to peers. + +Remote editing: + +1. The session imports received update bytes directly into the runtime's `LoroDoc`. +2. Pending dependency status triggers immediate version-vector anti-entropy. +3. The runtime's permanent subscription produces projection invalidations and update events. +4. The UI applies derived projection patches or a defensive full projection rebuild. + +Image bytes are synchronized separately through the content-addressed asset store. Loro stores BLAKE3 identities and image metadata; missing bytes render as recoverable placeholders. + +## Ownership + +Each open document has one CRDT runtime actor. The actor owns: + +- the canonical `LoroDoc`; +- permanent Loro subscriptions; +- the per-replica Loro `UndoManager`; +- package persistence and revision checkpoints; +- projection construction and invalidation; +- update import/export and asset metadata coordination. + +A joined collaboration tab installs the session's existing runtime handle directly. It does not create a temporary mirror runtime. + +## Presence + +Presence is ephemeral. Carets and selections are encoded as Loro cursors with affinity, visual gravity, and direction. User identity is distinct from replica identity; every active replica uses a unique Loro peer ID. + +## Persistence + +Saved `.db8` files are the chunked Loro-native Flowstate package. Collaboration does not change the file format and does not maintain session-only canonical history. Explicit saves create package checkpoints/revision boundaries. PDF source recovery embeds only these package bytes. + +## Verification + +- `crates/flowstate-collab/src/crdt_runtime.rs` contains runtime, projection, undo, table, revision, and anti-entropy coverage. +- `crates/flowstate-collab/src/crdt_runtime_actor.rs` covers actor serialization and stale-frontier error preservation. +- `crates/gpui-flowtext/src/rich_text/tests/collab_capture.rs` covers semantic command capture and projection patch application. +- `helpers/docs/collab_qa.md` contains the manual multi-peer QA matrix. diff --git a/helpers/scripts/register_windows_file_associations.ps1 b/helpers/scripts/register_windows_file_associations.ps1 index 47c1ccca..f6ee06f5 100644 --- a/helpers/scripts/register_windows_file_associations.ps1 +++ b/helpers/scripts/register_windows_file_associations.ps1 @@ -1,6 +1,7 @@ param( [string]$ExecutablePath = "", - [switch]$Machine + [switch]$Machine, + [switch]$Unregister ) $ErrorActionPreference = "Stop" @@ -9,9 +10,24 @@ if ([string]::IsNullOrWhiteSpace($ExecutablePath)) { $ExecutablePath = Join-Path (Resolve-Path ".").Path "target\release\flowstate.exe" } -$ExecutablePath = (Resolve-Path -LiteralPath $ExecutablePath).Path $root = if ($Machine) { "Registry::HKEY_LOCAL_MACHINE\Software\Classes" } else { "Registry::HKEY_CURRENT_USER\Software\Classes" } +if ($Unregister) { + @( + "$root\.db8", + "$root\.flowinvite", + "$root\.docx\OpenWithProgids", + "$root\Flowstate.db8", + "$root\Flowstate.invite", + "$root\Flowstate.docx.import", + "$root\flowstate" + ) | ForEach-Object { Remove-Item -LiteralPath $_ -Recurse -Force -ErrorAction SilentlyContinue } + Write-Host "Removed Flowstate document associations and the flowstate:// protocol" + exit 0 +} + +$ExecutablePath = (Resolve-Path -LiteralPath $ExecutablePath).Path + function Set-KeyValue { param([string]$Path, [string]$Name, [string]$Value) New-Item -Path $Path -Force | Out-Null @@ -34,7 +50,16 @@ function Register-Extension { Set-KeyValue -Path "$root\$ProgId\shell\open\command" -Name "" -Value "`"$ExecutablePath`" `"%1`"" } +function Register-UrlProtocol { + param([string]$Scheme, [string]$Description) + Set-KeyValue -Path "$root\$Scheme" -Name "" -Value $Description + Set-KeyValue -Path "$root\$Scheme" -Name "URL Protocol" -Value "" + Set-KeyValue -Path "$root\$Scheme\shell\open\command" -Name "" -Value "`"$ExecutablePath`" `"%1`"" +} + Register-Extension -Extension ".db8" -ProgId "Flowstate.db8" -Description "Flowstate Debate Document" -MakeDefault $true +Register-Extension -Extension ".flowinvite" -ProgId "Flowstate.invite" -Description "Flowstate Collaboration Invite" -MakeDefault $true Register-Extension -Extension ".docx" -ProgId "Flowstate.docx.import" -Description "Microsoft Word Document imported by Flowstate" -MakeDefault $false +Register-UrlProtocol -Scheme "flowstate" -Description "Flowstate collaboration invite" -Write-Host "Registered .db8 as a Flowstate document and added Flowstate to the .docx Open With list for $ExecutablePath" +Write-Host "Registered Flowstate document associations and the flowstate:// invite protocol for $ExecutablePath" diff --git a/tools/README-raw-authority-guard.md b/tools/README-raw-authority-guard.md new file mode 100644 index 00000000..a32e5951 --- /dev/null +++ b/tools/README-raw-authority-guard.md @@ -0,0 +1,25 @@ +# Raw-authority CI guard + +`check_raw_authority.sh` enforces invariant 6 of `flowstate_loro_first_spec.md` (§13.2): +after the Loro-first cutover, production code must never reference the condemned +raw projection-space authority surfaces (semantic-command batching, projection +replay/rebase, pending-edit flushing, stale-projection reconciliation), and the +local write path (`crates/flowstate-collab/src/local_write/`) must never grow a +frontier-validity concept (`base_frontier` there is banned; the legitimate +`ProjectionPatchBatch.base_frontier` patch-stream metadata elsewhere is not flagged). + +## Run it + + tools/check_raw_authority.sh + +It scans `crates/*/src` (excluding `vendor/`, `/tests/` paths, and `*tests.rs` +files), prints every offending `file:line`, and exits non-zero on any hit. + +## When it fires + +Never allowlist or suppress a hit. The fix is always to express the mutation +through the `LocalDocHandle` intent API so the CRDT document remains the sole +authority, then delete the raw-authority reference. + +Note: this guard is wired into CI only after the stage-4 cutover completes; +until then, hits are expected while the migration is in progress. diff --git a/tools/check_raw_authority.sh b/tools/check_raw_authority.sh new file mode 100755 index 00000000..d1740715 --- /dev/null +++ b/tools/check_raw_authority.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# check_raw_authority.sh — CI guard for the Loro-first cutover +# (flowstate_loro_first_spec.md §13.2, invariant 6). +# +# WHY: Under the ratified Loro-first architecture, raw projection-space +# authority is unrepresentable by law — all local mutation must flow through +# the LocalDocHandle intent API so the CRDT document is the single source of +# truth. The identifiers banned below are the condemned raw-authority surface: +# semantic-command batching, projection replay/rebase, pending-edit flushing, +# and stale-projection reconciliation. Once the stage-4 cutover completes, +# production code must never reference them again; this guard stops silent +# reintroduction. Additionally, `base_frontier` is banned specifically inside +# crates/flowstate-collab/src/local_write/ — the write path must not grow a +# frontier-validity concept (ProjectionPatchBatch.base_frontier elsewhere is +# legitimate patch-stream ordering metadata and is NOT flagged). +# +# NOTE: While the cutover is in progress this script is expected to report +# hits. It gets wired into CI at cutover completion (after stage 4). + +set -u + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +BANNED_IDENTIFIERS='SemanticEditCommand|SemanticCommandBatch|apply_editor_commands|apply_editor_transaction|pending_semantic_edits|take_pending_session_edits|take_pending_runtime_edits|replace_document_projection_replaying_pending|rebuild_visible_from_committed|replay_semantic_command_on_projection|StaleProjectionError|StableEditorSelection|flush_local_edits|flush_document_runtime_edits|semantic_commands_for_span_edit' + +fail=0 +hits_file="$(mktemp)" +trap 'rm -f "$hits_file"' EXIT + +# --- Pass 1: globally banned identifiers in production source ----------------- +# Scope: crates/*/src, Rust sources only. Excluded: vendor/ directories, any +# path containing /tests/, and files ending in `tests.rs` (which also covers +# `_tests.rs`). +grep -rnE --include='*.rs' --exclude-dir=vendor "$BANNED_IDENTIFIERS" \ + "$ROOT"/crates/*/src 2>/dev/null \ + | grep -vE '(/tests/|tests\.rs:)' >> "$hits_file" || true + +# --- Pass 2: base_frontier banned only inside the local write path ------------ +LOCAL_WRITE_DIR="$ROOT/crates/flowstate-collab/src/local_write" +if [ -d "$LOCAL_WRITE_DIR" ]; then + # commit.rs legitimately constructs ProjectionPatchBatch { base_frontier, .. } + # (patch-STREAM ordering metadata, spec §7) — everything else in the write + # path must not grow a frontier-validity concept. + grep -rnE --include='*.rs' --exclude-dir=vendor 'base_frontier' \ + "$LOCAL_WRITE_DIR" 2>/dev/null \ + | grep -vE '(/tests/|tests\.rs:|/commit\.rs:)' >> "$hits_file" || true +fi + +# --- Report -------------------------------------------------------------------- +if [ -s "$hits_file" ]; then + echo "FORBIDDEN raw-authority references found in production source:" >&2 + echo >&2 + cat "$hits_file" >&2 + echo >&2 + count="$(wc -l < "$hits_file" | tr -d ' ')" + echo "check_raw_authority: FAIL — $count offending line(s)." >&2 + echo "Do not allowlist. Migrate the code to the LocalDocHandle intent API" >&2 + echo "(see flowstate_loro_first_spec.md §13.2, invariant 6, and" >&2 + echo "tools/README-raw-authority-guard.md)." >&2 + fail=1 +else + echo "check_raw_authority: OK — no raw-authority references in production source." +fi + +exit "$fail" diff --git a/vendor/generic-btree/Cargo.lock b/vendor/generic-btree/Cargo.lock new file mode 100644 index 00000000..399f7a9c --- /dev/null +++ b/vendor/generic-btree/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "generic-btree" +version = "0.10.7" diff --git a/vendor/generic-btree/Cargo.toml b/vendor/generic-btree/Cargo.toml new file mode 100644 index 00000000..ce42b60b --- /dev/null +++ b/vendor/generic-btree/Cargo.toml @@ -0,0 +1,108 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "generic-btree" +version = "0.10.7" +authors = ["zxch3n "] +build = false +include = [ + "Cargo.toml", + "/src/**/*.rs", + "/benches/**/*.rs", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Generic BTree for versatile purposes" +homepage = "https://github.com/loro-dev/generic-btree" +documentation = "https://docs.rs/generic-btree" +readme = "README.md" +keywords = [ + "btree", + "data-structure", +] +license = "MIT" +repository = "https://github.com/loro-dev/generic-btree" + +[features] +test = [] + +[lib] +name = "generic_btree" +path = "src/lib.rs" + +[dependencies.arref] +version = "0.1.0" + +[dependencies.heapless] +version = "0.9.1" + +[dependencies.itertools] +version = "0.11.0" + +[dependencies.proc-macro2] +version = "1.0.67" + +[dependencies.rustc-hash] +version = "2.1.1" + +[dependencies.thunderdome] +version = "0.6.2" +package = "loro-thunderdome" + +[dev-dependencies.ahash] +version = "0.8.11" + +[dev-dependencies.arbitrary] +version = "1.2.3" +features = ["derive"] + +[dev-dependencies.color-backtrace] +version = "0.7.0" + +[dev-dependencies.criterion] +version = "0.7.0" + +[dev-dependencies.ctor] +version = "0.1.26" + +[dev-dependencies.flate2] +version = "1.0.25" + +[dev-dependencies.jumprope] +version = "1.0.0" + +[dev-dependencies.pprof] +version = "0.15.0" +features = [ + "flamegraph", + "criterion", + "frame-pointer", +] + +[dev-dependencies.rand] +version = "0.8.5" + +[dev-dependencies.serde_json] +version = "1.0.93" + +[profile.release] +debug = 2 + +[workspace] + +[package.metadata.cargo-machete] +# Vendored crate: unused-in-our-build deps kept to minimize diff vs upstream. +ignored = ["proc-macro2"] diff --git a/vendor/generic-btree/README.md b/vendor/generic-btree/README.md new file mode 100644 index 00000000..593b12ea --- /dev/null +++ b/vendor/generic-btree/README.md @@ -0,0 +1,12 @@ +# Generic B-Tree + +It’s a pure safe BTree that can be used to build your own special-purpose btree +data structure. It’s mainly developed to optimize the performance of Loro CRDT’s +components. + +It can be used to build: + +- Rope +- Run length encoding data structure +- RangeMap that uses range as its key +- BTreeSet & BTreeMap diff --git a/vendor/generic-btree/src/generic_impl/gap_buffer.rs b/vendor/generic-btree/src/generic_impl/gap_buffer.rs new file mode 100644 index 00000000..b9ce6f04 --- /dev/null +++ b/vendor/generic-btree/src/generic_impl/gap_buffer.rs @@ -0,0 +1,293 @@ +use std::ops::{Range, RangeBounds}; + +use crate::rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}; + +#[cfg(not(test))] +pub const MAX_STRING_SIZE: usize = 128; +#[cfg(test)] +pub const MAX_STRING_SIZE: usize = 12; + +#[derive(Debug, Clone)] +pub(super) struct GapBuffer { + buffer: [u8; MAX_STRING_SIZE], + gap_start: u16, + gap_len: u16, +} + +impl GapBuffer { + pub fn new() -> Self { + Self { + buffer: [0; MAX_STRING_SIZE], + gap_start: 0, + gap_len: MAX_STRING_SIZE as u16, + } + } + + pub fn shift_at(&mut self, index: usize) { + if index > self.len() { + panic!("index {} out of range len={}", index, self.len()); + } + + let gap_start = self.gap_start as usize; + let gap_end = (self.gap_start + self.gap_len) as usize; + match index.cmp(&gap_start) { + std::cmp::Ordering::Equal => {} + std::cmp::Ordering::Less => { + let gap_move = gap_start - index; + self.buffer + .copy_within(index..gap_start, gap_end - gap_move); + self.gap_start -= gap_move as u16; + } + std::cmp::Ordering::Greater => { + let gap_move = index - gap_start; + let move_end = self.buffer.len().min(gap_end + gap_move); + self.buffer.copy_within(gap_end..move_end, gap_start); + self.gap_start += gap_move as u16; + } + } + } + + #[allow(unused)] + pub fn push(&mut self, value: u8) -> Result<(), ()> { + if self.gap_len == 0 { + return Err(()); + } + self.buffer[self.gap_start as usize] = value; + self.gap_start += 1; + self.gap_len -= 1; + Ok(()) + } + + #[inline(always)] + pub fn push_bytes(&mut self, bytes: &[u8]) -> Result<(), ()> { + self.insert_bytes(self.len(), bytes) + } + + pub fn insert_bytes(&mut self, index: usize, bytes: &[u8]) -> Result<(), ()> { + if (self.gap_len as usize) < bytes.len() { + return Err(()); + } + + self.shift_at(index); + self.buffer[index..index + bytes.len()].copy_from_slice(bytes); + self.gap_start += bytes.len() as u16; + self.gap_len -= bytes.len() as u16; + Ok(()) + } + + pub fn insert_bytes_pair( + &mut self, + index: usize, + (left, right): (&[u8], &[u8]), + ) -> Result<(), ()> { + let len = left.len() + right.len(); + if (self.gap_len as usize) < len { + return Err(()); + } + + self.shift_at(index); + self.buffer[index..index + left.len()].copy_from_slice(left); + self.buffer[index + left.len()..index + len].copy_from_slice(right); + self.gap_start += len as u16; + self.gap_len -= len as u16; + Ok(()) + } + + pub fn delete(&mut self, range: impl RangeBounds) { + let mut start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + let mut end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.len(), + }; + + end = end.min(self.len()); + start = start.min(self.len()).min(end); + if start == end { + return; + } + + let len = end - start; + self.shift_at(end); + self.gap_start = start as u16; + self.gap_len += len as u16; + } + + #[inline] + pub fn capacity(&self) -> usize { + self.buffer.len() + } + + #[inline] + pub fn len(&self) -> usize { + self.buffer.len() - self.gap_len as usize + } + + pub fn as_bytes(&self) -> (&[u8], &[u8]) { + ( + &self.buffer[..self.gap_start as usize], + &self.buffer[(self.gap_start + self.gap_len) as usize..], + ) + } + + #[allow(unused)] + pub fn to_vec(&self) -> Vec { + let mut vec = Vec::with_capacity(self.len()); + let (left, right) = self.as_bytes(); + vec.extend_from_slice(left); + vec.extend_from_slice(right); + vec + } + + pub(crate) fn from_str(elem: &str) -> impl Iterator + '_ { + let mut i = 0; + let elem = elem.as_bytes(); + std::iter::from_fn(move || { + if i >= elem.len() { + return None; + } + + let mut gb = GapBuffer::new(); + gb.push_bytes(&elem[i..(i + MAX_STRING_SIZE).min(elem.len())]) + .unwrap(); + i += MAX_STRING_SIZE; + Some(gb) + }) + } +} + +impl HasLength for GapBuffer { + fn rle_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for GapBuffer { + fn _slice(&self, range: Range) -> Self { + let mut gb = Self::new(); + let start = range.start; + let end = range.end; + + let (l, r) = self.as_bytes(); + if start < l.len() { + gb.push_bytes(&l[start..end.min(l.len())]).unwrap(); + } + if end > l.len() { + gb.push_bytes(&r[start.saturating_sub(l.len())..end.saturating_sub(l.len())]) + .unwrap(); + } + + debug_assert_eq!(gb.len(), end - start); + gb + } + + fn slice_(&mut self, range: impl RangeBounds) + where + Self: Sized, + { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.len(), + }; + + self.delete(end..); + self.delete(..start); + debug_assert_eq!(self.len(), end - start); + } + + fn split(&mut self, pos: usize) -> Self + where + Self: Sized, + { + self.shift_at(pos); + let right = self.as_bytes().1; + let mut r = Self::new(); + r.push_bytes(right).unwrap(); + self.gap_len = (self.capacity() - pos) as u16; + r + } +} + +impl Mergeable for GapBuffer { + fn can_merge(&self, rhs: &Self) -> bool { + self.len() + rhs.len() <= MAX_STRING_SIZE + } + + fn merge_right(&mut self, rhs: &Self) { + let pair = rhs.as_bytes(); + self.insert_bytes_pair(self.len(), pair).unwrap(); + } + + fn merge_left(&mut self, left: &Self) { + let pair = left.as_bytes(); + self.insert_bytes_pair(0, pair).unwrap(); + } +} + +impl TryInsert for GapBuffer { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + if self.len() + elem.len() > MAX_STRING_SIZE { + return Err(elem); + } + + let pair = elem.as_bytes(); + self.insert_bytes_pair(pos, pair).unwrap(); + Ok(()) + } +} + +impl CanRemove for GapBuffer { + fn can_remove(&self) -> bool { + self.len() == 0 + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn basic() { + let mut gb: GapBuffer = GapBuffer::new(); + gb.insert_bytes(0, &[3, 8]).unwrap(); + assert_eq!(gb.to_vec(), vec![3, 8]); + gb.insert_bytes(1, &[4, 5, 6]).unwrap(); + assert_eq!(gb.to_vec(), vec![3, 4, 5, 6, 8]); + assert_eq!(gb.len(), 5); + gb.insert_bytes(4, &[7]).unwrap(); + assert_eq!(gb.to_vec(), vec![3, 4, 5, 6, 7, 8]); + gb.insert_bytes(0, &[1, 2, 9, 9]).unwrap(); + assert_eq!(gb.to_vec(), vec![1, 2, 9, 9, 3, 4, 5, 6, 7, 8]); + gb.delete(2..4); + assert_eq!(gb.len(), 8); + let (left, right) = gb.as_bytes(); + assert_eq!(left, &[1, 2]); + assert_eq!(right, &[3, 4, 5, 6, 7, 8]); + assert_eq!(gb.to_vec(), vec![1, 2, 3, 4, 5, 6, 7, 8]) + } + + #[test] + fn slice() { + let mut gb = GapBuffer::new(); + gb.push_bytes(&[0, 1, 2, 3, 4, 5, 6, 7]).unwrap(); + gb.shift_at(5); + let b = gb.slice(2..5); + assert_eq!(b.to_vec(), vec![2, 3, 4]); + + gb.slice_(2..5); + assert_eq!(gb.to_vec(), vec![2, 3, 4]); + } +} diff --git a/vendor/generic-btree/src/generic_impl/len_finder.rs b/vendor/generic-btree/src/generic_impl/len_finder.rs new file mode 100644 index 00000000..1fe0d6f8 --- /dev/null +++ b/vendor/generic-btree/src/generic_impl/len_finder.rs @@ -0,0 +1,91 @@ +use std::fmt::Debug; + +use thunderdome::Index; + +use crate::rle::HasLength; +use crate::{BTreeTrait, FindResult, Query}; + +/// A generic length finder +pub struct LengthFinder { + pub left: usize, + pub slot: u8, + pub parent: Option, +} + +impl LengthFinder { + #[inline(always)] + pub fn new() -> Self { + Self { + left: 0, + slot: 0, + parent: None, + } + } +} + +impl Default for LengthFinder { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +pub trait UseLengthFinder { + fn get_len(cache: &B::Cache) -> usize; +} + +impl + UseLengthFinder> Query + for LengthFinder +{ + type QueryArg = usize; + + #[inline(always)] + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + slot: 0, + parent: None, + } + } + + #[inline(always)] + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[crate::Child], + ) -> crate::FindResult { + let mut last_left = self.left; + let is_internal = child_caches.first().unwrap().is_internal(); + for (i, cache) in child_caches.iter().enumerate() { + let len = B::get_len(&cache.cache); + if self.left >= len { + last_left = self.left; + self.left -= len; + } else { + if is_internal { + self.parent = Some(cache.arena.unwrap()); + } else { + self.slot = i as u8; + } + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + if is_internal { + self.parent = Some(child_caches.last().unwrap().arena.unwrap()); + } else { + self.slot = child_caches.len() as u8 - 1; + } + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + #[inline(always)] + fn confirm_elem( + &mut self, + _: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + (self.left, self.left < elem.rle_len()) + } +} diff --git a/vendor/generic-btree/src/generic_impl/mod.rs b/vendor/generic-btree/src/generic_impl/mod.rs new file mode 100644 index 00000000..8272773b --- /dev/null +++ b/vendor/generic-btree/src/generic_impl/mod.rs @@ -0,0 +1,7 @@ +mod gap_buffer; +mod len_finder; +mod ord; +mod rope; +pub use len_finder::{LengthFinder, UseLengthFinder}; +pub use ord::{OrdTreeMap, OrdTreeSet}; +pub use rope::Rope; diff --git a/vendor/generic-btree/src/generic_impl/ord.rs b/vendor/generic-btree/src/generic_impl/ord.rs new file mode 100644 index 00000000..cd570fea --- /dev/null +++ b/vendor/generic-btree/src/generic_impl/ord.rs @@ -0,0 +1,408 @@ +use core::fmt::Debug; +use std::cmp::Ordering; +use std::ops::Range; + +use crate::rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}; +use crate::{BTree, BTreeTrait, FindResult, Query, SplitInfo}; + +#[derive(Debug)] +#[repr(transparent)] +struct OrdTrait { + _phantom: core::marker::PhantomData<(Key, Value)>, +} + +#[derive(Debug)] +pub struct OrdTreeMap { + tree: BTree>, + len: usize, +} + +#[derive(Debug)] +pub struct OrdTreeSet(OrdTreeMap); + +impl OrdTreeMap { + #[inline(always)] + pub fn new() -> Self { + Self { + tree: BTree::new(), + len: 0, + } + } + + #[inline(always)] + pub fn insert(&mut self, key: Key, value: Value) { + let Some(result) = self.tree.query::>(&key) else { + self.len += 1; + self.tree.push(Unmergeable((key, value))); + return; + }; + + if !result.found { + self.len += 1; + let tree = &mut self.tree; + let data = Unmergeable((key, value)); + let index = result.leaf(); + let leaf = tree.leaf_nodes.get_mut(index.0).unwrap(); + let parent = leaf.parent(); + + let mut is_full = false; + // Try to merge + if result.cursor.offset == 0 && data.can_merge(&leaf.elem) { + leaf.elem.merge_left(&data); + } else if result.cursor.offset == leaf.elem.rle_len() && leaf.elem.can_merge(&data) { + leaf.elem.merge_right(&data); + } else { + // Insert new leaf node + let child = tree.alloc_leaf_child(data, parent.unwrap_internal()); + let SplitInfo { + parent_idx: parent_index, + insert_slot: insert_index, + .. + } = tree.split_leaf_if_needed(result.cursor); + let parent = tree.in_nodes.get_mut(parent_index).unwrap(); + parent.children.insert(insert_index, child).unwrap(); + is_full = parent.is_full(); + } + + tree.recursive_update_cache(parent, false, None); + if is_full { + tree.split(parent); + } + } else { + let leaf = self.tree.get_elem_mut(result.leaf()).unwrap(); + leaf.0 .1 = value; + } + } + + #[inline(always)] + pub fn delete(&mut self, key: &Key) -> Option<(Key, Value)> { + let q = self.tree.query::>(key)?; + match self.tree.remove_leaf(q.cursor) { + Some(v) => { + self.len -= 1; + Some(v.0) + } + None => None, + } + } + + #[inline(always)] + pub fn iter(&self) -> impl Iterator { + self.tree.iter().map(|x| &x.0) + } + + #[inline(always)] + pub fn iter_key(&self) -> impl Iterator { + self.tree.iter().map(|x| &x.0 .0) + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.len + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + #[allow(unused)] + pub(crate) fn check(&self) { + self.tree.check() + } +} + +impl OrdTreeSet { + #[inline(always)] + pub fn new() -> Self { + Self(OrdTreeMap::new()) + } + + #[inline(always)] + pub fn insert(&mut self, key: Key) { + self.0.insert(key, ()); + } + + #[inline(always)] + pub fn delete(&mut self, key: &Key) -> bool { + self.0.delete(key).is_some() + } + + #[inline(always)] + pub fn iter(&self) -> impl Iterator { + self.0.iter_key() + } + + pub fn len(&self) -> usize { + self.0.len + } + + pub fn is_empty(&self) -> bool { + self.0.len == 0 + } + + #[allow(unused)] + fn check(&self) { + self.0.check() + } +} + +impl Default for OrdTreeSet { + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl Default + for OrdTreeMap +{ + #[inline(always)] + fn default() -> Self { + Self::new() + } +} + +impl Default for OrdTrait { + #[inline(always)] + fn default() -> Self { + Self { + _phantom: Default::default(), + } + } +} + +#[repr(transparent)] +#[derive(Debug, Clone)] +pub struct Unmergeable(T); + +impl HasLength for Unmergeable { + fn rle_len(&self) -> usize { + 1 + } +} + +impl Sliceable for Unmergeable { + fn _slice(&self, range: Range) -> Self { + if range.end - range.start != 1 { + panic!("Invalid range"); + } + + self.clone() + } +} + +impl Mergeable for Unmergeable { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } +} + +impl TryInsert for Unmergeable { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> { + Err(elem) + } +} + +impl CanRemove for Unmergeable { + fn can_remove(&self) -> bool { + false + } +} + +impl BTreeTrait for OrdTrait { + type Elem = Unmergeable<(Key, Value)>; + type Cache = Option<(Key, Key)>; + type CacheDiff = (); + const USE_DIFF: bool = false; + + #[inline(always)] + fn calc_cache_internal(cache: &mut Self::Cache, caches: &[crate::Child]) { + if caches.is_empty() { + return; + } + + *cache = Some(( + caches[0].cache.as_ref().unwrap().0.clone(), + caches[caches.len() - 1].cache.as_ref().unwrap().1.clone(), + )); + } + + #[inline(always)] + fn apply_cache_diff(_: &mut Self::Cache, _: &Self::CacheDiff) { + unreachable!() + } + + #[inline(always)] + fn merge_cache_diff(_: &mut Self::CacheDiff, _: &Self::CacheDiff) {} + + #[inline(always)] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + Some((elem.0 .0.clone(), elem.0 .0.clone())) + } + + #[inline(always)] + fn new_cache_to_diff(_: &Self::Cache) -> Self::CacheDiff {} + + fn sub_cache(_: &Self::Cache, _: &Self::Cache) -> Self::CacheDiff {} +} + +impl Query> + for OrdTrait +{ + type QueryArg = Key; + + #[inline(always)] + fn init(_target: &Self::QueryArg) -> Self { + Self::default() + } + + #[inline] + fn find_node( + &mut self, + target: &Self::QueryArg, + child_caches: &[crate::Child>], + ) -> crate::FindResult { + let result = child_caches.binary_search_by(|x| { + let (min, max) = x.cache.as_ref().unwrap(); + if target < min { + core::cmp::Ordering::Greater + } else if target > max { + core::cmp::Ordering::Less + } else { + core::cmp::Ordering::Equal + } + }); + match result { + Ok(i) => FindResult::new_found(i, 0), + Err(i) => FindResult::new_missing( + i.min(child_caches.len() - 1), + if i == child_caches.len() { 1 } else { 0 }, + ), + } + } + + #[inline(always)] + fn confirm_elem( + &mut self, + q: &Self::QueryArg, + elem: & as BTreeTrait>::Elem, + ) -> (usize, bool) { + match q.cmp(&elem.0 .0) { + Ordering::Less => (0, false), + Ordering::Equal => (0, true), + Ordering::Greater => (1, false), + } + } +} + +#[cfg(test)] +mod test { + use std::cmp::Ordering; + + use rand::{Rng, SeedableRng}; + + use crate::HeapVec; + + use super::*; + + #[test] + fn test() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + let mut rng = rand::rngs::StdRng::seed_from_u64(123); + let mut data: HeapVec = (0..1000).map(|_| rng.gen()).collect(); + for &value in data.iter() { + tree.insert(value); + } + data.sort_unstable(); + assert_eq!(tree.iter().copied().collect::>(), data); + tree.check(); + } + + #[test] + fn test_delete() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + tree.insert(12); + tree.delete(&12); + assert_eq!(tree.len(), 0); + } + + #[test] + fn test_compare_pos() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + for i in 0..100 { + tree.insert(i); + } + + for i in 0..99 { + let a = tree.0.tree.query::>(&i).unwrap(); + assert_eq!( + tree.0.tree.compare_pos(a.cursor(), a.cursor()), + Ordering::Equal + ); + for j in i + 1..100 { + let b = tree.0.tree.query::>(&j).unwrap(); + assert_eq!( + tree.0.tree.compare_pos(a.cursor(), b.cursor()), + Ordering::Less + ); + assert_eq!( + tree.0.tree.compare_pos(b.cursor(), a.cursor()), + Ordering::Greater + ); + } + } + } + + mod move_event_test { + + use super::*; + + #[test] + fn test() { + let mut tree: OrdTreeMap = OrdTreeMap::new(); + let mut rng = rand::rngs::StdRng::seed_from_u64(123); + let mut data: HeapVec = (0..1000).map(|_| rng.gen()).collect(); + for &value in data.iter() { + tree.insert(value, 0); + } + for value in data.drain(0..100) { + tree.delete(&value); + } + for value in data.drain(0..800) { + tree.delete(&value); + } + tree.tree.check(); + for _ in (0..100).rev() { + tree.delete(&data.pop().unwrap()); + } + } + } + + #[test] + #[ignore] + fn depth_test() { + let mut tree: OrdTreeSet = OrdTreeSet::new(); + for i in 0..2_100_000 { + tree.insert(i as u64); + let m = (!i) + 1; + if (i & m) == i { + eprintln!( + "i={}, Depth={}, Avg Children={}", + i, + tree.0.tree.depth(), + tree.0.tree.internal_avg_children_num() + ); + } + } + tree.check(); + } +} diff --git a/vendor/generic-btree/src/generic_impl/rope.rs b/vendor/generic-btree/src/generic_impl/rope.rs new file mode 100644 index 00000000..e1c0c71a --- /dev/null +++ b/vendor/generic-btree/src/generic_impl/rope.rs @@ -0,0 +1,3442 @@ +extern crate alloc; + +use core::ops::RangeBounds; +use std::assert_eq; +use std::fmt::Display; + +use crate::generic_impl::gap_buffer::MAX_STRING_SIZE; +use crate::rle::Sliceable; +use crate::{BTree, BTreeTrait, LeafIndex, LengthFinder, QueryResult}; + +use super::gap_buffer::GapBuffer; +use super::len_finder::UseLengthFinder; + +#[derive(Debug)] +struct RopeTrait; + +#[derive(Debug)] +struct Cursor { + pos: usize, + leaf: LeafIndex, +} + +// TODO: move Rope into a separate project +#[derive(Debug)] +pub struct Rope { + tree: BTree, + cursor: Option, +} + +impl UseLengthFinder for RopeTrait { + #[inline(always)] + fn get_len(cache: &::Cache) -> usize { + *cache as usize + } +} + +impl Rope { + #[inline(always)] + pub fn len(&self) -> usize { + self.tree.root_cache as usize + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.tree.root_cache == 0 + } + + pub fn insert(&mut self, index: usize, elem: &str) { + if index > self.len() { + panic!("index {} out of range len={}", index, self.len()); + } + + if self.is_empty() { + for chunk in GapBuffer::from_str(elem) { + self.tree.push(chunk); + } + return; + } + + if let Some(Cursor { pos, leaf }) = self.cursor { + if pos <= index { + let node = self.tree.leaf_nodes.get(leaf.0).unwrap(); + if index <= pos + node.elem.len() { + let mut success = true; + let offset = index - pos; + let valid = self + .tree + .update_leaf(leaf, |leaf| { + if leaf.len() + elem.len() < MAX_STRING_SIZE { + leaf.insert_bytes(offset, elem.as_bytes()).unwrap(); + (true, None, None) + } else { + let mut right = leaf.split(offset); + if leaf.len() + elem.len() < MAX_STRING_SIZE { + success = leaf.push_bytes(elem.as_bytes()).is_ok(); + } else { + success = right.insert_bytes(0, elem.as_bytes()).is_ok(); + } + + (true, Some(right), None) + } + }) + .0; + + if !valid { + self.cursor = None; + } + + if success { + return; + } + } + } + } + + let (q, f) = self.tree.query_with_finder_return::(&index); + self.cursor = q.and_then(|q| { + if q.offset() == 0 { + if f.slot == 0 || f.parent.is_none() { + None + } else { + let node = self.tree.in_nodes.get(f.parent.unwrap()).unwrap(); + let child = &node.children[f.slot as usize - 1]; + Some(Cursor { + pos: index - child.cache as usize, + leaf: child.arena.unwrap().into(), + }) + } + } else { + Some(Cursor { + pos: index - q.offset(), + leaf: q.leaf(), + }) + } + }); + + self.tree + .insert_many_by_cursor(q.map(|x| x.cursor), GapBuffer::from_str(elem)); + } + + pub fn delete_range(&mut self, range: impl RangeBounds) { + if self.is_empty() { + return; + } + + let start = match range.start_bound() { + core::ops::Bound::Included(x) => *x, + core::ops::Bound::Excluded(x) => *x + 1, + core::ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + core::ops::Bound::Included(&x) => x + 1, + core::ops::Bound::Excluded(&x) => x, + core::ops::Bound::Unbounded => self.len(), + }; + let end = end.min(self.len()); + let start = start.min(end); + if start == end { + return; + } + + if let Some(Cursor { pos, leaf }) = self.cursor { + if pos <= start { + let node = self.tree.leaf_nodes.get(leaf.0).unwrap(); + if end <= pos + node.elem.len() { + let start_offset = start - pos; + let end_offset = end - pos; + let valid = self + .tree + .update_leaf(leaf, |leaf| { + leaf.delete(start_offset..end_offset); + (true, None, None) + }) + .0; + + if !valid { + self.cursor = None; + } + + return; + } + } + } + + if end - start == 1 { + let q = self + .tree + .update_leaf_by_search::(&start, |leaf, pos| { + leaf.delete(pos.cursor.offset..pos.cursor.offset + 1); + Some((-1, None, None)) + }); + self.cursor = q.0.map(|q| Cursor { + pos: start - q.offset, + leaf: q.leaf, + }); + + return; + } + + self.cursor = None; + let from = self.tree.query::(&start); + let to = self.tree.query::(&end); + match (from, to) { + (Some(from), Some(to)) if from.cursor.leaf == to.cursor.leaf => { + let leaf = self.tree.leaf_nodes.get_mut(from.arena()).unwrap(); + if from.cursor.offset == 0 && to.cursor.offset == leaf.elem.len() { + // delete the whole leaf + self.tree.remove_leaf(from.cursor); + } else { + leaf.elem.delete(from.cursor.offset..to.cursor.offset); + self.tree.recursive_update_cache( + from.leaf().into(), + true, + Some(start as isize - end as isize), + ); + } + } + _ => { + crate::iter::Drain::new(&mut self.tree, from, to); + } + } + } + + fn iter(&self) -> impl Iterator { + let mut node_iter = self + .tree + .first_path() + .map(|first| crate::iter::Iter::new(&self.tree, first, self.tree.last_path().unwrap())); + std::iter::from_fn(move || match &mut node_iter { + Some(node_iter) => { + if let Some(node) = node_iter.next() { + Some(&node.1.elem) + } else { + None + } + } + None => None, + }) + } + + pub fn slice(&mut self, _range: impl RangeBounds) { + unimplemented!() + } + + pub fn new() -> Self { + Self { + tree: BTree::new(), + cursor: None, + } + } + + #[allow(unused)] + fn node_len(&self) -> usize { + self.tree.node_len() + } + + #[allow(unused)] + fn update_in_place(&mut self, pos: usize, new: &str) { + todo!() + } + + pub fn clear(&mut self) { + self.tree.clear(); + } + + #[allow(unused)] + pub fn check(&self) { + // dbg!(&self.tree); + self.tree.check() + } + + pub fn diagnose(&self) { + self.tree.diagnose_balance(); + } +} + +impl Default for Rope { + fn default() -> Self { + Self::new() + } +} + +impl Display for Rope { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ans = Vec::with_capacity(self.len()); + for elem in self.iter() { + let (left, right) = elem.as_bytes(); + ans.extend_from_slice(left); + ans.extend_from_slice(right); + } + + f.write_str(std::str::from_utf8(ans.as_slice()).unwrap()) + } +} + +impl BTreeTrait for RopeTrait { + type Elem = GapBuffer; + type Cache = isize; + type CacheDiff = isize; + + #[inline(always)] + fn calc_cache_internal(cache: &mut Self::Cache, caches: &[crate::Child]) -> isize { + let new_cache = caches.iter().map(|x| x.cache).sum::(); + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2; + } + + #[inline(always)] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + elem.len() as isize + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + cache_lhs - cache_rhs + } +} + +#[allow(unused)] +fn test_prev_length(rope: &Rope, q: QueryResult) -> usize { + let mut count = 0; + rope.tree + .visit_previous_caches(q.cursor(), |cache| match cache { + crate::PreviousCache::NodeCache(cache) => { + count += *cache as usize; + } + crate::PreviousCache::PrevSiblingElem(p) => { + count += p.len(); + } + crate::PreviousCache::ThisElemAndOffset { offset, .. } => { + count += offset; + } + }); + count +} + +#[allow(unused)] +fn test_index(rope: &Rope) { + for index in 0..rope.len() { + let q = rope.tree.query::(&index).unwrap(); + let i = test_prev_length(rope, q); + assert_eq!(i, index); + } +} + +#[cfg(test)] +mod test { + + use Action::*; + + use crate::HeapVec; + + use super::*; + + #[test] + fn test() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + assert_eq!(rope.len(), 3); + rope.insert(1, "x"); + test_index(&rope); + assert_eq!(rope.len(), 4); + rope.delete_range(2..4); + assert_eq!(&rope.to_string(), "1x"); + rope.delete_range(..1); + assert_eq!(&rope.to_string(), "x"); + rope.delete_range(..); + assert_eq!(&rope.to_string(), ""); + assert_eq!(rope.len(), 0); + } + + #[test] + fn test_delete_middle() { + let mut rope = Rope::new(); + rope.insert(0, "135"); + rope.delete_range(1..2); + assert_eq!(&rope.to_string(), "15"); + } + + #[test] + fn test_insert_repeatedly() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + rope.insert(1, "x"); + rope.insert(2, "y"); + rope.insert(3, "z"); + test_index(&rope); + assert_eq!(&rope.to_string(), "1xyz23"); + } + + #[test] + #[ignore] + fn test_update() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + rope.insert(3, "xyz"); + rope.update_in_place(1, "kkkk"); + assert_eq!(&rope.to_string(), "1kkkkz"); + } + + #[test] + fn test_clear() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + assert_eq!(rope.len(), 3); + rope.clear(); + assert_eq!(rope.len(), 0); + assert_eq!(&rope.to_string(), ""); + rope.insert(0, "kkk"); + assert_eq!(&rope.to_string(), "kkk"); + } + + #[test] + fn test_insert_many() { + let mut rope = Rope::new(); + let s = "_12345678_".repeat(10); + let mut expected = String::new(); + for i in 0..100 { + expected.insert_str(i, &s); + rope.insert(i, &s); + assert_eq!(&rope.to_string(), &expected) + } + } + + #[test] + fn test_repeat_insert() { + let mut rope = Rope::new(); + rope.insert(0, "123"); + for _ in 0..10000 { + rope.insert(rope.len() / 2, "k"); + } + } + + #[test] + #[ignore] + fn test_update_1() { + let mut rope = Rope::new(); + for i in 0..100 { + rope.insert(i, &(i % 10).to_string()); + } + + rope.update_in_place(15, "kkkkk"); + assert_eq!(&rope.to_string()[10..20], "01234kkkkk"); + test_index(&rope); + } + + #[derive(Debug)] + enum Action { + Insert { pos: u8, content: u8 }, + Delete { pos: u8, len: u8 }, + } + + fn fuzz(data: HeapVec) { + let mut rope = Rope::new(); + let mut truth = String::new(); + for action in data { + match action { + Action::Insert { pos, content } => { + let pos = pos as usize % (truth.len() + 1); + let s = content.to_string(); + dbg!("INS", pos, &s); + dbg!(&rope); + truth.insert_str(pos, &s); + rope.insert(pos, &s); + dbg!(&rope); + rope.check(); + assert_eq!(rope.len(), truth.len()); + assert_eq!(rope.to_string(), truth, "{:#?}", &rope.tree); + } + Action::Delete { pos, len } => { + let pos = pos as usize % (truth.len() + 1); + let mut len = len as usize % 10; + len = len.min(truth.len() - pos); + dbg!("DEL", pos, len); + dbg!(&rope); + rope.delete_range(pos..(pos + len)); + dbg!(&rope); + truth.drain(pos..pos + len); + rope.check(); + assert_eq!(rope.len(), truth.len()); + assert_eq!(rope.to_string(), truth, "{:#?}", &rope.tree); + } + } + } + + assert_eq!(rope.to_string(), truth); + } + + #[test] + fn fuzz_0() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 192, len: 193 }, + Insert { + pos: 106, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 100, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 111, + content: 127, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 36 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 135, len: 169 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + ]) + } + + #[test] + fn fuzz_1() { + fuzz(vec![ + Insert { + pos: 157, + content: 108, + }, + Insert { + pos: 255, + content: 255, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 8, + content: 101, + }, + Insert { + pos: 111, + content: 127, + }, + Delete { pos: 255, len: 169 }, + ]) + } + + #[test] + fn fuzz_2() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 0, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 249, + }, + Insert { + pos: 135, + content: 255, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 169, len: 169 }, + ]) + } + + #[test] + fn fuzz_3() { + fuzz(vec![ + Insert { + pos: 111, + content: 140, + }, + Insert { + pos: 111, + content: 107, + }, + Insert { + pos: 35, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 0, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 93, + content: 93, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 102, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 111, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 101, + }, + Insert { + pos: 36, + content: 146, + }, + Delete { pos: 74, len: 102 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 17, + content: 17, + }, + Insert { + pos: 17, + content: 17, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 102, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 111, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 3, + content: 73, + }, + Insert { + pos: 146, + content: 74, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 21, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 111, + content: 111, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 3, + }, + Insert { + pos: 36, + content: 146, + }, + Insert { + pos: 119, + content: 119, + }, + Delete { pos: 111, len: 119 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 73, + content: 36, + }, + Delete { pos: 74, len: 102 }, + Delete { pos: 255, len: 255 }, + Insert { + pos: 42, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 0, + content: 15, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 3, + }, + Insert { + pos: 36, + content: 146, + }, + Insert { + pos: 255, + content: 255, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 38, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 89, + content: 89, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 42, + content: 42, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 37, + }, + Insert { + pos: 101, + content: 102, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 193, len: 63 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 0, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 199 }, + Delete { pos: 199, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Delete { pos: 187, len: 187 }, + Insert { + pos: 3, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Delete { pos: 163, len: 163 }, + Delete { pos: 163, len: 163 }, + Delete { pos: 163, len: 102 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 108, + content: 249, + }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 111, len: 255 }, + Insert { + pos: 111, + content: 111, + }, + Insert { + pos: 255, + content: 255, + }, + ]) + } + + #[test] + fn fuzz_4() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 0, + }, + Insert { + pos: 108, + content: 108, + }, + ]) + } + + #[test] + fn fuzz_5() { + fuzz(vec![ + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 123, + }, + Delete { pos: 108, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 12, + content: 0, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 255, len: 246 }, + Delete { pos: 246, len: 246 }, + Delete { pos: 246, len: 246 }, + Delete { pos: 246, len: 246 }, + Insert { + pos: 101, + content: 101, + }, + Insert { + pos: 101, + content: 101, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 169, len: 169 }, + ]) + } + + #[test] + fn fuzz_6() { + fuzz(vec![ + Insert { + pos: 0, + content: 128, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 171, + content: 171, + }, + Delete { pos: 171, len: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 171, + }, + Delete { pos: 187, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 171, + content: 171, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 110, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 171, + }, + Delete { pos: 187, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 8, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 50, + content: 108, + }, + Delete { pos: 108, len: 108 }, + Insert { + pos: 108, + content: 87, + }, + Insert { + pos: 249, + content: 1, + }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 163, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 8, content: 0 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 41, len: 164 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 86, + }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 123, + }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 86, + content: 254, + }, + Insert { + pos: 33, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 2, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 0, + content: 121, + }, + Insert { + pos: 26, + content: 0, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 254 }, + Insert { + pos: 144, + content: 238, + }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 0, len: 51 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 123 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 86, + }, + Delete { pos: 101, len: 144 }, + Delete { pos: 238, len: 91 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 3, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 171, + content: 63, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 0, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 0, + content: 171, + }, + Delete { pos: 1, len: 126 }, + Delete { pos: 235, len: 154 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 249, + content: 1, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 32, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 255, len: 6 }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 171, + }, + Insert { + pos: 126, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 87, + content: 0, + }, + Delete { pos: 1, len: 111 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 254, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 0, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 254, len: 193 }, + Delete { pos: 63, len: 64 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 111, + content: 127, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 8, len: 0 }, + Delete { pos: 249, len: 1 }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 8, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 50, + content: 108, + }, + Delete { pos: 108, len: 108 }, + Insert { + pos: 108, + content: 8, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 8, content: 0 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 41, len: 164 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 171, + content: 171, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 41, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 41 }, + Insert { + pos: 171, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 165, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 170 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 171, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 8, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 8, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 18 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 254, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 0, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 123 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 86, + content: 86, + }, + Insert { + pos: 202, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 2, + }, + Insert { + pos: 254, + content: 123, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 255, len: 101 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 123, + }, + Delete { pos: 123, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 123, + }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 1, len: 0 }, + Insert { pos: 0, content: 7 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 235, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 111, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 171, + content: 8, + }, + Delete { pos: 171, len: 249 }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 87, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 11, len: 238 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 41, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 171, len: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 108, + }, + Delete { pos: 63, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 157, len: 157 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 108, len: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 248, + }, + Delete { pos: 154, len: 127 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 0 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 8, len: 0 }, + Delete { pos: 249, len: 1 }, + Delete { pos: 169, len: 235 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 49, + }, + Delete { pos: 235, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 0, + content: 249, + }, + Insert { + pos: 135, + content: 169, + }, + Delete { pos: 238, len: 123 }, + Insert { pos: 2, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 1, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 193, + content: 192, + }, + Delete { pos: 63, len: 127 }, + Insert { + pos: 0, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 87, + content: 0, + }, + Delete { pos: 1, len: 111 }, + Delete { pos: 235, len: 154 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 0, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 235, len: 235 }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 127, + content: 135, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 172 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 235, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 111, + content: 111, + }, + Delete { pos: 171, len: 0 }, + Insert { + pos: 48, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Insert { + pos: 84, + content: 84, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 235 }, + Delete { pos: 254, len: 86 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 8 }, + Insert { + pos: 108, + content: 171, + }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 20 }, + Delete { pos: 171, len: 108 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 235, + content: 235, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 111, + content: 111, + }, + Delete { pos: 154, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 171, len: 171 }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 86, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 36, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 254, + content: 255, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 123 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 123, len: 123 }, + Insert { + pos: 238, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 238, + content: 238, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 91, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 238, + content: 238, + }, + Insert { + pos: 108, + content: 108, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 238, len: 238 }, + Insert { + pos: 123, + content: 2, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 238, + content: 238, + }, + Insert { + pos: 0, + content: 238, + }, + Delete { pos: 238, len: 238 }, + Delete { pos: 0, len: 249 }, + Insert { + pos: 135, + content: 255, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 144, len: 255 }, + Delete { pos: 169, len: 169 }, + ]) + } + + #[test] + fn ben() { + use arbitrary::Arbitrary; + #[derive(Arbitrary, Debug, Clone, Copy)] + enum Action { + Insert { pos: u8, content: u8 }, + Delete { pos: u8, len: u8 }, + } + + use rand::{Rng, SeedableRng}; + let mut rng = rand::rngs::StdRng::seed_from_u64(123); + let mut expected = String::new(); + let unstructured: Vec = (0..10_000).map(|_| rng.gen()).collect(); + let mut gen = arbitrary::Unstructured::new(&unstructured); + let actions: [Action; 1_000] = gen.arbitrary().unwrap(); + let mut rope = Rope::new(); + for action in actions.iter() { + match *action { + Action::Insert { pos, content } => { + let pos = pos as usize % (rope.len() + 1); + let s = content.to_string(); + expected.insert_str(pos, &s); + rope.insert(pos, &s); + assert_eq!(expected.len(), rope.len()); + } + Action::Delete { pos, len } => { + let pos = pos as usize % (rope.len() + 1); + let mut len = len as usize % 10; + len = len.min(rope.len() - pos); + expected.drain(pos..pos + len); + rope.delete_range(pos..(pos + len)); + assert_eq!(expected.len(), rope.len()); + } + } + } + assert_eq!(rope.to_string(), expected); + } + + #[test] + fn fuzz_7() { + fuzz(vec![ + Insert { + pos: 111, + content: 111, + }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 108, + }, + Insert { + pos: 108, + content: 255, + }, + Delete { pos: 255, len: 0 }, + Insert { + pos: 140, + content: 140, + }, + Insert { + pos: 102, + content: 101, + }, + Insert { + pos: 36, + content: 146, + }, + Insert { + pos: 102, + content: 119, + }, + Insert { + pos: 118, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 21, + content: 0, + }, + Insert { + pos: 140, + content: 140, + }, + Insert { + pos: 107, + content: 19, + }, + Insert { + pos: 102, + content: 47, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 102, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 102, + }, + Insert { + pos: 102, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 123, + content: 123, + }, + Delete { pos: 255, len: 136 }, + Delete { pos: 119, len: 111 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 3, + content: 73, + }, + Insert { + pos: 146, + content: 74, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 0, len: 102 }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { pos: 0, content: 0 }, + Delete { pos: 255, len: 255 }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 0, + content: 255, + }, + Delete { pos: 111, len: 108 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Insert { + pos: 83, + content: 108, + }, + Insert { + pos: 111, + content: 111, + }, + Insert { + pos: 119, + content: 21, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 64, + content: 64, + }, + Insert { + pos: 55, + content: 119, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { pos: 0, content: 0 }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Delete { pos: 130, len: 130 }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 96, + content: 102, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 156, + content: 111, + }, + Insert { + pos: 123, + content: 37, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 37, + content: 121, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 121, + content: 86, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 123, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Delete { pos: 239, len: 239 }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 123, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 123, + content: 0, + }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 255, len: 255 }, + Delete { pos: 125, len: 125 }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 125, + content: 125, + }, + Insert { + pos: 123, + content: 123, + }, + Insert { + pos: 118, + content: 118, + }, + Insert { + pos: 255, + content: 255, + }, + Insert { + pos: 119, + content: 119, + }, + Insert { + pos: 102, + content: 102, + }, + Delete { pos: 209, len: 255 }, + Delete { pos: 255, len: 255 }, + ]) + } + + #[test] + fn from_str() { + for i in 0..100000 { + let s = i.to_string(); + let mut g = GapBuffer::from_str(&s); + assert_eq!(s.len(), g.next().unwrap().len()); + } + } + + #[test] + fn from_iter() { + let mut v = vec![]; + for i in 0..100000 { + v.push(i.to_string()); + } + + let rope = Rope { + tree: v + .iter() + .flat_map(|x| GapBuffer::from_str(x.as_str())) + .collect(), + cursor: None, + }; + + let s = v.join(""); + assert_eq!(rope.to_string(), s); + assert_eq!(rope.len(), s.len()); + rope.tree.check(); + } + + #[test] + fn drain() { + let mut rope = Rope::new(); + for i in 0..100000 { + rope.insert(0, &i.to_string()); + } + + while !rope.is_empty() { + let leaf = rope.tree.first_leaf(); + rope.tree.update_leaf(leaf.unwrap(), |elem| { + elem.slice_(1..1); + (true, None, None) + }); + } + } + + #[test] + fn fuzz_empty() { + fuzz(vec![]) + } +} diff --git a/vendor/generic-btree/src/iter.rs b/vendor/generic-btree/src/iter.rs new file mode 100644 index 00000000..e8314a93 --- /dev/null +++ b/vendor/generic-btree/src/iter.rs @@ -0,0 +1,322 @@ +use crate::{ + delete_range, rle::HasLength, ArenaIndex, BTree, BTreeTrait, Cursor, LeafNode, NodePath, + QueryResult, +}; + +/// iterate node (not element) from the start path to the **inclusive** end path +pub(super) struct Iter<'a, B: BTreeTrait> { + tree: &'a BTree, + inclusive_end: NodePath, + path: NodePath, + done: bool, +} + +struct TempStore { + start_path: NodePath, + end_path: NodePath, + leaf_before_drain_range: Option, + leaf_after_drain_range: Option, +} + +pub struct Drain<'a, B: BTreeTrait> { + tree: &'a mut BTree, + current_path: NodePath, + done: bool, + end_cursor: Option, + store: Option>, +} + +impl<'a, B: BTreeTrait> Drain<'a, B> { + pub fn new( + tree: &'a mut BTree, + start_result: Option, + end_result: Option, + ) -> Self { + if start_result.is_none() || end_result.is_none() { + return Self::none(tree); + } + + let start_result = start_result.unwrap(); + let end_result = end_result.unwrap(); + let end_result = tree.split_leaf_if_needed(end_result.cursor).new_pos; + let Some(start_result) = tree.split_leaf_if_needed(start_result.cursor).new_pos else { + // if start from the right most leaf, the range is empty + return Self::none(tree); + }; + let start_path = tree.get_path(start_result.leaf.into()); + let end_path = tree.get_path( + end_result + .map(|x| x.leaf.into()) + .unwrap_or_else(|| tree.last_leaf().unwrap().into()), + ); + let leaf_before_drain_range = { + let node_idx = start_path.last().unwrap().arena; + if start_result.offset == 0 { + tree.prev_same_level_in_node(node_idx) + } else { + Some(node_idx) + } + }; + let leaf_after_drain_range = { + let node_idx = end_path.last().unwrap().arena; + if let Some(end) = end_result { + let len = tree.leaf_nodes.get(end.leaf.0).unwrap().elem.rle_len(); + if len == end.offset { + tree.next_same_level_in_node(node_idx) + } else { + Some(node_idx) + } + } else { + None + } + }; + Self { + current_path: tree.get_path(start_result.leaf.into()), + tree, + done: false, + end_cursor: end_result, + store: Some(Box::new(TempStore { + start_path, + end_path, + leaf_before_drain_range, + leaf_after_drain_range, + })), + } + } + + fn none(tree: &'a mut BTree) -> Drain { + Self { + current_path: Default::default(), + done: true, + end_cursor: None, + tree, + store: None, + } + } +} + +impl<'a, B: BTreeTrait> Iterator for Drain<'a, B> { + type Item = B::Elem; + + fn next(&mut self) -> Option { + if self.done { + return None; + } + + // end iteration if pointing to the end leaf + if let Some(end_cursor) = self.end_cursor { + if end_cursor.leaf.0 == self.current_path.last().unwrap().arena.unwrap_leaf() { + return None; + } + } + + let idx = *self.current_path.last().unwrap(); + if !self.tree.next_sibling(&mut self.current_path) { + self.done = true; + } + + // NOTE: we removed the node here, the tree is in an invalid state + let node = self + .tree + .leaf_nodes + .remove(idx.arena.unwrap_leaf()) + .unwrap(); + Some(node.elem) + } +} + +impl<'a, B: BTreeTrait> Drain<'a, B> { + fn ensure_finished(&mut self) { + while self.next().is_some() {} + } +} + +impl<'a, B: BTreeTrait> Drop for Drain<'a, B> { + fn drop(&mut self) { + self.ensure_finished(); + let TempStore { + start_path, + end_path, + leaf_before_drain_range, + leaf_after_drain_range, + } = *self.store.take().unwrap(); + // the deepest internal node level + let mut level = start_path.len() - 2; + let mut deleted = Vec::new(); + + // The deepest internal node level, need to filter deleted children + // to ensure is_empty() has correct result + self.tree.filter_deleted_children(start_path[level].arena); + self.tree.filter_deleted_children(end_path[level].arena); + while start_path[level].arena != end_path[level].arena { + let start_node = self.tree.get_internal(start_path[level].arena); + let end_node = self.tree.get_internal(end_path[level].arena); + let del_start = if start_node.is_empty() { + start_path[level].arr + } else { + start_path[level].arr + 1 + }; + let del_end = if end_node.is_empty() { + end_path[level].arr + 1 + } else { + end_path[level].arr + }; + + // remove del_start.. in start_node's parent + // remove ..del_end in end_node's parent + let start_arena = start_path[level - 1].arena; + let end_arena = end_path[level - 1].arena; + if start_arena == end_arena { + // parent is the same, delete start..end + let parent = self.tree.get_internal_mut(start_arena); + for x in &parent.children[del_start as usize..del_end as usize] { + deleted.push(x.arena); + } + + delete_range(&mut parent.children, del_start as usize..del_end as usize); + self.tree + .update_children_parent_slot_from(start_arena, del_start as usize); + } else { + // parent is different + { + // delete start.. + let start_parent = self.tree.get_internal_mut(start_arena); + for x in &start_parent.children[del_start as usize..] { + deleted.push(x.arena); + } + delete_range(&mut start_parent.children, del_start as usize..); + } + { + // delete ..end + let end_parent = self.tree.get_internal_mut(end_arena); + for x in &end_parent.children[..del_end as usize] { + deleted.push(x.arena); + } + delete_range(&mut end_parent.children, ..del_end as usize); + self.tree.update_children_parent_slot_from(end_arena, 0); + } + } + + level -= 1 + // this loop will abort before overflow, because level=0 is guaranteed to be the same + } + + while level >= 1 { + let (child, parent) = self + .tree + .get2_mut(start_path[level].arena, start_path[level - 1].arena); + if child.is_empty() { + assert_eq!( + parent.children[start_path[level].arr as usize].arena, + start_path[level].arena + ); + deleted.push(parent.children.remove(start_path[level].arr as usize).arena); + self.tree.update_children_parent_slot_from( + start_path[level - 1].arena, + start_path[level].arr as usize, + ); + } else { + break; + } + level -= 1; + } + + // release memory + for x in deleted { + self.tree.purge(x); + } + + if let Some(after) = leaf_after_drain_range { + self.tree.recursive_update_cache( + after, + leaf_after_drain_range == leaf_before_drain_range, + None, + ); + } + + // otherwise the path is invalid (e.g. the tree is empty) + if let Some(before) = leaf_before_drain_range { + if leaf_before_drain_range == leaf_after_drain_range { + self.tree.recursive_update_cache(before, B::USE_DIFF, None); + } else { + self.tree.recursive_update_cache(before, false, None); + if let Some(after) = leaf_after_drain_range { + self.tree.recursive_update_cache(after, false, None); + } + } + seal(self.tree, before); + } else { + self.tree.update_root_cache(); + self.tree.try_reduce_levels(); + } + } +} + +fn seal(tree: &mut BTree, leaf: ArenaIndex) { + handle_lack_on_path_to_leaf(tree, leaf); + if let Some(sibling) = tree.next_same_level_in_node(leaf) { + handle_lack_on_path_to_leaf(tree, sibling); + } + tree.try_reduce_levels(); +} + +fn handle_lack_on_path_to_leaf(tree: &mut BTree, leaf: ArenaIndex) { + let mut last_lack_count = 0; + let mut lack_count; + loop { + lack_count = 0; + let path = tree.get_path(leaf); + for i in 1..path.len() - 1 { + let Some(node) = tree.in_nodes.get(path[i].arena.unwrap_internal()) else { + unreachable!() + }; + let is_lack = node.is_lack(); + if is_lack { + let lack_info = tree.handle_lack_single_layer(path[i].arena); + if lack_info.parent_lack.is_some() { + lack_count += 1; + } + } + } + // parent may be lack after some children is merged + if lack_count == 0 || lack_count == last_lack_count { + break; + } + + last_lack_count = lack_count; + } +} + +impl<'a, B: BTreeTrait> Iter<'a, B> { + pub fn new(tree: &'a BTree, start: NodePath, inclusive_end: NodePath) -> Self { + Self { + tree, + inclusive_end, + path: start, + done: false, + } + } +} + +impl<'a, B: BTreeTrait> Iterator for Iter<'a, B> { + type Item = (NodePath, &'a LeafNode); + + fn next(&mut self) -> Option { + if self.done { + return None; + } + + if self.inclusive_end.last() == self.path.last() { + self.done = true; + } + + let last = *self.path.last().unwrap(); + let path = self.path.clone(); + if !self.tree.next_sibling(&mut self.path) { + self.done = true; + } + + let node = self.tree.leaf_nodes.get(last.arena.unwrap_leaf()).unwrap(); + Some((path, node)) + } +} diff --git a/vendor/generic-btree/src/lib.rs b/vendor/generic-btree/src/lib.rs new file mode 100644 index 00000000..95a7526d --- /dev/null +++ b/vendor/generic-btree/src/lib.rs @@ -0,0 +1,3105 @@ +#![doc = include_str!("../README.md")] +#![forbid(unsafe_code)] + +use core::{fmt::Debug, ops::Range}; +use std::collections::{BTreeSet, VecDeque}; +use std::ops::AddAssign; +use std::{cmp::Ordering, mem::take, ops::RangeBounds}; + +pub(crate) use heapless::Vec as HeaplessVec; +use itertools::Itertools; +use rle::{CanRemove, TryInsert}; +use rustc_hash::{FxHashMap, FxHashSet}; +use thunderdome::Arena; +use thunderdome::Index as RawArenaIndex; + +pub use generic_impl::*; + +use crate::rle::{HasLength, Mergeable, Sliceable}; + +mod generic_impl; +pub mod iter; + +pub mod rle; + +pub type HeapVec = Vec; + +const MAX_CHILDREN_NUM: usize = 12; + +/// `Elem` should has length. `offset` in search result should always >= `Elem.rle_len()` +pub trait BTreeTrait { + /// Sometime an [Elem] with length of 0, but it's not empty. + /// + /// The empty [Elem]s are the ones that can be safely ignored. + type Elem: Debug + HasLength + Sliceable + Mergeable + TryInsert + CanRemove; + type Cache: Debug + Default + Clone + Eq; + type CacheDiff: Debug + Default + CanRemove; + // Whether we should use cache diff by default + const USE_DIFF: bool = true; + + /// If diff.is_some, return value should be some too + fn calc_cache_internal(cache: &mut Self::Cache, caches: &[Child]) -> Self::CacheDiff; + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff); + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff); + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache; + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff; + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff; +} + +pub trait Query { + type QueryArg: Clone; + + fn init(target: &Self::QueryArg) -> Self; + + fn find_node(&mut self, target: &Self::QueryArg, child_caches: &[Child]) -> FindResult; + + /// Confirm the search result and returns (offset, found) + /// + /// If elem is not target, `found=false` + fn confirm_elem(&mut self, q: &Self::QueryArg, elem: &B::Elem) -> (usize, bool); +} + +pub struct BTree { + /// internal nodes + in_nodes: Arena>, + /// leaf nodes + leaf_nodes: Arena>, + /// root is always a internal node + /// TODO: we may use a constant as root index + root: ArenaIndex, + root_cache: B::Cache, +} + +impl> Clone for BTree { + fn clone(&self) -> Self { + Self { + in_nodes: self.in_nodes.clone(), + leaf_nodes: self.leaf_nodes.clone(), + root: self.root, + root_cache: self.root_cache.clone(), + } + } +} + +pub struct FindResult { + pub index: usize, + pub offset: usize, + pub found: bool, +} + +impl FindResult { + pub fn new_found(index: usize, offset: usize) -> Self { + Self { + index, + offset, + found: true, + } + } + + pub fn new_missing(index: usize, offset: usize) -> Self { + Self { + index, + offset, + found: false, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Idx { + pub arena: ArenaIndex, + pub arr: u8, +} + +impl Idx { + pub fn new(arena: ArenaIndex, arr: u8) -> Self { + Self { arena, arr } + } +} + +type NodePath = HeaplessVec; + +#[derive(Debug, Clone, PartialEq, Eq, Copy, Hash)] +pub struct Cursor { + pub leaf: LeafIndex, + pub offset: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub struct QueryResult { + pub cursor: Cursor, + pub found: bool, +} + +/// Exposed arena index +/// +/// Only exposed arena index of leaf node. +/// +/// +#[repr(transparent)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, PartialOrd, Ord)] +pub struct LeafIndex(RawArenaIndex); + +impl LeafIndex { + pub fn inner(&self) -> RawArenaIndex { + self.0 + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub enum ArenaIndex { + Leaf(RawArenaIndex), + Internal(RawArenaIndex), +} + +impl ArenaIndex { + fn unwrap(self) -> RawArenaIndex { + match self { + ArenaIndex::Leaf(x) => x, + ArenaIndex::Internal(x) => x, + } + } + + pub fn unwrap_leaf(self) -> RawArenaIndex { + match self { + ArenaIndex::Leaf(x) => x, + ArenaIndex::Internal(_) => panic!("unwrap_leaf on internal node"), + } + } + + pub fn unwrap_internal(self) -> RawArenaIndex { + match self { + ArenaIndex::Leaf(_) => panic!("unwrap_internal on leaf node"), + ArenaIndex::Internal(x) => x, + } + } +} + +impl From for ArenaIndex { + fn from(value: LeafIndex) -> Self { + Self::Leaf(value.0) + } +} + +impl From for LeafIndex { + fn from(value: RawArenaIndex) -> Self { + Self(value) + } +} + +/// A slice of element +/// +/// - `start` is Some(start_offset) when it's first element of the given range. +/// - `end` is Some(end_offset) when it's last element of the given range. +#[derive(Debug)] +pub struct ElemSlice<'a, Elem> { + cursor: Cursor, + pub elem: &'a Elem, + pub start: Option, + pub end: Option, +} + +impl<'a, Elem> ElemSlice<'a, Elem> { + pub fn cursor(&self) -> &Cursor { + &self.cursor + } +} + +impl QueryResult { + pub fn elem<'b, Elem: Debug, B: BTreeTrait>( + &self, + tree: &'b BTree, + ) -> Option<&'b Elem> { + tree.leaf_nodes.get(self.cursor().leaf.0).map(|x| &x.elem) + } + + #[inline(always)] + pub fn cursor(&self) -> Cursor { + self.cursor + } + + #[inline(always)] + pub fn leaf(&self) -> LeafIndex { + self.cursor().leaf + } + + #[inline(always)] + pub fn offset(&self) -> usize { + self.cursor().offset + } + + #[inline(always)] + pub fn found(&self) -> bool { + self.found + } + + #[inline(always)] + pub fn arena(&self) -> RawArenaIndex { + self.cursor.leaf.0 + } +} + +#[derive(Debug, Clone)] +pub struct LeafNode { + elem: Elem, + parent: RawArenaIndex, +} + +impl LeafNode { + pub fn parent(&self) -> ArenaIndex { + ArenaIndex::Internal(self.parent) + } + + pub fn elem(&self) -> &T { + &self.elem + } +} + +impl LeafNode { + fn split(&mut self, offset: usize) -> Self { + let new_elem = self.elem.split(offset); + Self { + elem: new_elem, + parent: self.parent, + } + } +} + +pub struct Node { + parent: Option, + parent_slot: u8, + children: HeaplessVec, MAX_CHILDREN_NUM>, +} + +#[repr(transparent)] +#[derive(Debug, Default, Clone)] +pub struct SplittedLeaves { + pub arr: HeaplessVec, +} + +impl SplittedLeaves { + #[inline] + fn push_option(&mut self, leaf: Option) { + if let Some(leaf) = leaf { + self.arr.push(leaf.unwrap().into()).unwrap(); + } + } + + #[inline] + fn push(&mut self, leaf: ArenaIndex) { + self.arr.push(leaf.unwrap().into()).unwrap(); + } +} + +impl> Debug for BTree { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + fn fmt_node>( + tree: &BTree, + node_idx: ArenaIndex, + f: &mut core::fmt::Formatter<'_>, + indent_size: usize, + ) -> core::fmt::Result { + match node_idx { + ArenaIndex::Leaf(_) => {} + ArenaIndex::Internal(_) => { + let node = tree.get_internal(node_idx); + for child in node.children.iter() { + indent(f, indent_size)?; + if child.is_internal() { + let child_node = tree.get_internal(child.arena); + f.write_fmt(format_args!( + "{} Arena({:?}) Cache: {:?}\n", + child_node.parent_slot, &child.arena, &child.cache + ))?; + fmt_node::(tree, child.arena, f, indent_size + 1)?; + } else { + let node = tree.get_leaf(child.arena); + f.write_fmt(format_args!( + "Leaf({:?}) Arena({:?}) Parent({:?}) Cache: {:?}\n", + &node.elem, child.arena, node.parent, &child.cache + ))?; + } + } + } + } + + Ok(()) + } + + fn indent(f: &mut core::fmt::Formatter<'_>, indent: usize) -> core::fmt::Result { + for _ in 0..indent { + f.write_str(" ")?; + } + Ok(()) + } + + f.write_str("BTree\n")?; + indent(f, 1)?; + f.write_fmt(format_args!( + "Root Arena({:?}) Cache: {:?}\n", + &self.root, &self.root_cache + ))?; + fmt_node::(self, self.root, f, 1) + } +} + +impl> Debug for Node { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Node") + .field("children", &self.children) + .finish() + } +} + +impl> Debug for Child { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Child") + .field("index", &self.arena) + .field("cache", &self.cache) + .finish() + } +} + +impl> Clone for Node { + fn clone(&self) -> Self { + Self { + parent: self.parent, + parent_slot: self.parent_slot, + children: self.children.clone(), + } + } +} + +pub struct Child { + pub arena: ArenaIndex, + pub cache: B::Cache, +} + +impl Child { + #[inline] + fn is_internal(&self) -> bool { + matches!(self.arena, ArenaIndex::Internal(_)) + } + + #[inline] + #[allow(unused)] + fn is_leaf(&self) -> bool { + matches!(self.arena, ArenaIndex::Leaf(_)) + } +} + +impl Clone for Child { + fn clone(&self) -> Self { + Self { + arena: self.arena, + cache: self.cache.clone(), + } + } +} + +impl Child { + pub fn cache(&self) -> &B::Cache { + &self.cache + } + + fn new(arena: ArenaIndex, cache: B::Cache) -> Self { + Self { arena, cache } + } +} + +impl Node { + #[inline(always)] + pub fn new() -> Self { + Self { + parent: None, + parent_slot: u8::MAX, + children: HeaplessVec::new(), + } + } + + #[inline(always)] + pub fn is_full(&self) -> bool { + self.children.len() >= MAX_CHILDREN_NUM + } + + #[inline(always)] + pub fn is_lack(&self) -> bool { + self.children.len() < MAX_CHILDREN_NUM / 2 + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.children.len() + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline(always)] + pub fn is_child_leaf(&self) -> bool { + if self.children.is_empty() { + return true; + } + + self.children[0].is_leaf() + } + + /// if diff is not provided, the cache will be calculated from scratch + #[inline(always)] + fn calc_cache(&self, cache: &mut B::Cache, diff: Option) -> B::CacheDiff { + match diff { + Some(inner) => { + B::apply_cache_diff(cache, &inner); + inner + } + None => B::calc_cache_internal(cache, &self.children), + } + } +} + +impl Default for Node { + fn default() -> Self { + Self::new() + } +} + +type LeafDirtyMap = FxHashMap; + +/// Whether the parent node is lack of children +#[repr(transparent)] +struct LackInfo { + /// if Some, the parent node is lack + parent_lack: Option, +} + +impl BTree { + pub fn new() -> Self { + let mut arena = Arena::new(); + let root = arena.insert(Node::new()); + Self { + in_nodes: arena, + leaf_nodes: Arena::new(), + root: ArenaIndex::Internal(root), + root_cache: B::Cache::default(), + } + } + + /// Get the number of nodes in the tree. + /// It includes all the internal nodes and the leaf nodes. + #[inline(always)] + pub fn node_len(&self) -> usize { + self.in_nodes.len() + self.leaf_nodes.len() + } + + /// Insert new element to the tree. + /// + /// Returns (insert_pos, splitted_leaves) + pub fn insert(&mut self, q: &Q::QueryArg, data: B::Elem) -> (Cursor, SplittedLeaves) + where + Q: Query, + { + let Some(result) = self.query::(q) else { + return (self.push(data), Default::default()); + }; + + self.insert_by_path(result.cursor, data) + } + + pub fn insert_by_path(&mut self, cursor: Cursor, data: B::Elem) -> (Cursor, SplittedLeaves) { + let index = cursor.leaf; + let leaf = self.leaf_nodes.get_mut(index.0).unwrap(); + let mut parent_idx = leaf.parent(); + let cache_diff = if B::USE_DIFF { + Some(B::new_cache_to_diff(&B::get_elem_cache(&data))) + } else { + None + }; + + let mut is_full = false; + let mut splitted: SplittedLeaves = Default::default(); + let ans = match leaf.elem.try_insert(cursor.offset, data) { + Ok(_) => cursor, + Err(data) => { + // Try to merge + if cursor.offset == 0 && data.can_merge(&leaf.elem) { + leaf.elem.merge_left(&data); + Cursor { + leaf: index, + offset: 0, + } + } else if cursor.offset == leaf.elem.rle_len() && leaf.elem.can_merge(&data) { + let offset = leaf.elem.rle_len(); + leaf.elem.merge_right(&data); + Cursor { + leaf: index, + offset, + } + } else { + // Insert new leaf node + let SplitInfo { + parent_idx: parent_index, + insert_slot: insert_index, + new_leaf, + .. + } = self.split_leaf_if_needed(cursor); + parent_idx = ArenaIndex::Internal(parent_index); + let child = self.alloc_leaf_child(data, parent_index); + let ans = child.arena; + splitted.push_option(new_leaf); + let parent = self.in_nodes.get_mut(parent_index).unwrap(); + parent.children.insert(insert_index, child).unwrap(); + is_full = parent.is_full(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + } + } + }; + + self.recursive_update_cache(cursor.leaf.into(), B::USE_DIFF, cache_diff); + if is_full { + self.split(parent_idx); + } + + (ans, splitted) + } + + fn alloc_leaf_child( + &mut self, + data: ::Elem, + parent_index: RawArenaIndex, + ) -> Child { + let elem_cache = B::get_elem_cache(&data); + let new_leaf_index = self.alloc_new_leaf(LeafNode { + elem: data, + parent: parent_index, + }); + Child { + arena: new_leaf_index, + cache: elem_cache, + } + } + + /// Split a leaf node at offset if it's not the start/end of the leaf node. + /// + /// This method should be called when inserting at target pos. + fn split_leaf_if_needed(&mut self, pos: Cursor) -> SplitInfo { + let leaf = self.leaf_nodes.get_mut(pos.leaf.0).unwrap(); + let parent_idx = leaf.parent; + let parent = self.in_nodes.get_mut(leaf.parent).unwrap(); + let mut new_pos = Some(pos); + let mut rt_new_leaf = None; + let leaf_slot = parent + .children + .iter() + .position(|x| x.arena.unwrap() == pos.leaf.0) + .unwrap(); + let left_neighbour = if leaf_slot == 0 { + None + } else { + Some(parent.children[leaf_slot - 1].arena.unwrap().into()) + }; + let insert_pos = if pos.offset == 0 { + leaf_slot + } else if pos.offset == leaf.elem.rle_len() { + if leaf_slot + 1 < parent.children.len() { + new_pos = Some(Cursor { + leaf: parent.children[leaf_slot + 1].arena.unwrap().into(), + offset: 0, + }); + } else { + new_pos = self.next_elem(pos); + } + leaf_slot + 1 + } else { + assert!( + pos.offset < leaf.elem.rle_len(), + "elem.rle_len={} but pos.offset={} Elem:{:?}", + leaf.elem.rle_len(), + pos.offset, + &leaf.elem + ); + + if parent.children.len() + 1 >= MAX_CHILDREN_NUM { + self.split(ArenaIndex::Internal(parent_idx)); + // parent may be changed because of splitting + return self.split_leaf_if_needed(pos); + } + + let new_leaf = leaf.split(pos.offset); + let left_cache = B::get_elem_cache(&leaf.elem); + let cache = B::get_elem_cache(&new_leaf.elem); + // alloc new leaf node + let leaf_arena_index = { + let arena_index = self.leaf_nodes.insert(new_leaf); + ArenaIndex::Leaf(arena_index) + }; + rt_new_leaf = Some(leaf_arena_index); + new_pos = Some(Cursor { + leaf: leaf_arena_index.unwrap().into(), + offset: 0, + }); + parent.children[leaf_slot].cache = left_cache; + parent + .children + .insert( + leaf_slot + 1, + Child { + arena: leaf_arena_index, + cache, + }, + ) + .unwrap(); + + leaf_slot + 1 + }; + + SplitInfo { + left_neighbour, + new_pos, + parent_idx, + insert_slot: insert_pos, + new_leaf: rt_new_leaf, + } + } + + fn alloc_new_leaf(&mut self, leaf: LeafNode) -> ArenaIndex { + let arena_index = self.leaf_nodes.insert(leaf); + ArenaIndex::Leaf(arena_index) + } + + /// Insert many elements into the tree at once + /// + /// It will invoke [`BTreeTrait::insert_batch`] + pub fn insert_many_by_cursor( + &mut self, + cursor: Option, + mut data_iter: impl Iterator, + ) { + let Some(first) = data_iter.next() else { + return; + }; + + let Some(second) = data_iter.next() else { + if let Some(c) = cursor { + self.insert_by_path(c, first); + return; + } else { + self.push(first); + return; + } + }; + + let mut data = Vec::with_capacity(data_iter.size_hint().0 + 2); + data.push(first); + data.push(second); + for elem in data_iter { + data.push(elem); + } + + merge_adj(&mut data); + if data.len() == 1 { + if let Some(c) = cursor { + self.insert_by_path(c, data.pop().unwrap()); + return; + } else { + self.push(data.pop().unwrap()); + return; + } + } + + if cursor.is_none() && self.is_empty() { + assert!(self.is_empty()); + let (new_root, _) = self.create_subtrees_from_elem(data); + self.in_nodes.remove(self.root.unwrap()).unwrap(); + self.root = new_root; + return; + } + + // dbg!(cursor, &data); + // dbg!(&self); + let cursor = cursor.expect("Cursor must be provided when tree is not empty"); + let SplitInfo { + new_pos, + left_neighbour, + .. + } = self.split_leaf_if_needed(cursor); + let mut inserted = 0; + if let Some(left) = left_neighbour { + let left_node = self.leaf_nodes.get_mut(left.0).unwrap(); + let mut i = 0; + while i < data.len() && left_node.elem.can_merge(&data[i]) { + left_node.elem.merge_right(&data[i]); + i += 1; + } + + self.recursive_update_cache(left.into(), B::USE_DIFF, None); + inserted = i; + } + + let mut pos = new_pos.unwrap_or(cursor); + // TODO: PERF this can be optimized further + for item in data.drain(inserted..).rev() { + let (p, _) = self.insert_by_path(pos, item); + pos = p + } + } + + /// The returned height starts from 0. Leaf level is 0. + /// + /// Returns (newly created subtree's root, height) + fn create_subtrees_from_elem(&mut self, data: Vec) -> (ArenaIndex, usize) { + let mut height = 0; + let mut nodes = Vec::with_capacity(data.len() / MAX_CHILDREN_NUM + 1); + for elem in data.into_iter().chunks(MAX_CHILDREN_NUM).into_iter() { + let parent_index = self.in_nodes.insert(Node { + parent: None, + parent_slot: 0, + children: Default::default(), + }); + + nodes.push(parent_index); + let parent = self.in_nodes.get_mut(parent_index).unwrap(); + for (i, elem) in elem.enumerate() { + let leaf = { + // alloc new leaf child + let elem_cache = B::get_elem_cache(&elem); + let new_leaf_index = { + let leaf = LeafNode { + elem, + parent: parent_index, + }; + let arena_index = self.leaf_nodes.insert(leaf); + ArenaIndex::Leaf(arena_index) + }; + Child { + arena: new_leaf_index, + cache: elem_cache, + } + }; + parent.children[i] = leaf; + } + } + + while nodes.len() > 1 { + let mut new_nodes = Vec::with_capacity(nodes.len() / MAX_CHILDREN_NUM + 1); + for chunk in nodes.into_iter().chunks(MAX_CHILDREN_NUM).into_iter() { + let parent_index = self.in_nodes.insert(Node { + parent: None, + parent_slot: 0, + children: Default::default(), + }); + + new_nodes.push(parent_index); + for (i, child_idx) in chunk.enumerate() { + let (parent, child) = self.in_nodes.get2_mut(parent_index, child_idx); + let parent = parent.unwrap(); + let child = child.unwrap(); + let mut cache = B::Cache::default(); + B::calc_cache_internal(&mut cache, &child.children); + parent.children[i] = Child { + arena: ArenaIndex::Internal(child_idx), + cache, + }; + child.parent = Some(ArenaIndex::Internal(parent_index)); + child.parent_slot = i as u8; + } + } + nodes = new_nodes; + height += 1; + } + + (ArenaIndex::Internal(nodes[0]), height) + } + + /// Shift by offset 1. + /// + /// It will not stay on empty spans but scan forward + pub fn shift_path_by_one_offset(&self, mut path: Cursor) -> Option + where + B::Elem: rle::HasLength, + { + let leaf = self.leaf_nodes.get(path.leaf.0).unwrap(); + if path.offset + 1 < leaf.elem.rle_len() { + path.offset += 1; + return Some(path); + } + + let mut parent_idx = leaf.parent; + let mut parent = self.in_nodes.get(leaf.parent).unwrap(); + let mut elem_slot_index = Self::get_leaf_slot(path.leaf.0, parent); + path.offset += 1; + loop { + if elem_slot_index == parent.children.len() { + if let Some(next) = self.next_same_level_in_node(ArenaIndex::Internal(parent_idx)) { + elem_slot_index = 0; + path.offset = 0; + parent_idx = next.unwrap_internal(); + parent = self.in_nodes.get(parent_idx).unwrap(); + } else { + return None; + } + } + + let elem = &parent.children[elem_slot_index]; + let leaf = self.leaf_nodes.get(elem.arena.unwrap()).unwrap(); + // skip empty span + if leaf.elem.rle_len() <= path.offset { + path.offset -= leaf.elem.rle_len(); + elem_slot_index += 1; + } else { + path.leaf = elem.arena.unwrap_leaf().into(); + break; + } + } + + Some(path) + } + + fn get_leaf_slot(leaf_arena_index: RawArenaIndex, parent: &Node) -> usize { + parent + .children + .iter() + .position(|x| x.arena.unwrap_leaf() == leaf_arena_index) + .unwrap() + } + + /// Query the tree by custom query type + /// + /// Return None if the tree is empty + pub fn query(&self, query: &Q::QueryArg) -> Option + where + Q: Query, + { + self.query_with_finder_return::(query).0 + } + + pub fn query_with_finder_return(&self, query: &Q::QueryArg) -> (Option, Q) + where + Q: Query, + { + let mut finder = Q::init(query); + if self.is_empty() { + return (None, finder); + } + + let mut node = self.in_nodes.get(self.root.unwrap()).unwrap(); + let mut index; + let mut found = true; + loop { + let result = finder.find_node(query, &node.children); + debug_assert!(!node.children.is_empty()); + let i = result.index; + found = found && result.found; + index = node.children[i].arena; + match index { + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + ArenaIndex::Leaf(_) => { + let (offset, leaf_found) = finder.confirm_elem( + query, + &self.leaf_nodes.get(index.unwrap_leaf()).unwrap().elem, + ); + return ( + Some(QueryResult { + cursor: Cursor { + leaf: index.unwrap_leaf().into(), + offset, + }, + found: found && leaf_found, + }), + finder, + ); + } + } + } + } + + pub fn get_elem_mut(&mut self, leaf: LeafIndex) -> Option<&mut B::Elem> { + let node = self.leaf_nodes.get_mut(leaf.0)?; + Some(&mut node.elem) + } + + pub fn get_elem(&self, leaf: LeafIndex) -> Option<&::Elem> { + self.leaf_nodes.get(leaf.0).map(|x| &x.elem) + } + + /// Remove leaf node from the tree + /// + /// If it's already removed, this method will return None + pub fn remove_leaf(&mut self, path: Cursor) -> Option { + let leaf = self.leaf_nodes.get_mut(path.leaf.0)?; + let parent_idx = leaf.parent(); + let parent = self.in_nodes.get_mut(leaf.parent).unwrap(); + let index = Self::get_leaf_slot(path.leaf.0, parent); + let child = parent.children.remove(index); + let is_lack = parent.is_lack(); + let is_empty = parent.is_empty(); + debug_assert_eq!(child.arena.unwrap(), path.leaf.0); + let elem = self.leaf_nodes.remove(child.arena.unwrap()).unwrap().elem; + + self.recursive_update_cache(parent_idx, B::USE_DIFF, None); + if is_empty { + self.remove_internal_node(parent_idx.unwrap()); + } else if is_lack { + self.handle_lack_recursively(parent_idx); + } + + Some(elem) + } + + fn remove_internal_node(&mut self, node: RawArenaIndex) { + if node == self.root.unwrap() { + return; + } + + let node = self.in_nodes.remove(node).unwrap(); + if let Some(parent_idx) = node.parent { + let parent = self.in_nodes.get_mut(parent_idx.unwrap_internal()).unwrap(); + parent.children.remove(node.parent_slot as usize); + let is_lack = parent.is_lack(); + let is_empty = parent.is_empty(); + self.update_children_parent_slot_from(parent_idx, node.parent_slot as usize); + if is_empty { + self.remove_internal_node(parent_idx.unwrap_internal()); + } else if is_lack { + self.handle_lack_recursively(parent_idx); + } + } else { + // ignore remove root + unreachable!() + } + } + + /// Update the elements in place. + /// + /// If the range.start or range.end is in the middle of a leaf node, the leaf node + /// will be splitted into two leaf nodes. The new leaf nodes will be returned. + /// + /// F should returns `Some(cache_diff)` if cache needs to be updated. Otherwise, returns None. + /// + /// If the given range has zero length, f will still be called, and the slice will + /// have same `start` and `end` field + /// + /// TODO: need better test coverage + pub fn update(&mut self, range: Range, f: &mut F) -> SplittedLeaves + where + F: FnMut(&mut B::Elem) -> Option, + { + let mut splitted = SplittedLeaves::default(); + let start = range.start; + let SplitInfo { + new_pos: end, + new_leaf, + .. + } = self.split_leaf_if_needed(range.end); + splitted.push_option(new_leaf); + let SplitInfo { + new_pos: start, + new_leaf, + .. + } = self.split_leaf_if_needed(start); + splitted.push_option(new_leaf); + let Some(start) = start else { + return splitted; + }; + let start_leaf = start.leaf; + let mut path = self.get_path(start_leaf.into()); + let mut dirty_map: LeafDirtyMap = FxHashMap::default(); + let mut to_remove = Vec::default(); + + loop { + let current_leaf = path.last().unwrap(); + if let Some(end) = end { + if current_leaf.arena.unwrap_leaf() == end.leaf.0 { + break; + } + } + + let node = self + .leaf_nodes + .get_mut(current_leaf.arena.unwrap_leaf()) + .unwrap(); + let cache_diff = f(&mut node.elem); + if node.elem.can_remove() { + to_remove.push(current_leaf.arena); + } + + if let Some(diff) = cache_diff { + add_leaf_dirty_map(current_leaf.arena, &mut dirty_map, diff); + } + + if !self.next_sibling(&mut path) { + break; + } + } + + if !dirty_map.is_empty() { + self.update_dirty_cache_map(dirty_map); + } else { + self.in_nodes + .get(self.root.unwrap_internal()) + .unwrap() + .calc_cache(&mut self.root_cache, None); + } + + for leaf in to_remove { + self.remove_leaf(Cursor { + leaf: leaf.unwrap().into(), + offset: 0, + }); + } + splitted + } + + /// Prefer begin of the next leaf node than end of the current leaf node + /// + /// When path.offset == leaf.rle_len(), this method will return + /// the next leaf node with offset 0 + #[allow(unused)] + pub fn prefer_right(&self, path: Cursor) -> Option { + if path.offset == 0 { + return Some(path); + } + + let leaf = self.leaf_nodes.get(path.leaf.0).unwrap(); + if path.offset == leaf.elem.rle_len() { + self.next_elem(path) + } else { + Some(path) + } + } + + /// Prefer end of the previous leaf node than begin of the current leaf node + /// + /// When path.offset == 0, this method will return + /// the previous leaf node with offset leaf.rle_len() + #[allow(unused)] + pub fn prefer_left(&self, path: Cursor) -> Option { + if path.offset != 0 { + return Some(path); + } + + let elem = self.prev_elem(path); + if let Some(elem) = elem { + let leaf = self.leaf_nodes.get(elem.leaf.0).unwrap(); + Some(Cursor { + leaf: elem.leaf, + offset: leaf.elem.rle_len(), + }) + } else { + None + } + } + + /// Update leaf node's elements. + /// + /// `f` returns Option<(cache_diff, new_insert_1, new_insert2)> + /// + /// - If returned value is `None`, the cache will not be updated. + /// - If leaf_node.can_remove(), it will be removed from the tree. + /// + /// Returns (path, splitted_leaves), if is is still valid after this method. (If the leaf node is removed, the path will be None) + pub fn update_leaf_by_search>( + &mut self, + q: &Q::QueryArg, + f: impl FnOnce( + &mut B::Elem, + QueryResult, + ) -> Option<(B::CacheDiff, Option, Option)>, + ) -> (Option, SplittedLeaves) { + if self.is_empty() { + panic!("update_leaf_by_search called on empty tree"); + } + + let mut splitted = SplittedLeaves::default(); + let mut finder = Q::init(q); + let mut path = NodePath::default(); + let mut node_idx = self.root; + let mut child_arr_pos = 0; + while let ArenaIndex::Internal(node_idx_inner) = node_idx { + path.push(Idx { + arena: ArenaIndex::Internal(node_idx_inner), + arr: child_arr_pos, + }) + .unwrap(); + let node = self.in_nodes.get(node_idx_inner).unwrap(); + let result = finder.find_node(q, &node.children); + child_arr_pos = result.index as u8; + node_idx = node.children[result.index].arena; + } + + let leaf = self.get_leaf_mut(node_idx); + let (offset, found) = finder.confirm_elem(q, &leaf.elem); + let ans = QueryResult { + cursor: Cursor { + leaf: node_idx.unwrap_leaf().into(), + offset, + }, + found, + }; + let Some((diff, new_insert_1, new_insert_2)) = f(&mut leaf.elem, ans) else { + return (Some(ans.cursor), splitted); + }; + + if new_insert_2.is_some() { + unimplemented!() + } + + // Delete + if leaf.elem.can_remove() { + // handle deletion + // leaf node should be deleted + assert!(new_insert_1.is_none()); + assert!(new_insert_2.is_none()); + self.leaf_nodes.remove(node_idx.unwrap()).unwrap(); + let mut is_first = true; + let mut is_child_lack = false; + let mut child_idx = node_idx; + + // iterate from leaf to root, child to parent + while let Some(Idx { + arena: parent_idx, + arr: parent_arr_pos, + }) = path.pop() + { + let parent = self.get_internal_mut(parent_idx); + if is_first { + parent.children.remove(child_arr_pos as usize); + is_first = false; + } else { + B::apply_cache_diff(&mut parent.children[child_arr_pos as usize].cache, &diff); + } + + let is_lack = parent.is_lack(); + + if is_child_lack { + self.handle_lack_single_layer(child_idx); + } + + is_child_lack = is_lack; + child_idx = parent_idx; + child_arr_pos = parent_arr_pos; + } + + B::apply_cache_diff(&mut self.root_cache, &diff); + + if is_child_lack { + let root = self.get_internal_mut(self.root); + if root.children.len() == 1 && !root.is_child_leaf() { + self.try_reduce_levels(); + } + } + + return (None, splitted); + } + + let mut new_cache_and_child = None; + if let Some(new_insert_1) = new_insert_1 { + let cache = B::get_elem_cache(&leaf.elem); + let child = self.alloc_leaf_child(new_insert_1, path.last().unwrap().arena.unwrap()); + splitted.push(child.arena); + new_cache_and_child = Some((cache, child)); + } + + while let Some(Idx { + arena: parent_idx, + arr: parent_arr_pos, + }) = path.pop() + { + let parent = self.get_internal_mut(parent_idx); + match take(&mut new_cache_and_child) { + Some((cache, child)) => { + parent.children[child_arr_pos as usize].cache = cache; + parent + .children + .insert(child_arr_pos as usize + 1, child) + .unwrap(); + let is_full = parent.is_full(); + if !parent.is_child_leaf() { + self.update_children_parent_slot_from( + parent_idx, + child_arr_pos as usize + 1, + ); + } + if is_full { + let (_, _, this_cache, right_child) = self.split_node(parent_idx, None); + new_cache_and_child = Some((this_cache, right_child)); + } + } + None => { + B::apply_cache_diff(&mut parent.children[child_arr_pos as usize].cache, &diff); + } + } + + child_arr_pos = parent_arr_pos; + } + + if let Some((cache, child)) = new_cache_and_child { + self.split_root(cache, child); + } else { + B::apply_cache_diff(&mut self.root_cache, &diff); + } + + (Some(ans.cursor), splitted) + } + + /// Update leaf node's elements, return true if cache need to be updated + /// + /// `f` returns (is_cache_updated, cache_diff, new_insert_1, new_insert2) + /// + /// - If leaf_node.can_remove(), it will be removed from the tree. + /// + /// Returns true if the node_idx is still valid. (If the leaf node is removed, it will return false). + pub fn update_leaf( + &mut self, + node_idx: LeafIndex, + f: impl FnOnce(&mut B::Elem) -> (bool, Option, Option), + ) -> (bool, SplittedLeaves) { + let mut splitted = SplittedLeaves::default(); + let node = self.leaf_nodes.get_mut(node_idx.0).unwrap(); + let mut parent_idx = node.parent(); + let (need_update_cache, mut new_insert_1, mut new_insert_2) = f(&mut node.elem); + { + // Normalize returned values + // + // If the node can be removed, then both new_insert_1 & new_insert_2 should be None + // The priority is node.elem > new_insert_1 > new_insert_2 + // + // And new_insert_1 and new_insert_2 should not match `can_remove` condition + if let Some(ref new_1) = new_insert_1 { + if new_1.can_remove() { + new_insert_1 = new_insert_2.take(); + if let Some(ref new_1) = new_insert_1 { + if new_1.can_remove() { + new_insert_1 = None; + } + } + } + } + + if let Some(ref new_2) = new_insert_2 { + if new_2.can_remove() { + new_insert_2 = None; + } else if new_insert_1.is_none() { + std::mem::swap(&mut new_insert_1, &mut new_insert_2); + } + } + + if node.elem.can_remove() { + if let Some(new_1) = new_insert_1 { + node.elem = new_1; + new_insert_1 = new_insert_2.take(); + } + } + } + + let deleted = node.elem.can_remove(); + + if need_update_cache { + self.recursive_update_cache(node_idx.into(), B::USE_DIFF, None); + } + + if deleted { + debug_assert!(new_insert_1.is_none()); + debug_assert!(new_insert_2.is_none()); + self.leaf_nodes.remove(node_idx.0).unwrap(); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let slot = Self::get_leaf_slot(node_idx.0, parent); + parent.children.remove(slot); + let is_lack = parent.is_lack(); + if is_lack { + self.handle_lack_recursively(parent_idx); + } + + (false, splitted) + } else if new_insert_1.is_none() { + debug_assert!(new_insert_2.is_none()); + return (true, splitted); + } else { + if new_insert_1.is_some() && new_insert_2.is_none() { + // try merge new insert to next element + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let slot = Self::get_leaf_slot(node_idx.0, parent); + if slot + 1 < parent.children.len() { + let next_idx = parent.children[slot + 1].arena.unwrap().into(); + let next = self.get_elem_mut(next_idx).unwrap(); + let new = new_insert_1.as_ref().unwrap(); + if new.can_merge(next) { + next.merge_left(new); + self.recursive_update_cache(next_idx.into(), B::USE_DIFF, None); + splitted.push(next_idx.into()); + return (true, splitted); + } + } + } + + let count = if new_insert_1.is_some() { 1 } else { 0 } + + if new_insert_2.is_some() { 1 } else { 0 }; + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let parent = if parent.children.len() + count >= MAX_CHILDREN_NUM { + self.split(parent_idx); + let node = self.leaf_nodes.get(node_idx.0).unwrap(); + parent_idx = node.parent(); + self.in_nodes.get_mut(parent_idx.unwrap()).unwrap() + } else { + parent + }; + + let new: HeaplessVec<_, 2> = new_insert_1 + .into_iter() + .chain(new_insert_2) + .map(|elem| { + // Allocate new leaf node + let parent_index = parent_idx.unwrap(); + let elem_cache = B::get_elem_cache(&elem); + let new_leaf_index = { + let leaf = LeafNode { + elem, + parent: parent_index, + }; + let arena_index = self.leaf_nodes.insert(leaf); + ArenaIndex::Leaf(arena_index) + }; + Child { + arena: new_leaf_index, + cache: elem_cache, + } + }) + .collect(); + let slot = Self::get_leaf_slot(node_idx.0, parent); + for (i, v) in new.into_iter().enumerate() { + splitted.push(v.arena); + parent.children.insert(slot + 1 + i, v).unwrap(); + } + + assert!(!parent.is_full()); + self.recursive_update_cache(parent_idx, B::USE_DIFF, None); + (true, splitted) + } + } + + /// Update the given leaves with the given function in the given range. + /// + /// - The range descibes the range inside the leaf node. + /// - There can be multiple ranges in the same leaf node. + /// - The cahce will be recalculated for each affected node + /// - It doesn't guarantee the applying order + /// + /// Currently, the time complexity is O(m^2) for each leaf node, + /// where m is the number of ranges inside the same leaf node. + /// If we have a really large m, this function need to be optimized. + pub fn update_leaves_with_arg_in_ranges( + &mut self, + mut args: Vec<(LeafIndex, Range, A)>, + mut f: impl FnMut(&mut B::Elem, &A), + ) -> Vec { + args.sort_by_key(|x| x.0); + let mut new_leaves = Vec::new(); + let mut dirty_map: LeafDirtyMap = Default::default(); + let mut new_elems_at_cursor: FxHashMap> = Default::default(); + for (leaf, group) in &args.into_iter().group_by(|x| x.0) { + // This loop doesn't change the shape of the tree. It only changes each leaf element. + // A leaf element may be splitted into several parts. The first part stay in the tree, + // while the rest of them are inserted into `new_elem_at_cursor`, which will be inserted + // into the tree later. + let leaf_node = self.leaf_nodes.get_mut(leaf.0).unwrap(); + let len = leaf_node.elem().rle_len(); + let mut split_at = BTreeSet::new(); + // PERF we can avoid this alloc and `group_by` + let group: Vec<_> = group.into_iter().collect(); + for (_, range, _) in group.iter() { + split_at.insert(range.start); + split_at.insert(range.end); + } + + split_at.remove(&0); + split_at.remove(&len); + + // leaf_node.elem is the first elem + let old_cache = B::get_elem_cache(&leaf_node.elem); + if split_at.is_empty() { + // doesn't need to split + for (_, range, a) in group.iter() { + assert_eq!(range.start, 0); + assert_eq!(range.end, len); + f(&mut leaf_node.elem, a); + } + } else { + let mut new_elems = Vec::new(); + + let first_split = split_at.first().copied().unwrap(); + + // handle first element + let mut elem = leaf_node.elem.split(first_split); + for (_, r, a) in group.iter() { + if r.start == 0 { + f(&mut leaf_node.elem, a); + } + } + + // handle elements in the middle + let mut last_index = first_split; + for &index in split_at.iter().skip(1) { + let next_elem = elem.split(index - last_index); + let cur_range = last_index..index; + for (_, r, a) in group.iter() { + if r.start <= cur_range.start && cur_range.end <= r.end { + f(&mut elem, a); + } + } + + new_elems.push(elem); + elem = next_elem; + last_index = index; + } + + // handle the last element + for (_, r, a) in group.iter() { + if r.end == len { + f(&mut elem, a); + } + } + + new_elems.push(elem); + new_elems_at_cursor.insert( + Cursor { + leaf, + offset: leaf_node.elem().rle_len(), + }, + new_elems, + ); + } + + let new_cache = B::get_elem_cache(&leaf_node.elem); + let diff = B::sub_cache(&new_cache, &old_cache); + dirty_map.insert(leaf.into(), diff); + } + + // update cache + self.update_dirty_cache_map(dirty_map); + + // PERF we can use batch insert to optimize this + // insert the new leaf nodes + for (mut cursor, elems) in new_elems_at_cursor { + for elem in elems.into_iter() { + // PERF can use insert many to optimize when it's supported + let result = self.insert_by_path(cursor, elem); + let len = self.get_elem(result.0.leaf).unwrap().rle_len(); + new_leaves.push(result.0.leaf); + debug_assert_eq!(result.1.arr.len(), 0); + cursor = Cursor { + leaf: result.0.leaf, + offset: len, + }; + } + } + + new_leaves + } + + fn update_root_cache(&mut self) { + self.in_nodes + .get(self.root.unwrap_internal()) + .unwrap() + .calc_cache(&mut self.root_cache, None); + } + + fn update_dirty_cache_map(&mut self, mut diff_map: LeafDirtyMap) { + // diff_map only contains leaf nodes when this function is called + let mut visit_set: FxHashSet = diff_map.keys().copied().collect(); + while !visit_set.is_empty() { + for child_idx in take(&mut visit_set) { + let (parent_idx, cache_diff) = match child_idx { + ArenaIndex::Leaf(leaf_idx) => { + let node = self.leaf_nodes.get(leaf_idx).unwrap(); + let parent_idx = node.parent; + let parent = self.in_nodes.get_mut(parent_idx).unwrap(); + let cache_diff = diff_map.remove(&child_idx).unwrap(); + for child in parent.children.iter_mut() { + if child.arena == child_idx { + B::apply_cache_diff(&mut child.cache, &cache_diff); + break; + } + } + + (ArenaIndex::Internal(parent_idx), cache_diff) + } + ArenaIndex::Internal(_) => { + let node = self.in_nodes.get(child_idx.unwrap_internal()).unwrap(); + let Some(parent_idx) = node.parent else { + continue; + }; + let (child, parent) = self.get2_mut(child_idx, parent_idx); + let cache_diff = child.calc_cache( + &mut parent.children[child.parent_slot as usize].cache, + diff_map.remove(&child_idx), + ); + + (parent_idx, cache_diff) + } + }; + + visit_set.insert(parent_idx); + if let Some(e) = diff_map.get_mut(&parent_idx) { + B::merge_cache_diff(e, &cache_diff); + } else { + diff_map.insert(parent_idx, cache_diff); + } + } + } + + self.in_nodes + .get(self.root.unwrap_internal()) + .unwrap() + .calc_cache(&mut self.root_cache, None); + } + + /// Removed deleted children. `deleted` means they are removed from the arena. + fn filter_deleted_children(&mut self, internal_node: ArenaIndex) { + let node = self + .in_nodes + .get_mut(internal_node.unwrap_internal()) + .unwrap(); + // PERF: I hate this pattern... + let mut children = take(&mut node.children); + children.retain(|x| match x.arena { + ArenaIndex::Leaf(leaf) => self.leaf_nodes.contains(leaf), + ArenaIndex::Internal(index) => self.in_nodes.contains(index), + }); + let node = self + .in_nodes + .get_mut(internal_node.unwrap_internal()) + .unwrap(); + node.children = children; + } + + pub fn iter(&self) -> impl Iterator + '_ { + let mut path = self.first_path().unwrap_or_default(); + path.pop(); + let idx = path.last().copied().unwrap_or(Idx::new(self.root, 0)); + debug_assert!(matches!(idx.arena, ArenaIndex::Internal(_))); + let node = self.get_internal(idx.arena); + let mut iter = node.children.iter(); + core::iter::from_fn(move || loop { + if path.is_empty() { + return None; + } + + match iter.next() { + None => { + if !self.next_sibling(&mut path) { + return None; + } + + let idx = *path.last().unwrap(); + debug_assert!(matches!(idx.arena, ArenaIndex::Internal(_))); + let node = self.get_internal(idx.arena); + iter = node.children.iter(); + } + Some(elem) => { + let leaf = self.leaf_nodes.get(elem.arena.unwrap_leaf()).unwrap(); + return Some(&leaf.elem); + } + } + }) + } + + pub fn drain(&mut self, range: Range) -> iter::Drain { + iter::Drain::new(self, Some(range.start), Some(range.end)) + } + + pub fn drain_by_query>(&mut self, range: Range) -> iter::Drain { + let start = self.query::(&range.start); + let end = self.query::(&range.end); + iter::Drain::new(self, start, end) + } + + fn first_path(&self) -> Option { + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + if node.is_empty() { + return None; + } + + let mut path = NodePath::new(); + loop { + path.push(Idx::new(index, 0)).unwrap(); + match index { + ArenaIndex::Leaf(_) => { + break; + } + ArenaIndex::Internal(_) => { + index = node.children[0].arena; + if let ArenaIndex::Internal(i) = index { + node = self.in_nodes.get(i).unwrap(); + }; + } + } + } + + Some(path) + } + + fn last_path(&self) -> Option { + let mut path = NodePath::new(); + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + let mut pos_in_parent = 0; + if node.is_empty() { + return None; + } + + loop { + path.push(Idx::new(index, pos_in_parent)).unwrap(); + match index { + ArenaIndex::Leaf(_) => { + break; + } + ArenaIndex::Internal(_) => { + pos_in_parent = node.children.len() as u8 - 1; + index = node.children[node.children.len() - 1].arena; + if let ArenaIndex::Internal(i) = index { + node = self.in_nodes.get(i).unwrap(); + } + } + } + } + + Some(path) + } + + pub fn first_leaf(&self) -> Option { + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + loop { + index = node.children.first()?.arena; + match index { + ArenaIndex::Leaf(leaf) => { + return Some(leaf.into()); + } + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + } + } + } + + pub fn last_leaf(&self) -> Option { + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + loop { + index = node.children.last()?.arena; + match index { + ArenaIndex::Leaf(leaf) => { + return Some(leaf.into()); + } + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + } + } + } + + pub fn range(&self, range: Range) -> Option> + where + Q: Query, + { + if self.is_empty() { + return None; + } + + Some(self.query::(&range.start).unwrap()..self.query::(&range.end).unwrap()) + } + + pub fn iter_range( + &self, + range: impl RangeBounds, + ) -> impl Iterator> + '_ { + let start = match range.start_bound() { + std::ops::Bound::Included(start) => *start, + std::ops::Bound::Excluded(_) => unreachable!(), + std::ops::Bound::Unbounded => self.start_cursor().unwrap(), + }; + let (inclusive, end) = match range.end_bound() { + std::ops::Bound::Included(end) => (true, *end), + std::ops::Bound::Excluded(end) => (false, *end), + std::ops::Bound::Unbounded => (true, self.end_cursor().unwrap()), + }; + self._iter_range(start, end, inclusive) + } + + fn _iter_range( + &self, + start: Cursor, + end: Cursor, + inclusive_end: bool, + ) -> impl Iterator> + '_ { + let node_iter = iter::Iter::new( + self, + self.get_path(start.leaf.into()), + self.get_path(end.leaf.into()), + ); + node_iter.filter_map(move |(path, node)| { + let leaf = LeafIndex(path.last().unwrap().arena.unwrap_leaf()); + if end.leaf == leaf && end.offset == 0 && !inclusive_end { + return None; + } + + Some(ElemSlice { + cursor: Cursor { leaf, offset: 0 }, + elem: &node.elem, + start: if start.leaf == leaf { + Some(start.offset) + } else { + None + }, + end: if end.leaf == leaf { + Some(end.offset) + } else { + None + }, + }) + }) + } + + pub fn start_cursor(&self) -> Option { + Some(Cursor { + leaf: self.first_leaf()?, + offset: 0, + }) + } + + pub fn end_cursor(&self) -> Option { + let leaf = self.last_leaf()?; + let node = self.get_leaf(leaf.into()); + Some(Cursor { + leaf, + offset: node.elem.rle_len(), + }) + } + + /// Split the internal node at path into two nodes recursively upwards. + /// + // at call site the cache at path can be out-of-date. + // the cache will be up-to-date after this method + fn split(&mut self, node_idx: ArenaIndex) { + self.split_at(node_idx, None) + } + + fn split_at(&mut self, node_idx: ArenaIndex, at: Option) { + let (node_parent, node_parent_slot, this_cache, right_child) = + self.split_node(node_idx, at); + + self.inner_insert_node( + node_parent, + node_parent_slot as usize, + this_cache, + right_child, + ); + // don't need to recursive update cache + } + + fn split_node( + &mut self, + node_idx: ArenaIndex, + at: Option, + ) -> (Option, u8, ::Cache, Child) { + let node = self.in_nodes.get_mut(node_idx.unwrap_internal()).unwrap(); + let node_parent = node.parent; + let node_parent_slot = node.parent_slot; + let right: Node = Node { + parent: node.parent, + parent_slot: u8::MAX, + children: HeaplessVec::new(), + }; + + // split + let split = at.unwrap_or(node.children.len() / 2); + let right_children = HeaplessVec::from_slice(&node.children[split..]).unwrap(); + delete_range(&mut node.children, split..); + + // update cache + let mut right_cache = B::Cache::default(); + let right_arena_idx = self.in_nodes.insert(right); + let this_cache = { + let node = self.get_internal_mut(node_idx); + let mut cache = Default::default(); + node.calc_cache(&mut cache, None); + cache + }; + + // update children's parent info + for (i, child) in right_children.iter().enumerate() { + if matches!(child.arena, ArenaIndex::Internal(_)) { + let child = self.get_internal_mut(child.arena); + child.parent = Some(ArenaIndex::Internal(right_arena_idx)); + child.parent_slot = i as u8; + } else { + self.get_leaf_mut(child.arena).parent = right_arena_idx; + } + } + + let right = self.in_nodes.get_mut(right_arena_idx).unwrap(); + right.children = right_children; + // update parent cache + right.calc_cache(&mut right_cache, None); + let right_child = Child { + arena: ArenaIndex::Internal(right_arena_idx), + cache: right_cache, + }; + (node_parent, node_parent_slot, this_cache, right_child) + } + + // call site should ensure the cache is up-to-date after this method + fn inner_insert_node( + &mut self, + parent_idx: Option, + index: usize, + new_cache: B::Cache, + node: Child, + ) { + if let Some(parent_idx) = parent_idx { + let parent = self.get_internal_mut(parent_idx); + parent.children[index].cache = new_cache; + parent.children.insert(index + 1, node).unwrap(); + let is_full = parent.is_full(); + self.update_children_parent_slot_from(parent_idx, index + 1); + if is_full { + self.split(parent_idx); + } + } else { + self.split_root(new_cache, node); + } + } + + /// Update the `parent_slot` fields in `children[index..]` + fn update_children_parent_slot_from(&mut self, parent_idx: ArenaIndex, index: usize) { + let parent = self.get_internal_mut(parent_idx); + if parent.children.len() <= index || parent.is_child_leaf() { + return; + } + + // PERF: Is there a way to avoid `take` like this? + let children = take(&mut parent.children); + for (i, child) in children[index..].iter().enumerate() { + let idx = index + i; + let child = self.get_internal_mut(child.arena); + child.parent_slot = idx as u8; + } + let parent = self.get_internal_mut(parent_idx); + parent.children = children; + } + + /// right's cache should be up-to-date + fn split_root(&mut self, new_cache: B::Cache, right: Child) { + let root_idx = self.root; + // set right parent + let right_node = &mut self.get_internal_mut(right.arena); + right_node.parent_slot = 1; + right_node.parent = Some(root_idx); + let root = self.get_internal_mut(self.root); + // let left be root + let mut left_node: Node = core::mem::replace( + root, + Node { + parent: None, + parent_slot: 0, + children: Default::default(), + }, + ); + left_node.parent_slot = 0; + // set left parent + left_node.parent = Some(root_idx); + + // push left and right to root.children + root.children = Default::default(); + let left_children = left_node.children.clone(); + let left_arena = self.in_nodes.insert(left_node); + let left = Child::new(ArenaIndex::Internal(left_arena), new_cache); + let mut cache = std::mem::take(&mut self.root_cache); + let root = self.get_internal_mut(self.root); + root.children.push(left).unwrap(); + root.children.push(right).unwrap(); + + // update new root cache + root.calc_cache(&mut cache, None); + + for (i, child) in left_children.iter().enumerate() { + if child.is_internal() { + let node = self.get_internal_mut(child.arena); + node.parent = Some(ArenaIndex::Internal(left_arena)); + node.parent_slot = i as u8; + } else { + self.get_leaf_mut(child.arena).parent = left_arena; + } + } + + self.root_cache = cache; + } + + #[inline] + pub fn get_internal_mut(&mut self, index: ArenaIndex) -> &mut Node { + self.in_nodes.get_mut(index.unwrap_internal()).unwrap() + } + + #[inline] + pub fn get_leaf_mut(&mut self, index: ArenaIndex) -> &mut LeafNode { + self.leaf_nodes.get_mut(index.unwrap_leaf()).unwrap() + } + + #[inline] + fn get2_mut(&mut self, a: ArenaIndex, b: ArenaIndex) -> (&mut Node, &mut Node) { + let (a, b) = self + .in_nodes + .get2_mut(a.unwrap_internal(), b.unwrap_internal()); + (a.unwrap(), b.unwrap()) + } + + /// # Panic + /// + /// If the given index is not valid or deleted + #[inline] + pub fn get_internal(&self, index: ArenaIndex) -> &Node { + self.in_nodes.get(index.unwrap_internal()).unwrap() + } + + #[inline] + pub fn get_leaf(&self, index: ArenaIndex) -> &LeafNode { + self.leaf_nodes.get(index.unwrap_leaf()).unwrap() + } + + /// The given node is lack of children. + /// We should merge it into its neighbor or borrow from its neighbor. + /// + /// Given a random neighbor is neither full or lack, it's guaranteed + /// that we can either merge into or borrow from it without breaking + /// the balance rule. + /// + /// - The caches in parent's subtree should be up-to-date when calling this. + /// - The caches in the parent node will be updated + fn handle_lack_recursively(&mut self, node_idx: ArenaIndex) { + let mut lack_info = self.handle_lack_single_layer(node_idx); + while let Some(parent) = lack_info.parent_lack { + lack_info = self.handle_lack_single_layer(parent); + } + } + + /// The given node is lack of children. This method doesn't handle parent's lack. + /// + /// - The caches in parent's subtree should be up-to-date when calling this. + /// - The caches in the parent node will be updated + fn handle_lack_single_layer(&mut self, node_idx: ArenaIndex) -> LackInfo { + if self.root == node_idx { + self.try_reduce_levels(); + return LackInfo { parent_lack: None }; + } + + let node = self.get_internal(node_idx); + let parent_idx = node.parent.unwrap(); + let parent = self.get_internal(parent_idx); + debug_assert_eq!(parent.children[node.parent_slot as usize].arena, node_idx); + if node.children.is_empty() { + let slot = node.parent_slot as usize; + self.get_internal_mut(parent_idx).children.remove(slot); + self.in_nodes.remove(node_idx.unwrap_internal()); + self.update_children_parent_slot_from(parent_idx, slot); + return LackInfo { + parent_lack: Some(parent_idx), + }; + } + let ans = match self.pair_neighbor(node_idx) { + Some((a_idx, b_idx)) => { + let parent = self.get_internal_mut(parent_idx); + let mut a_cache = std::mem::take(&mut parent.children[a_idx.arr as usize].cache); + let mut b_cache = std::mem::take(&mut parent.children[b_idx.arr as usize].cache); + let mut re_parent = FxHashMap::default(); + + let (a, b) = self + .in_nodes + .get2_mut(a_idx.arena.unwrap_internal(), b_idx.arena.unwrap_internal()); + let a = a.unwrap(); + let b = b.unwrap(); + let ans = if a.len() + b.len() >= MAX_CHILDREN_NUM { + // move partially + if a.len() < b.len() { + // move part of b's children to a + let move_len = (b.len() - a.len()) / 2; + for child in &b.children[..move_len] { + re_parent.insert(child.arena, (a_idx.arena, a.children.len())); + a.children.push(child.clone()).unwrap(); + } + delete_range(&mut b.children, ..move_len); + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i)); + } + } else { + // move part of a's children to b + let move_len = (a.len() - b.len()) / 2; + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i + move_len)); + } + let mut b_children = + HeaplessVec::from_slice(&a.children[a.children.len() - move_len..]) + .unwrap(); + for child in take(&mut b.children) { + b_children.push(child).unwrap(); + } + b.children = b_children; + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i)); + } + let len = a.children.len(); + delete_range(&mut a.children, len - move_len..); + } + a.calc_cache(&mut a_cache, None); + b.calc_cache(&mut b_cache, None); + let parent = self.get_internal_mut(parent_idx); + parent.children[a_idx.arr as usize].cache = a_cache; + parent.children[b_idx.arr as usize].cache = b_cache; + LackInfo { + parent_lack: if parent.is_lack() { + Some(parent_idx) + } else { + None + }, + } + } else { + // merge + let is_parent_lack = if node_idx == a_idx.arena { + // merge b to a, delete b + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (a_idx.arena, a.children.len() + i)); + } + + for child in take(&mut b.children) { + a.children.push(child).unwrap(); + } + + a.calc_cache(&mut a_cache, None); + let parent = self.get_internal_mut(parent_idx); + parent.children[a_idx.arr as usize].cache = a_cache; + parent.children.remove(b_idx.arr as usize); + let is_lack = parent.is_lack(); + self.purge(b_idx.arena); + self.update_children_parent_slot_from(parent_idx, b_idx.arr as usize); + is_lack + } else { + // merge a to b, delete a + for (i, child) in a.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i)); + } + for (i, child) in b.children.iter().enumerate() { + re_parent.insert(child.arena, (b_idx.arena, i + a.children.len())); + } + + for child in take(&mut b.children) { + a.children.push(child).unwrap(); + } + + b.children = take(&mut a.children); + b.calc_cache(&mut b_cache, None); + let parent = self.get_internal_mut(parent_idx); + parent.children[b_idx.arr as usize].cache = b_cache; + parent.children.remove(a_idx.arr as usize); + let is_lack = parent.is_lack(); + self.purge(a_idx.arena); + self.update_children_parent_slot_from(parent_idx, a_idx.arr as usize); + is_lack + }; + + LackInfo { + parent_lack: if is_parent_lack { + Some(parent_idx) + } else { + None + }, + } + }; + + // FIXME: make this work + if cfg!(debug_assertions) { + // let (a, b) = self + // .in_nodes + // .get2_mut(a_idx.arena.unwrap_internal(), b_idx.arena.unwrap_internal()); + // if let Some(a) = a { + // assert!(!a.is_lack() && !a.is_full()); + // } + // if let Some(b) = b { + // assert!(!b.is_lack() && !b.is_full()); + // } + } + + for (child, (parent, slot)) in re_parent { + match child { + ArenaIndex::Leaf(_) => { + let child = self.get_leaf_mut(child); + child.parent = parent.unwrap_internal(); + } + ArenaIndex::Internal(_) => { + let child = self.get_internal_mut(child); + child.parent = Some(parent); + child.parent_slot = slot as u8; + } + } + } + ans + } + None => LackInfo { + parent_lack: Some(parent_idx), + }, + }; + ans + } + + fn try_reduce_levels(&mut self) { + let mut reduced = false; + while self.get_internal(self.root).children.len() == 1 { + let root = self.get_internal(self.root); + if root.is_child_leaf() { + break; + } + + let child_arena = root.children[0].arena; + let child = self.in_nodes.remove(child_arena.unwrap_internal()).unwrap(); + let root = self.get_internal_mut(self.root); + let _ = core::mem::replace(root, child); + reduced = true; + // root cache should be the same as child cache because there is only one child + } + if reduced { + let root_idx = self.root; + let root = self.get_internal_mut(self.root); + root.parent = None; + root.parent_slot = u8::MAX; + self.reset_children_parent_pointer(root_idx); + } + } + + fn reset_children_parent_pointer(&mut self, parent_idx: ArenaIndex) { + let parent = self.in_nodes.get(parent_idx.unwrap_internal()).unwrap(); + let children = parent.children.clone(); + for child in children { + match child.arena { + ArenaIndex::Leaf(_) => { + let child = self.get_leaf_mut(child.arena); + child.parent = parent_idx.unwrap_internal(); + } + ArenaIndex::Internal(_) => { + let child = self.get_internal_mut(child.arena); + child.parent = Some(parent_idx); + } + } + } + } + + fn pair_neighbor(&self, this: ArenaIndex) -> Option<(Idx, Idx)> { + let node = self.get_internal(this); + let arr = node.parent_slot as usize; + let parent = self.get_internal(node.parent.unwrap()); + + if arr == 0 { + parent + .children + .get(1) + .map(|x| (Idx::new(this, arr as u8), Idx::new(x.arena, 1))) + } else { + parent + .children + .get(arr - 1) + .map(|x| (Idx::new(x.arena, arr as u8 - 1), Idx::new(this, arr as u8))) + } + } + + /// Sometimes we cannot use diff because no only the given node is changed, but also its siblings. + /// For example, after delete a range of nodes, we cannot use the diff from child to infer the diff of parent. + pub fn recursive_update_cache( + &mut self, + mut node_idx: ArenaIndex, + can_use_diff: bool, + cache_diff: Option, + ) { + if let ArenaIndex::Leaf(index) = node_idx { + let leaf = self.leaf_nodes.get(index).unwrap(); + let cache = B::get_elem_cache(&leaf.elem); + node_idx = leaf.parent(); + let node = self.get_internal_mut(node_idx); + node.children + .iter_mut() + .find(|x| x.arena.unwrap_leaf() == index) + .unwrap() + .cache = cache; + } + + if can_use_diff { + if let Some(diff) = cache_diff { + return self.recursive_update_cache_with_diff(node_idx, diff); + } + } + + let mut this_idx = node_idx; + let mut node = self.get_internal_mut(node_idx); + let mut this_arr = node.parent_slot; + if can_use_diff { + if node.parent.is_some() { + let parent_idx = node.parent.unwrap(); + let (parent, this) = self.get2_mut(parent_idx, this_idx); + let diff = + this.calc_cache(&mut parent.children[this_arr as usize].cache, cache_diff); + return self.recursive_update_cache_with_diff(parent_idx, diff); + } + } else { + while node.parent.is_some() { + let parent_idx = node.parent.unwrap(); + let (parent, this) = self.get2_mut(parent_idx, this_idx); + this.calc_cache(&mut parent.children[this_arr as usize].cache, None); + this_idx = parent_idx; + this_arr = parent.parent_slot; + node = parent; + } + } + + let mut root_cache = std::mem::take(&mut self.root_cache); + let root = self.root_mut(); + root.calc_cache( + &mut root_cache, + if can_use_diff { cache_diff } else { None }, + ); + self.root_cache = root_cache; + } + + fn recursive_update_cache_with_diff(&mut self, node_idx: ArenaIndex, diff: B::CacheDiff) { + let mut node = self.get_internal_mut(node_idx); + let mut this_arr = node.parent_slot; + while node.parent.is_some() { + let parent_idx = node.parent.unwrap(); + let parent = self.get_internal_mut(parent_idx); + B::apply_cache_diff(&mut parent.children[this_arr as usize].cache, &diff); + this_arr = parent.parent_slot; + node = parent; + } + + B::apply_cache_diff(&mut self.root_cache, &diff); + } + + fn purge(&mut self, index: ArenaIndex) { + let mut stack = vec![index]; + while let Some(x) = stack.pop() { + if let ArenaIndex::Leaf(index) = x { + self.leaf_nodes.remove(index); + + continue; + } + + let Some(node) = self.in_nodes.remove(x.unwrap()) else { + continue; + }; + + for x in node.children.iter() { + stack.push(x.arena); + } + } + } + + /// find the next sibling at the same level + /// + /// return false if there is no next sibling + #[must_use] + fn next_sibling(&self, path: &mut [Idx]) -> bool { + if path.len() <= 1 { + return false; + } + + let depth = path.len(); + let parent_idx = path[depth - 2]; + let this_idx = path[depth - 1]; + let parent = self.get_internal(parent_idx.arena); + match parent.children.get(this_idx.arr as usize + 1) { + Some(next) => { + path[depth - 1] = Idx::new(next.arena, this_idx.arr + 1); + } + None => { + if !self.next_sibling(&mut path[..depth - 1]) { + return false; + } + + let parent = self.get_internal(path[depth - 2].arena); + path[depth - 1] = Idx::new(parent.children[0].arena, 0); + } + } + + true + } + + fn next_same_level_in_node(&self, node_idx: ArenaIndex) -> Option { + match node_idx { + ArenaIndex::Leaf(_) => { + let leaf_idx = node_idx.unwrap_leaf(); + let leaf1 = self.leaf_nodes.get(leaf_idx).unwrap(); + let parent1 = self.get_internal(leaf1.parent()); + let (leaf, parent, index) = + (leaf1, parent1, Self::get_leaf_slot(leaf_idx, parent1)); + if index + 1 < parent.children.len() { + Some(parent.children[index + 1].arena) + } else if let Some(parent_next) = self.next_same_level_in_node(leaf.parent()) { + let parent_next = self.get_internal(parent_next); + Some(parent_next.children.first().unwrap().arena) + } else { + None + } + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + let parent = self.get_internal(node.parent?); + if let Some(next) = parent.children.get(node.parent_slot as usize + 1) { + Some(next.arena) + } else if let Some(parent_next) = self.next_same_level_in_node(node.parent?) { + let parent_next = self.get_internal(parent_next); + parent_next.children.first().map(|x| x.arena) + } else { + None + } + } + } + } + + fn prev_same_level_in_node(&self, node_idx: ArenaIndex) -> Option { + match node_idx { + ArenaIndex::Leaf(leaf_idx) => { + let leaf = self.leaf_nodes.get(leaf_idx).unwrap(); + let parent = self.get_internal(leaf.parent()); + let index = Self::get_leaf_slot(leaf_idx, parent); + if index > 0 { + Some(parent.children[index - 1].arena) + } else if let Some(parent_next) = self.prev_same_level_in_node(leaf.parent()) { + let parent_next = self.get_internal(parent_next); + Some(parent_next.children.last().unwrap().arena) + } else { + None + } + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + let parent = self.get_internal(node.parent?); + if node.parent_slot > 0 { + let Some(next) = parent.children.get(node.parent_slot as usize - 1) else { + unreachable!() + }; + Some(next.arena) + } else if let Some(parent_prev) = self.prev_same_level_in_node(node.parent?) { + let parent_prev = self.get_internal(parent_prev); + parent_prev.children.last().map(|x| x.arena) + } else { + None + } + } + } + } + + /// find the next element in the tree + pub fn next_elem(&self, path: Cursor) -> Option { + self.next_same_level_in_node(path.leaf.into()) + .map(|x| Cursor { + leaf: x.unwrap_leaf().into(), + offset: 0, + }) + } + + /// §flowstate vendor patch (bimodal-undo fix): the first leaf strictly + /// AFTER `path.leaf` in document order whose cache satisfies `pred`, + /// skipping whole subtrees whose aggregated caches fail it — + /// O(log n · branch factor) where the `next_elem` chain is linear. + /// + /// `pred` must be MONOTONE over subtree aggregation: if any descendant + /// leaf's cache satisfies it, every ancestor cache must too (true for + /// counting caches such as "number of elements with property P > 0"). + /// Returns `None` when no later leaf matches. + /// §flowstate debug helper: print each ancestor level's child caches for + /// `leaf`, marking the child on the path. Diagnostic only. + pub fn debug_dump_ancestor_caches(&self, leaf: LeafIndex, fmt: &mut dyn FnMut(&B::Cache) -> String) { + let mut node_idx: ArenaIndex = leaf.into(); + let mut level = 0; + loop { + let (parent_idx, slot) = match node_idx { + ArenaIndex::Leaf(leaf_idx) => { + let Some(leaf_node) = self.leaf_nodes.get(leaf_idx) else { + eprintln!(" dump: leaf {leaf_idx:?} missing"); + return; + }; + let parent = self.get_internal(leaf_node.parent()); + (leaf_node.parent(), Self::get_leaf_slot(leaf_idx, parent)) + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + match node.parent { + Some(p) => (p, node.parent_slot as usize), + None => return, + } + } + }; + let parent = self.get_internal(parent_idx); + let caches: Vec = parent + .children + .iter() + .enumerate() + .map(|(i, c)| { + let mark = if i == slot { "*" } else { "" }; + format!("[{i}{mark} {}]", fmt(&c.cache)) + }) + .collect(); + eprintln!(" dump level {level}: slot {slot}: {}", caches.join(" ")); + node_idx = parent_idx; + level += 1; + } + } + + pub fn next_leaf_matching( + &self, + path: Cursor, + pred: &mut dyn FnMut(&B::Cache) -> bool, + ) -> Option { + let mut node_idx: ArenaIndex = path.leaf.into(); + loop { + // Climb one level: identify our slot in the parent, then scan the + // siblings to our right. + let (parent_idx, slot) = match node_idx { + ArenaIndex::Leaf(leaf_idx) => { + let leaf = self.leaf_nodes.get(leaf_idx)?; + let parent = self.get_internal(leaf.parent()); + (leaf.parent(), Self::get_leaf_slot(leaf_idx, parent)) + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + (node.parent?, node.parent_slot as usize) + } + }; + let parent = self.get_internal(parent_idx); + for child in parent.children.iter().skip(slot + 1) { + if !pred(&child.cache) { + continue; + } + // Descend to the leftmost matching leaf of this subtree. + let mut cur = child.arena; + loop { + match cur { + ArenaIndex::Leaf(_) => return Some(cur.unwrap_leaf().into()), + ArenaIndex::Internal(_) => { + let node = self.get_internal(cur); + cur = node.children.iter().find(|c| pred(&c.cache))?.arena; + } + } + } + } + node_idx = parent_idx; + } + } + + pub fn prev_elem(&self, path: Cursor) -> Option { + self.prev_same_level_in_node(path.leaf.into()) + .map(|x| Cursor { + leaf: x.unwrap_leaf().into(), + offset: 0, + }) + } + + #[inline(always)] + pub fn root_cache(&self) -> &B::Cache { + &self.root_cache + } + + /// This method will release the memory back to OS. + /// Currently, it's just `*self = Self::new()` + #[inline(always)] + pub fn clear(&mut self) { + *self = Self::new(); + } + + #[inline(always)] + fn root_mut(&mut self) -> &mut Node { + self.get_internal_mut(self.root) + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.get_internal(self.root).is_empty() + } + + fn get_path(&self, idx: ArenaIndex) -> NodePath { + let mut path = NodePath::new(); + let mut node_idx = idx; + while node_idx != self.root { + match node_idx { + ArenaIndex::Leaf(inner_node_idx) => { + let node = self.leaf_nodes.get(inner_node_idx).unwrap(); + let parent = self.in_nodes.get(node.parent).unwrap(); + let index = Self::get_leaf_slot(inner_node_idx, parent); + path.push(Idx::new(node_idx, index as u8)).unwrap(); + node_idx = ArenaIndex::Internal(node.parent); + } + ArenaIndex::Internal(_) => { + let node = self.get_internal(node_idx); + path.push(Idx::new(node_idx, node.parent_slot)).unwrap(); + node_idx = node.parent.unwrap(); + } + } + } + path.push(Idx::new(self.root, 0)).unwrap(); + path.reverse(); + path + } + + pub fn push(&mut self, elem: B::Elem) -> Cursor { + let mut is_full = false; + let mut parent_idx = self.root; + let mut update_cache_idx = parent_idx; + let cache = B::get_elem_cache(&elem); + let ans = if self.is_empty() { + let data = self.alloc_leaf_child(elem, parent_idx.unwrap()); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let ans = data.arena; + parent.children.push(data).unwrap(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + } else { + let leaf_idx = self.last_leaf().unwrap(); + let leaf = self.leaf_nodes.get_mut(leaf_idx.0).unwrap(); + parent_idx = leaf.parent(); + if leaf.elem.can_merge(&elem) { + update_cache_idx = leaf_idx.into(); + let offset = leaf.elem.rle_len(); + leaf.elem.merge_right(&elem); + Cursor { + leaf: leaf_idx, + offset, + } + } else { + let data = self.alloc_leaf_child(elem, parent_idx.unwrap()); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let ans = data.arena; + update_cache_idx = parent_idx; + parent.children.push(data).unwrap(); + is_full = parent.is_full(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + } + }; + + self.recursive_update_cache( + update_cache_idx, + B::USE_DIFF, + if B::USE_DIFF { + Some(B::new_cache_to_diff(&cache)) + } else { + None + }, + ); + if is_full { + self.split(parent_idx); + } + + ans + } + + pub fn prepend(&mut self, elem: B::Elem) -> Cursor { + let Some(leaf_idx) = self.first_leaf() else { + let parent_idx = self.root; + let data = self.alloc_leaf_child(elem, parent_idx.unwrap()); + let parent = self.in_nodes.get_mut(parent_idx.unwrap()).unwrap(); + let ans = data.arena; + parent.children.push(data).unwrap(); + return Cursor { + leaf: ans.unwrap().into(), + offset: 0, + }; + }; + let leaf = self.leaf_nodes.get_mut(leaf_idx.0).unwrap(); + let parent_idx = leaf.parent(); + let mut is_full = false; + let ans = if elem.can_merge(&leaf.elem) { + leaf.elem.merge_left(&elem); + Cursor { + leaf: leaf_idx, + offset: 0, + } + } else { + let parent_idx = leaf.parent; + let data = self.alloc_leaf_child(elem, parent_idx); + let parent = self.in_nodes.get_mut(parent_idx).unwrap(); + let ans = data.arena; + parent.children.insert(0, data).unwrap(); + is_full = parent.is_full(); + Cursor { + leaf: ans.unwrap().into(), + offset: 0, + } + }; + + self.recursive_update_cache(leaf_idx.into(), B::USE_DIFF, None); + if is_full { + self.split(parent_idx); + } + + ans + } + + /// compare the position of a and b + pub fn compare_pos(&self, a: Cursor, b: Cursor) -> Ordering { + if a.leaf == b.leaf { + return a.offset.cmp(&b.offset); + } + + let leaf_a = self.leaf_nodes.get(a.leaf.0).unwrap(); + let leaf_b = self.leaf_nodes.get(b.leaf.0).unwrap(); + let mut node_a = self.get_internal(leaf_a.parent()); + if leaf_a.parent == leaf_b.parent { + for child in node_a.children.iter() { + if child.arena.unwrap() == a.leaf.0 { + return Ordering::Less; + } + if child.arena.unwrap() == b.leaf.0 { + return Ordering::Greater; + } + } + } + + let mut node_b = self.get_internal(leaf_b.parent()); + while node_a.parent != node_b.parent { + node_a = self.get_internal(node_a.parent.unwrap()); + node_b = self.get_internal(node_b.parent.unwrap()); + } + + node_a.parent_slot.cmp(&node_b.parent_slot) + } + + /// Iterate the caches of previous nodes/elements. + /// This method will visit as less caches as possible. + /// For example, if all nodes in a subtree need to be visited, we will only visit the root cache. + /// + /// f: (node_cache, previous_sibling_elem, (this_elem, offset)) + pub fn visit_previous_caches(&self, cursor: Cursor, mut f: F) + where + F: FnMut(PreviousCache<'_, B>), + { + // the last index of path points to the leaf element + let path = self.get_path(cursor.leaf.into()); + let mut path_index = 0; + let mut child_index = 0; + let mut node = self.get_internal(path[path_index].arena); + 'outer: loop { + if path_index + 1 >= path.len() { + break; + } + + while child_index == path.get(path_index + 1).map(|x| x.arr).unwrap() { + path_index += 1; + if path_index + 1 < path.len() { + node = self.get_internal(path[path_index].arena); + child_index = 0; + } else { + break 'outer; + } + } + + f(PreviousCache::NodeCache( + &node.children[child_index as usize].cache, + )); + child_index += 1; + } + + let node = self.leaf_nodes.get(cursor.leaf.0).unwrap(); + f(PreviousCache::ThisElemAndOffset { + elem: &node.elem, + offset: cursor.offset, + }); + } + + pub fn diagnose_balance(&self) { + let mut size_counter: FxHashMap = Default::default(); + for (_, node) in self.in_nodes.iter() { + *size_counter.entry(node.children.len()).or_default() += 1; + } + dbg!(size_counter); + + let mut size_counter: FxHashMap = Default::default(); + for (_, node) in self.leaf_nodes.iter() { + *size_counter.entry(node.elem.rle_len()).or_default() += 1; + } + dbg!(size_counter); + } + + /// Iterate over the leaf elements in the tree if the filter returns true for all its ancestors' caches, including its own cache. + pub fn iter_with_filter<'a, R: Default + Copy + AddAssign + 'a>( + &'a self, + mut f: impl FnMut(&B::Cache) -> (bool, R) + 'a, + ) -> impl Iterator + '_ { + let mut queue = VecDeque::new(); + queue.push_back((self.root, R::default())); + std::iter::from_fn(move || { + while let Some((node_idx, mut r)) = queue.pop_front() { + match node_idx { + ArenaIndex::Leaf(leaf) => { + let node = self.leaf_nodes.get(leaf).unwrap(); + return Some((r, &node.elem)); + } + ArenaIndex::Internal(idx) => { + let node = self.in_nodes.get(idx).unwrap(); + for child in node.children.iter() { + let (drill, new_r) = f(&child.cache); + if drill { + queue.push_back((child.arena, r)); + } + r += new_r; + } + } + } + } + + None + }) + } + + /// This method allows users to update the caches and the elements with a filter. + /// + /// If `f` returns true for a node, it will drill down into the subtree whose root is the node. + /// + /// It's the caller's responsibility to ensure the invariance of caches being up to date. + pub fn update_cache_and_elem_with_filter<'a>( + &'a mut self, + mut f: impl FnMut(&mut B::Cache) -> bool + 'a, + mut g: impl FnMut(&mut B::Elem) + 'a, + ) { + let mut stack = vec![self.root]; + while let Some(node_idx) = stack.pop() { + match node_idx { + ArenaIndex::Leaf(leaf) => { + let node = self.leaf_nodes.get_mut(leaf).unwrap(); + g(&mut node.elem); + } + ArenaIndex::Internal(idx) => { + let node = self.in_nodes.get_mut(idx).unwrap(); + for child in node.children.iter_mut() { + if f(&mut child.cache) { + stack.push(child.arena); + } + } + } + } + } + } + + pub fn depth(&self) -> usize { + let mut depth = 0; + let mut index = self.root; + let mut node = self.in_nodes.get(index.unwrap_internal()).unwrap(); + loop { + depth += 1; + index = node.children.first().unwrap().arena; + match index { + ArenaIndex::Leaf(_) => return depth, + ArenaIndex::Internal(index) => { + node = self.in_nodes.get(index).unwrap(); + } + } + } + } + + pub fn internal_avg_children_num(&self) -> f64 { + let mut sum = 0; + for (_, node) in self.in_nodes.iter() { + sum += node.children.len(); + } + sum as f64 / self.in_nodes.len() as f64 + } +} + +fn merge_adj(data: &mut Vec) { + // Merge adjacent elements + let mut i = 0; + let last = data.len() - 1; + let mut to_delete_start = 0; + let mut del_len = 0; + while i < last { + if data[i].can_merge(&data[i + 1]) { + let (a, b) = arref::mut_twice(data.as_mut_slice(), i, i + 1).unwrap(); + a.merge_right(b); + if del_len == 0 { + to_delete_start = i + 1; + } + + data.swap(i + 1, to_delete_start + del_len); + del_len += 1; + i += 1; + } + i += 1; + } + + if del_len > 0 { + data.drain(to_delete_start..to_delete_start + del_len); + } +} + +pub enum PreviousCache<'a, B: BTreeTrait> { + NodeCache(&'a B::Cache), + PrevSiblingElem(&'a B::Elem), + ThisElemAndOffset { elem: &'a B::Elem, offset: usize }, +} + +#[inline(always)] +fn add_leaf_dirty_map(leaf: ArenaIndex, dirty_map: &mut LeafDirtyMap, leaf_diff: T) { + dirty_map.insert(leaf, leaf_diff); +} + +impl BTree { + pub fn check(&self) { + // check cache + let mut leaf_level = None; + for (index, node) in self.in_nodes.iter() { + if index != self.root.unwrap() { + assert!(!node.is_empty()); + } + + for (i, child_info) in node.children.iter().enumerate() { + if matches!(child_info.arena, ArenaIndex::Internal(_)) { + assert!(!node.is_child_leaf()); + let child = self.get_internal(child_info.arena); + let mut cache = Default::default(); + child.calc_cache(&mut cache, None); + assert_eq!(child.parent_slot, i as u8); + assert_eq!(child.parent, Some(ArenaIndex::Internal(index))); + assert_eq!( + cache, child_info.cache, + "index={:?} child_index={:?}", + index, child_info.arena + ); + } + } + + if let Some(parent) = node.parent { + let parent = self.get_internal(parent); + assert_eq!( + parent.children[node.parent_slot as usize].arena, + ArenaIndex::Internal(index) + ); + self.get_path(ArenaIndex::Internal(index)); + } else { + assert_eq!(index, self.root.unwrap_internal()) + } + + // if index != self.root.unwrap() { + // assert!(!node.is_lack(), "len={}\n", node.len()); + // } + // + // assert!(!node.is_full(), "len={}", node.len()); + } + + let root = self.get_internal(self.root); + let mut root_cache = Default::default(); + root.calc_cache(&mut root_cache, None); + assert_eq!(&self.root_cache, &root_cache); + + for (leaf_index, leaf_node) in self.leaf_nodes.iter() { + let mut length = 1; + let mut node_idx = leaf_node.parent; + while node_idx != self.root.unwrap() { + let node = self.get_internal(ArenaIndex::Internal(node_idx)); + length += 1; + node_idx = node.parent.unwrap().unwrap(); + } + match leaf_level { + Some(expected) => { + if length != expected { + dbg!(leaf_index, leaf_node); + assert_eq!(length, expected); + } + } + None => { + leaf_level = Some(length); + } + } + + let cache = B::get_elem_cache(&leaf_node.elem); + let parent = self.get_internal(leaf_node.parent()); + assert_eq!( + parent + .children + .iter() + .find(|x| x.arena.unwrap_leaf() == leaf_index) + .unwrap() + .cache, + cache + ); + self.get_path(ArenaIndex::Leaf(leaf_index)); + } + } +} + +impl> FromIterator for BTree { + fn from_iter>(iter: I) -> Self { + let mut tree = Self::new(); + let iter = iter.into_iter(); + let min_size = iter.size_hint().0; + tree.leaf_nodes.reserve(min_size); + let max_child_size = MAX_CHILDREN_NUM - 2; + + struct TempInternalNode { + children: HeaplessVec, MAX_CHILDREN_NUM>, + cache: B::Cache, + arena_index: RawArenaIndex, + } + + let parent_num = (min_size + max_child_size - 1) / max_child_size; + let mut internal_nodes: Vec> = Vec::with_capacity(parent_num); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + + // create all leaf nodes and their parents + for elem in iter { + let parent = match internal_nodes.last_mut() { + Some(last) if last.children.len() < max_child_size => last, + Some(last) => { + // calculate cache + B::calc_cache_internal(&mut last.cache, &last.children); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + internal_nodes.last_mut().unwrap() + } + _ => unreachable!(), + }; + + let leaf = LeafNode { + elem: elem.into(), + parent: parent.arena_index, + }; + + let cache = B::get_elem_cache(&leaf.elem); + let leaf_index = tree.leaf_nodes.insert(leaf); + parent + .children + .push(Child { + arena: ArenaIndex::Leaf(leaf_index), + cache, + }) + .unwrap(); + } + + // recursively create the internal nodes in higher level, until we reach root + while internal_nodes.len() > 1 { + let parent_num = (internal_nodes.len() + max_child_size - 1) / max_child_size; + let children = std::mem::replace(&mut internal_nodes, Vec::with_capacity(parent_num)); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + + let mut parent_slot = 0; + // eprintln!( + // "children.len={} max_child_size={}", + // children.len(), + // max_child_size + // ); + for mut child in children { + let parent = match internal_nodes.last_mut() { + Some(last) if last.children.len() < max_child_size => last, + Some(last) => { + // calculate cache + B::calc_cache_internal(&mut last.cache, &last.children); + let index = tree.in_nodes.insert(Default::default()); + internal_nodes.push(TempInternalNode { + children: Default::default(), + cache: Default::default(), + arena_index: index, + }); + internal_nodes.last_mut().unwrap() + } + _ => unreachable!(), + }; + + B::calc_cache_internal(&mut child.cache, &child.children); + let child_node = tree.in_nodes.get_mut(child.arena_index).unwrap(); + child_node.children = child.children; + child_node.parent = Some(ArenaIndex::Internal(parent.arena_index)); + child_node.parent_slot = parent_slot; + parent_slot = (parent_slot + 1) % (max_child_size as u8); + parent + .children + .push(Child { + arena: ArenaIndex::Internal(child.arena_index), + cache: child.cache, + }) + .unwrap(); + } + + debug_assert_eq!(parent_num, internal_nodes.len()); + } + + debug_assert_eq!(internal_nodes.len(), 1); + let node = internal_nodes.remove(0); + B::calc_cache_internal(&mut tree.root_cache, &node.children); + tree.in_nodes.remove(tree.root.unwrap()); + tree.root = ArenaIndex::Internal(node.arena_index); + let root = tree.root.unwrap(); + tree.in_nodes.get_mut(root).unwrap().children = node.children; + tree + } +} + +struct SplitInfo { + new_pos: Option, + left_neighbour: Option, + parent_idx: RawArenaIndex, + insert_slot: usize, + new_leaf: Option, +} + +impl Default for BTree { + fn default() -> Self { + Self::new() + } +} + +fn delete_range( + arr: &mut heapless::Vec, + range: impl RangeBounds, +) { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => arr.len(), + }; + + if start == end { + return; + } + + if end - start == 1 { + arr.remove(start); + return; + } + + let mut ans = heapless::Vec::from_slice(&arr[..start]).unwrap(); + ans.extend_from_slice(&arr[end..]).unwrap(); + *arr = ans; +} diff --git a/vendor/generic-btree/src/rle.rs b/vendor/generic-btree/src/rle.rs new file mode 100644 index 00000000..52ec0a2f --- /dev/null +++ b/vendor/generic-btree/src/rle.rs @@ -0,0 +1,141 @@ +use core::ops::RangeBounds; +use std::ops::Range; + +/// For better performance, it's advised to impl split +pub trait Sliceable: HasLength + Sized { + #[must_use] + fn _slice(&self, range: Range) -> Self; + + #[must_use] + #[inline(always)] + fn slice(&self, range: impl RangeBounds) -> Self { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.rle_len(), + }; + + self._slice(start..end) + } + + /// slice in-place + #[inline(always)] + fn slice_(&mut self, range: impl RangeBounds) { + *self = self.slice(range); + } + + #[must_use] + fn split(&mut self, pos: usize) -> Self { + let right = self.slice(pos..); + self.slice_(..pos); + right + } + + /// Update the slice in the given range. + /// This method may split `self` into two or three parts. + /// If so, it will make `self` the leftmost part and return the next split parts. + /// + /// # Example + /// + /// If `self.rle_len() == 10`, `self.update(1..5)` will split self into three parts and update the middle part. + /// It returns the middle and the right part. + fn update_with_split( + &mut self, + range: impl RangeBounds, + f: impl FnOnce(&mut Self), + ) -> (Option, Option) { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.rle_len(), + }; + + if start >= end { + return (None, None); + } + + match (start == 0, end == self.rle_len()) { + (true, true) => { + f(self); + (None, None) + } + (true, false) => { + let right = self.split(end); + f(self); + (Some(right), None) + } + (false, true) => { + let mut right = self.split(start); + f(&mut right); + (Some(right), None) + } + (false, false) => { + let right = self.split(end); + let mut middle = self.split(start); + f(&mut middle); + (Some(middle), Some(right)) + } + } + } +} + +pub trait Mergeable { + /// Whether self can merge rhs with self on the left. + /// + /// Note: This is not symmetric. + fn can_merge(&self, rhs: &Self) -> bool; + fn merge_right(&mut self, rhs: &Self); + fn merge_left(&mut self, left: &Self); +} + +pub trait HasLength { + fn rle_len(&self) -> usize; +} + +pub trait TryInsert { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized; +} + +pub trait CanRemove { + fn can_remove(&self) -> bool; +} + +impl CanRemove for () { + fn can_remove(&self) -> bool { + true + } +} +impl CanRemove for usize { + fn can_remove(&self) -> bool { + *self == 0 + } +} +impl CanRemove for isize { + fn can_remove(&self) -> bool { + *self == 0 + } +} +impl CanRemove for u32 { + fn can_remove(&self) -> bool { + *self == 0 + } +} +impl CanRemove for i32 { + fn can_remove(&self) -> bool { + *self == 0 + } +} diff --git a/vendor/loro-internal/.cargo_vcs_info.json b/vendor/loro-internal/.cargo_vcs_info.json new file mode 100644 index 00000000..11fdc879 --- /dev/null +++ b/vendor/loro-internal/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "34cbaba63c37c6758ce183888424bdc890a25367" + }, + "path_in_vcs": "crates/loro-internal" +} \ No newline at end of file diff --git a/vendor/loro-internal/.gitignore b/vendor/loro-internal/.gitignore new file mode 100644 index 00000000..1b5d0435 --- /dev/null +++ b/vendor/loro-internal/.gitignore @@ -0,0 +1 @@ +*dhat-heap.json diff --git a/vendor/loro-internal/Cargo.lock b/vendor/loro-internal/Cargo.lock new file mode 100644 index 00000000..8bb5e8d4 --- /dev/null +++ b/vendor/loro-internal/Cargo.lock @@ -0,0 +1,2123 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.3", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "append-only-bytes" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac436601d6bdde674a0d7fb593e829ffe7b3387c351b356dd20e2d40f5bf3ee5" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arref" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ccd462b64c3c72f1be8305905a85d85403d768e8690c9b8bd3b9009a5761679" + +[[package]] +name = "atomic-polyfill" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4" +dependencies = [ + "critical-section", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" + +[[package]] +name = "bitmaps" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" +dependencies = [ + "typenum", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.16", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.106", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "ensure-cov" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33753185802e107b8fa907192af1f0eca13b1fb33327a59266d650fef29b2b4e" + +[[package]] +name = "enum-as-inner" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generator" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "generic-btree" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c1bce85c110ab718fd139e0cc89c51b63bd647b14a767e24bdfc77c83df79b" +dependencies = [ + "arref", + "heapless 0.9.1", + "itertools 0.11.0", + "loro-thunderdome", + "proc-macro2", + "rustc-hash", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.4+wasi-0.2.4", +] + +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hash32" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "heapless" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" +dependencies = [ + "atomic-polyfill", + "hash32 0.2.1", + "rustc_version", + "serde", + "spin", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heapless" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1edcd5a338e64688fbdcb7531a846cfd3476a54784dcb918a0844682bc7ada5" +dependencies = [ + "hash32 0.3.1", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "im" +version = "15.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +dependencies = [ + "bitmaps", + "rand_core", + "rand_xoshiro", + "serde", + "sized-chunks", + "typenum", + "version_check", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "libc" +version = "0.2.175" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "loro-common" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b51042936156a1d537df8a38938ec8ed24b45de14530af933c1d02de39c9e056" +dependencies = [ + "arbitrary", + "enum-as-inner 0.6.1", + "js-sys", + "leb128", + "loro-rle", + "nonmax", + "rustc-hash", + "serde", + "serde_columnar", + "thiserror 1.0.69", + "tracing", + "wasm-bindgen", +] + +[[package]] +name = "loro-delta" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f381e7fb5d4bf5605b4f4a9241f3adf61403ded978d757552004c8302cccbb" +dependencies = [ + "arrayvec", + "enum-as-inner 0.5.1", + "generic-btree", + "heapless 0.8.0", +] + +[[package]] +name = "loro-internal" +version = "1.13.1" +dependencies = [ + "append-only-bytes", + "arbitrary", + "arref", + "bytes", + "criterion", + "ctor", + "either", + "ensure-cov", + "enum-as-inner 0.6.1", + "enum_dispatch", + "generic-btree", + "getrandom 0.2.16", + "im", + "itertools 0.12.1", + "js-sys", + "leb128", + "loom", + "loro-common", + "loro-delta", + "loro-kv-store", + "loro-rle", + "loro_fractional_index", + "md5", + "miniz_oxide", + "nonmax", + "num", + "num-traits", + "once_cell", + "parking_lot", + "pest", + "pest_derive", + "postcard", + "pretty_assertions", + "rand", + "rustc-hash", + "serde", + "serde_columnar", + "serde_json", + "smallvec", + "tabled", + "thiserror 1.0.69", + "thread_local", + "tracing", + "wasm-bindgen", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "loro-kv-store" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950cc8bcc64dcff536e949fbc56ccc3d0759646fa441c7f76b3cd7c3eafa2096" +dependencies = [ + "bytes", + "ensure-cov", + "loro-common", + "lz4_flex", + "once_cell", + "quick_cache", + "rustc-hash", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "loro-rle" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76400c3eea6bb39b013406acce964a8db39311534e308286c8d8721baba8ee20" +dependencies = [ + "append-only-bytes", + "num", + "smallvec", +] + +[[package]] +name = "loro-thunderdome" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3d053a135388e6b1df14e8af1212af5064746e9b87a06a345a7a779ee9695a" + +[[package]] +name = "loro_fractional_index" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aca7180674d0273ddf37049a5efcde4547fd5330d24abb7519bb9d9eb6780d5b" +dependencies = [ + "once_cell", + "rand", + "serde", +] + +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" + +[[package]] +name = "nu-ansi-term" +version = "0.50.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "papergrid" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1526bb6aa9f10ec339fb10360f22c57edf81d5678d0278e93bc12a47ffbe4b01" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "pest" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "pest_meta" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +dependencies = [ + "pest", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "heapless 0.7.17", + "serde", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick_cache" +version = "0.6.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ad6644cb07b7f3488b9f3d2fde3b4c0a7fa367cafefb39dff93a659f76eb786" +dependencies = [ + "ahash", + "equivalent", + "hashbrown", + "parking_lot", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_columnar" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a16e404f17b16d0273460350e29b02d76ba0d70f34afdc9a4fa034c97d6c6eb" +dependencies = [ + "itertools 0.11.0", + "postcard", + "serde", + "serde_columnar_derive", + "thiserror 1.0.69", +] + +[[package]] +name = "serde_columnar_derive" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45958fce4903f67e871fbf15ac78e289269b21ebd357d6fecacdba233629112e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_core" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.226" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "sized-chunks" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" +dependencies = [ + "bitmaps", + "typenum", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tabled" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c3ee73732ffceaea7b8f6b719ce3bb17f253fa27461ffeaf568ebd0cdb4b85" +dependencies = [ + "papergrid", + "tabled_derive", + "unicode-width", +] + +[[package]] +name = "tabled_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beca1b4eaceb4f2755df858b88d9b9315b7ccfd1ffd0d7a48a52602301f01a57" +dependencies = [ + "heck 0.4.1", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" +dependencies = [ + "thiserror-impl 2.0.16", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.4+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88a5f4a424faf49c3c2c344f166f0662341d470ea185e939657aaff130f0ec4a" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.3", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +dependencies = [ + "windows-link 0.1.3", + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "wit-bindgen" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/vendor/loro-internal/Cargo.toml b/vendor/loro-internal/Cargo.toml new file mode 100644 index 00000000..f22f724e --- /dev/null +++ b/vendor/loro-internal/Cargo.toml @@ -0,0 +1,365 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "loro-internal" +version = "1.13.1" +authors = [ + "Zixuan Chen", + "Liang Zhao", +] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Loro internal library. Do not use it directly as it's not stable." +homepage = "https://loro.dev" +documentation = "https://docs.rs/loro/" +readme = "README.md" +keywords = [ + "crdt", + "local-first", +] +categories = [] +license = "MIT" +repository = "https://github.com/loro-dev/loro/" + +[features] +counter = ["loro-common/counter"] +jsonpath = [] +logging = ["loro-common/logging"] +test_utils = [ + "arbitrary", + "tabled", +] +wasm = [ + "wasm-bindgen", + "js-sys", + "loro-common/wasm", +] + +[lib] +name = "loro_internal" +path = "src/lib.rs" +doctest = false +bench = false + +[[example]] +name = "automerge_x100" +path = "examples/automerge_x100.rs" + +[[example]] +name = "encoding" +path = "examples/encoding.rs" + +[[example]] +name = "encoding_parallel" +path = "examples/encoding_parallel.rs" + +[[example]] +name = "encoding_refactored" +path = "examples/encoding_refactored.rs" + +[[example]] +name = "event" +path = "examples/event.rs" + +[[example]] +name = "list" +path = "examples/list.rs" + +[[example]] +name = "many_actors" +path = "examples/many_actors.rs" + +[[example]] +name = "mem" +path = "examples/mem.rs" + +[[example]] +name = "obs" +path = "examples/obs.rs" + +[[example]] +name = "pending" +path = "examples/pending.rs" + +[[example]] +name = "state_size" +path = "examples/state_size.rs" + +[[example]] +name = "tree" +path = "examples/tree.rs" + +[[test]] +name = "autocommit" +path = "tests/autocommit.rs" + +[[test]] +name = "contracts" +path = "tests/contracts.rs" + +[[test]] +name = "ephemeral_store" +path = "tests/ephemeral_store.rs" + +[[test]] +name = "fugue" +path = "tests/fugue.rs" + +[[test]] +name = "mergeable_cid_encoding" +path = "tests/mergeable_cid_encoding.rs" + +[[test]] +name = "mergeable_container" +path = "tests/mergeable_container.rs" + +[[test]] +name = "richtext" +path = "tests/richtext.rs" + +[[test]] +name = "test" +path = "tests/test.rs" + +[[test]] +name = "tree" +path = "tests/tree.rs" + +[[test]] +name = "undo" +path = "tests/undo.rs" + +[[bench]] +name = "encode" +path = "benches/encode.rs" +harness = false + +[[bench]] +name = "event" +path = "benches/event.rs" +harness = false + +[[bench]] +name = "jsonpath" +path = "benches/jsonpath.rs" +harness = false + +[[bench]] +name = "list" +path = "benches/list.rs" +harness = false + +[[bench]] +name = "map" +path = "benches/map.rs" +harness = false + +[[bench]] +name = "pending" +path = "benches/pending.rs" +harness = false + +[[bench]] +name = "text_r" +path = "benches/text_r.rs" +harness = false + +[[bench]] +name = "tree" +path = "benches/tree.rs" +harness = false + +[dependencies.hotpath] +version = "0.16.1" + +[dependencies.append-only-bytes] +version = "0.1.12" +features = ["u32_range"] + +[dependencies.arbitrary] +version = "1" +optional = true + +[dependencies.arref] +version = "0.1.0" + +[dependencies.bytes] +version = "1" + +[dependencies.either] +version = "1" + +[dependencies.ensure-cov] +version = "0.1.0" + +[dependencies.enum-as-inner] +version = "0.6.0" + +[dependencies.enum_dispatch] +version = "0.3.11" + +[dependencies.fractional_index] +version = "1.13.0" +features = ["serde"] +package = "loro_fractional_index" + +[dependencies.generic-btree] +version = "^0.10.7" + +[dependencies.getrandom] +version = "0.2.15" + +[dependencies.im] +version = "15.1.0" +features = ["serde"] + +[dependencies.itertools] +version = "0.12.1" + +[dependencies.js-sys] +version = "0.3.60" +optional = true + +[dependencies.leb128] +version = "0.2.5" + +[dependencies.loro-common] +version = "1.13.1" + +[dependencies.loro-delta] +version = "1.13.0" +package = "loro-delta" + +[dependencies.loro-kv-store] +version = "1.13.1" + +[dependencies.md5] +version = "0.7.0" + +[dependencies.nonmax] +version = "0.5.5" + +[dependencies.num] +version = "0.4.0" + +[dependencies.num-traits] +version = "0.2" + +[dependencies.once_cell] +version = "1.18.0" + +[dependencies.parking_lot] +version = "0.12.5" + +[dependencies.pest] +version = "2.8.3" + +[dependencies.pest_derive] +version = "2.8.3" + +[dependencies.postcard] +version = "1" +features = ["use-std"] + +[dependencies.pretty_assertions] +version = "1.4.1" + +[dependencies.rand] +version = "0.8.5" + +[dependencies.rle] +version = "1.6.0" +package = "loro-rle" + +[dependencies.rustc-hash] +version = "2.1.1" + +[dependencies.serde] +version = "1" + +[dependencies.serde_columnar] +version = "0.3.14" + +[dependencies.serde_json] +version = "1.0" + +[dependencies.smallvec] +version = "1.8.0" +features = ["serde"] + +[dependencies.tabled] +version = "0.10.0" +optional = true + +[dependencies.thiserror] +version = "1" + +[dependencies.thread_local] +version = "1.1.8" + +[dependencies.tracing] +version = "0.1" + +[dependencies.wasm-bindgen] +version = "=0.2.100" +optional = true + +[dependencies.xxhash-rust] +version = "0.8.12" +features = ["xxh32"] + +[dev-dependencies.arbitrary] +version = "1" + +[dev-dependencies.criterion] +version = "0.7.0" +features = ["html_reports"] + +[dev-dependencies.ctor] +version = "0.1.23" + +[dev-dependencies.miniz_oxide] +version = "0.7.1" + +[dev-dependencies.rand] +version = "0.8.5" + +[dev-dependencies.tabled] +version = "0.10.0" + +[dev-dependencies.zstd] +version = "0.13.0" + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten"), not(target_os = "wasi")))'.dependencies.getrandom] +version = "0.2.15" +features = ["js"] + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten"), not(target_os = "wasi")))'.dependencies.wasm-bindgen] +version = "0.2.100" + +[target."cfg(loom)".dependencies.loom] +version = "0.7" +features = ["checkpoint"] + +[lints.rust.unexpected_cfgs] +level = "warn" +priority = 0 +check-cfg = ["cfg(loom)"] + +[package.metadata.cargo-machete] +# Vendored crate: unused-in-our-build deps kept to minimize diff vs upstream. +ignored = ["hotpath", "nonmax", "tabled"] diff --git a/vendor/loro-internal/Cargo.toml.orig b/vendor/loro-internal/Cargo.toml.orig new file mode 100644 index 00000000..60680657 --- /dev/null +++ b/vendor/loro-internal/Cargo.toml.orig @@ -0,0 +1,130 @@ +[package] +name = "loro-internal" +version = "1.13.1" +edition = "2021" +license = "MIT" +description = "Loro internal library. Do not use it directly as it's not stable." +documentation = "https://docs.rs/loro/" +homepage = "https://loro.dev" +repository = "https://github.com/loro-dev/loro/" +authors = ["Zixuan Chen", "Liang Zhao"] +categories = [] +keywords = ["crdt", "local-first"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +generic-btree = { version = "^0.10.7" } +smallvec = { workspace = true } +loro-delta = { path = "../delta", version = "1.13.0", package = "loro-delta" } +rle = { path = "../rle", version = "1.6.0", package = "loro-rle" } +loro-common = { path = "../loro-common", version = "1.13.1" } +fractional_index = { path = "../fractional_index", features = [ + "serde", +], version = "1.13.0", package = "loro_fractional_index" } +loro-kv-store = { path = "../kv-store", version = "1.13.1" } +rustc-hash = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } +enum-as-inner = { workspace = true } +serde_json = { workspace = true } +bytes = { workspace = true } +xxhash-rust = { workspace = true } +serde_columnar = { workspace = true } +itertools = { workspace = true } +enum_dispatch = { workspace = true } +once_cell = { workspace = true } +arbitrary = { version = "1", optional = true } +postcard = { version = "1", features = ["use-std"] } +append-only-bytes = { version = "0.1.12", features = ["u32_range"] } +im = { version = "15.1.0", features = ["serde"] } +tabled = { version = "0.10.0", optional = true } +wasm-bindgen = { version = "=0.2.100", optional = true } +js-sys = { version = "0.3.60", optional = true } +num = "0.4.0" +rand = { version = "0.8.5" } +getrandom = "0.2.15" +leb128 = "0.2.5" +num-traits = "0.2" +either = "1" +md5 = "0.7.0" +arref = "0.1.0" +tracing = { version = "0.1" } +nonmax = "0.5.5" +ensure-cov = { workspace = true } +pretty_assertions = "1.4.1" +thread_local = "1.1.8" +parking_lot = "0.12.5" +pest = "2.8.3" +pest_derive = "2.8.3" + +[target.'cfg(loom)'.dependencies] +loom = { version = "0.7", features = ["checkpoint"] } + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] } + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten"), not(target_os = "wasi")))'.dependencies.getrandom] +version = "0.2.15" +features = ["js"] + +[target.'cfg(all(target_arch = "wasm32", not(target_os = "emscripten"), not(target_os = "wasi")))'.dependencies] +wasm-bindgen = "0.2.100" + +[dev-dependencies] +arbitrary = { version = "1" } +miniz_oxide = "0.7.1" +rand = { version = "0.8.5" } +tabled = "0.10.0" +ctor = "0.1.23" +bench-utils = { path = "../bench-utils" } +dev-utils = { path = "../dev-utils" } +zstd = "0.13.0" +criterion = { version = "0.7.0", features = ["html_reports"] } + +# See https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html +[lib] +doctest = false +bench = false + +[features] +# "wasm" will use utf-16 index by default when using text related stuff; But it'll use utf-8 encoding internally. +wasm = ["wasm-bindgen", "js-sys", "loro-common/wasm"] +# whether to use list slice instead of raw str in text container +test_utils = ["arbitrary", "tabled"] +# whether enable the counter container +counter = ["loro-common/counter"] +logging = ["loro-common/logging"] +jsonpath = [] + +[[bench]] +name = "text_r" +harness = false + +[[bench]] +name = "list" +harness = false + +[[bench]] +name = "encode" +harness = false + +[[bench]] +name = "pending" +harness = false + +[[bench]] +name = "event" +harness = false + +[[bench]] +name = "map" +harness = false + +[[bench]] +name = "tree" +harness = false + +[[bench]] +name = "jsonpath" +harness = false diff --git a/vendor/loro-internal/Encoding.md b/vendor/loro-internal/Encoding.md new file mode 100644 index 00000000..2815951a --- /dev/null +++ b/vendor/loro-internal/Encoding.md @@ -0,0 +1,25 @@ +# Encoding Schema + +## Header + +The header has 22 bytes. + +- (0-4 bytes) Magic Bytes: The encoding starts with `loro` as magic bytes. +- (4-20 bytes) Checksum: MD5/xxhash checksum of the encoded data, including the header starting from 20th bytes. The checksum is encoded as a 16-byte array (for xxhash, the first 12 bytes are padded with 0). The `checksum` and `magic bytes` fields are trimmed when calculating the checksum. +- (20-21 bytes) Encoding Method (2 bytes, big endian): Multiple encoding methods are available for a specific encoding version. + +## Encode Mode: Updates + +In this approach, only ops, specifically their historical record, are encoded, while document states are excluded. + +Like Automerge's format, we employ columnar encoding for operations and changes. + +Previously, operations were ordered by their Operation ID (OpId) before columnar encoding. However, sorting operations based on their respective containers initially enhance compression potential. + +## Encode Mode: Snapshot + +This mode simultaneously captures document state and historical data. Upon importing a snapshot into a new document, initialization occurs directly from the snapshot, bypassing the need for CRDT-based recalculations. + +Unlike previous snapshot encoding methods, the current binary output in snapshot mode is compatible with the updates mode. This enhances the efficiency of importing snapshots into non-empty documents, where initialization via snapshot is infeasible. + +Additionally, when feasible, we leverage the sequence of operations to construct state snapshots. In CRDTs, deducing the specific ops constituting the current container state is feasible. These ops are tagged in relation to the container, facilitating direct state reconstruction from them. This approach, pioneered by Automerge, significantly improves compression efficiency. diff --git a/vendor/loro-internal/README.md b/vendor/loro-internal/README.md new file mode 100644 index 00000000..d66d799e --- /dev/null +++ b/vendor/loro-internal/README.md @@ -0,0 +1,5 @@ +# loro-internal + +This is the internal implementation of Loro. +Its interface subject to change. You should not use it as dependency directly. +Consider use `loro` crate instead. diff --git a/vendor/loro-internal/VERSION b/vendor/loro-internal/VERSION new file mode 100644 index 00000000..da38e07b --- /dev/null +++ b/vendor/loro-internal/VERSION @@ -0,0 +1 @@ +1.13.1 \ No newline at end of file diff --git a/vendor/loro-internal/benches/automerge-paper.json.gz b/vendor/loro-internal/benches/automerge-paper.json.gz new file mode 100644 index 00000000..aa7f67ec Binary files /dev/null and b/vendor/loro-internal/benches/automerge-paper.json.gz differ diff --git a/vendor/loro-internal/benches/encode.rs b/vendor/loro-internal/benches/encode.rs new file mode 100644 index 00000000..36e5da1f --- /dev/null +++ b/vendor/loro-internal/benches/encode.rs @@ -0,0 +1,394 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +#[cfg(feature = "test_utils")] +mod sync { + + use super::*; + use bench_utils::{get_automerge_actions, TextAction}; + use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc}; + + pub fn b4(c: &mut Criterion) { + let actions = get_automerge_actions(); + let mut b = c.benchmark_group("encode_with_sync"); + b.sample_size(10); + b.bench_function("update", |b| { + b.iter(|| { + let c1 = LoroDoc::new(); + c1.set_peer_id(1).unwrap(); + let c2 = LoroDoc::new(); + c2.set_peer_id(2).unwrap(); + let t1 = c1.get_text("text"); + let t2 = c2.get_text("text"); + for (i, action) in actions.iter().enumerate() { + if i > 2000 { + break; + } + let TextAction { pos, ins, del } = action; + if i % 2 == 0 { + let mut txn = c1.txn().unwrap(); + t1.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + t1.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + + let update = c1.export(ExportMode::updates(&c2.oplog_vv())).unwrap(); + c2.import(&update).unwrap(); + } else { + let mut txn = c2.txn().unwrap(); + t2.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + t2.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let update = c2.export(ExportMode::updates(&c1.oplog_vv())).unwrap(); + c1.import(&update).unwrap(); + } + } + }) + }); + } +} +#[cfg(feature = "test_utils")] +mod run { + use super::*; + use bench_utils::TextAction; + use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc}; + use std::hint::black_box; + + pub fn b4(c: &mut Criterion) { + let loro = LoroDoc::default(); + let mut ran = false; + let mut ensure_ran = || { + if !ran { + let actions = bench_utils::get_automerge_actions(); + let text = loro.get_text("text"); + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + ran = true; + } + }; + + let mut b = c.benchmark_group("encode"); + b.sample_size(10); + b.bench_function("B4_encode_updates", |b| { + ensure_ran(); + b.iter(|| { + let _ = loro.export(ExportMode::all_updates()).unwrap(); + }) + }); + b.bench_function("B4_decode_updates", |b| { + ensure_ran(); + let buf = loro.export(ExportMode::all_updates()).unwrap(); + + b.iter(|| { + let store2 = LoroDoc::default(); + store2.import(&buf).unwrap(); + }) + }); + b.bench_function("B4_decode_updates detached mode", |b| { + ensure_ran(); + let buf = loro.export(ExportMode::all_updates()).unwrap(); + + b.iter(|| { + let store2 = LoroDoc::default(); + store2.detach(); + store2.import(&buf).unwrap(); + }) + }); + b.bench_function("B4_encode_snapshot", |b| { + ensure_ran(); + b.iter(|| { + let _ = loro.export(ExportMode::Snapshot).unwrap(); + }) + }); + b.bench_function("B4_decode_snapshot", |b| { + ensure_ran(); + let buf = loro.export(ExportMode::Snapshot).unwrap(); + b.iter(|| { + let store2 = LoroDoc::default(); + store2.import(&buf).unwrap(); + }) + }); + + b.bench_function("B4_encode_json_update", |b| { + ensure_ran(); + b.iter(|| { + let _ = loro.export_json_updates(&Default::default(), &loro.oplog_vv(), true); + }) + }); + b.bench_function("B4_decode_json_update", |b| { + ensure_ran(); + let json = loro.export_json_updates(&Default::default(), &loro.oplog_vv(), true); + b.iter(|| { + let store2 = LoroDoc::default(); + store2.import_json_updates(json.clone()).unwrap(); + }) + }); + } + + fn build_issue_992_source_doc(paragraphs: usize) -> LoroDoc { + let doc = LoroDoc::new(); + doc.set_peer_id(1).unwrap(); + let text = doc.get_text("codemirror"); + let mut chunks = Vec::with_capacity(paragraphs); + for i in 0..paragraphs { + chunks.push(format!( + "# Section {i}\nParagraph {i} line A with repeated markdown content.\nParagraph {i} line B with **bold** and _italic_ markers.\n\n" + )); + } + text.insert(0, &chunks.concat(), PosType::Unicode).unwrap(); + doc.commit_then_renew(); + + for i in 0..6 { + text.insert( + text.len_unicode(), + &format!("Tail block {i}: {}\n", "x".repeat(2048)), + PosType::Unicode, + ) + .unwrap(); + doc.commit_then_renew(); + } + + for i in 0..3 { + let current = text.to_string(); + let next = current.replace(&format!("Section {i}"), &format!("Section {i} updated")) + + &format!("\nreplace-round-{i}:{}", "y".repeat(1024)); + text.delete(0, text.len_unicode(), PosType::Unicode) + .unwrap(); + text.insert(0, &next, PosType::Unicode).unwrap(); + doc.commit_then_renew(); + } + + doc + } + + fn clone_doc(doc: &LoroDoc) -> LoroDoc { + LoroDoc::from_snapshot(&doc.export(ExportMode::Snapshot).unwrap()).unwrap() + } + + fn build_issue_992_shallow_tail_doc(paragraphs: usize, tail_rounds: usize) -> LoroDoc { + let source = build_issue_992_source_doc(paragraphs); + let source = clone_doc(&source); + let shallow_bytes = source + .export(ExportMode::shallow_snapshot(&source.oplog_frontiers())) + .unwrap(); + let shallow_doc = LoroDoc::from_snapshot(&shallow_bytes).unwrap(); + let text = shallow_doc.get_text("codemirror"); + + for i in 0..tail_rounds { + let current = text.to_string(); + let next = current.replace(&format!("Section {i}"), &format!("Section {i} tail-{i}")) + + &format!("\npost-shallow-tail-{i}:{}", "z".repeat(4096)); + text.delete(0, text.len_unicode(), PosType::Unicode) + .unwrap(); + text.insert(0, &next, PosType::Unicode).unwrap(); + shallow_doc.commit_then_renew(); + } + + shallow_doc + } + + pub fn issue_992_shallow_snapshot_tail(c: &mut Criterion) { + let mut b = c.benchmark_group("encode_regression"); + b.sample_size(10); + + // This mirrors https://github.com/loro-dev/loro/issues/992: import a shallow + // snapshot, perform a few full-document text replacements, then export snapshot. + // The fixture is intentionally below the original report size so a regression is + // visible without making the default benchmark prohibitively slow. + let shallow_tail_doc = build_issue_992_shallow_tail_doc(1000, 2); + b.bench_function("issue_992_shallow_tail_snapshot_export", |b| { + b.iter(|| { + black_box(shallow_tail_doc.export(ExportMode::Snapshot).unwrap().len()); + }) + }); + } +} + +mod import { + use criterion::{BatchSize, Criterion}; + use loro_common::LoroValue; + use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc}; + + #[allow(dead_code)] + pub fn causal_iter(c: &mut Criterion) { + let mut b = c.benchmark_group("causal_iter"); + b.sample_size(10); + b.bench_function("parallel_500", |b| { + b.iter(|| { + let c1 = LoroDoc::new(); + c1.set_peer_id(1).unwrap(); + let c2 = LoroDoc::new(); + c1.set_peer_id(2).unwrap(); + + let text1 = c1.get_text("text"); + let text2 = c2.get_text("text"); + for _ in 0..500 { + text1 + .insert_with_txn(&mut c1.txn().unwrap(), 0, "1", PosType::Unicode) + .unwrap(); + text2 + .insert_with_txn(&mut c2.txn().unwrap(), 0, "2", PosType::Unicode) + .unwrap(); + } + + let updates = c2.export(ExportMode::updates(&c1.oplog_vv())).unwrap(); + c1.import(&updates).unwrap() + }) + }); + } + + struct BinaryImportFixture { + base_update: Vec, + incremental_update: Vec, + } + + fn text_split_fixture(fragments: usize) -> BinaryImportFixture { + const CHUNK_LEN: usize = 256; + const PEER_A: u64 = 1; + const PEER_B: u64 = 2; + + let doc_len = CHUNK_LEN * fragments; + let doc_a = LoroDoc::new(); + doc_a.set_peer_id(PEER_A).unwrap(); + let text_a = doc_a.get_text("text"); + let mut txn = doc_a.txn().unwrap(); + text_a + .insert_with_txn(&mut txn, 0, &"a".repeat(doc_len), PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let base_update = doc_a.export(ExportMode::all_updates()).unwrap(); + + let doc_b = LoroDoc::new(); + doc_b.set_peer_id(PEER_B).unwrap(); + let text_b = doc_b.get_text("text"); + doc_b.import(&base_update).unwrap(); + let base_vv = doc_b.oplog_vv(); + let mut txn = doc_b.txn().unwrap(); + for i in 0..(fragments - 1) { + let pos = (i + 1) * CHUNK_LEN + i; + text_b + .insert_with_txn(&mut txn, pos, "x", PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + let incremental_update = doc_b.export(ExportMode::updates(&base_vv)).unwrap(); + + BinaryImportFixture { + base_update, + incremental_update, + } + } + + fn list_diff_fixture(items: usize, inserts: usize) -> BinaryImportFixture { + const PEER_A: u64 = 11; + const PEER_B: u64 = 12; + + let doc_a = LoroDoc::new(); + doc_a.set_peer_id(PEER_A).unwrap(); + let list_a = doc_a.get_list("list"); + let mut txn = doc_a.txn().unwrap(); + for i in 0..items { + list_a + .insert_with_txn(&mut txn, i, LoroValue::I64(i as i64)) + .unwrap(); + } + txn.commit().unwrap(); + let base_update = doc_a.export(ExportMode::all_updates()).unwrap(); + + let doc_b = LoroDoc::new(); + doc_b.set_peer_id(PEER_B).unwrap(); + let list_b = doc_b.get_list("list"); + doc_b.import(&base_update).unwrap(); + let base_vv = doc_b.oplog_vv(); + let mut txn = doc_b.txn().unwrap(); + for i in 0..inserts { + let len = items + i; + let pos = (i * 7) % len; + list_b + .insert_with_txn(&mut txn, pos, LoroValue::I64(-((i as i64) + 1))) + .unwrap(); + } + txn.commit().unwrap(); + let incremental_update = doc_b.export(ExportMode::updates(&base_vv)).unwrap(); + + BinaryImportFixture { + base_update, + incremental_update, + } + } + + fn import_attached(fixture: &BinaryImportFixture) { + let doc = LoroDoc::new(); + doc.import(&fixture.base_update).unwrap(); + doc.import(&fixture.incremental_update).unwrap(); + } + + fn import_detached(fixture: &BinaryImportFixture) { + let doc = LoroDoc::new(); + doc.import(&fixture.base_update).unwrap(); + doc.detach(); + doc.import(&fixture.incremental_update).unwrap(); + } + + fn checkout_after_detached_import(fixture: &BinaryImportFixture) { + let doc = LoroDoc::new(); + doc.import(&fixture.base_update).unwrap(); + doc.detach(); + doc.import(&fixture.incremental_update).unwrap(); + doc.checkout_to_latest(); + } + + #[allow(dead_code)] + pub fn import_regression(c: &mut Criterion) { + let text_fixture = text_split_fixture(1024); + let list_fixture = list_diff_fixture(4096, 1024); + + let mut b = c.benchmark_group("import_regression"); + b.sample_size(10); + + b.bench_function("text_split_attached_import_1024", |b| { + b.iter_batched(|| &text_fixture, import_attached, BatchSize::SmallInput) + }); + b.bench_function("text_split_detached_import_1024", |b| { + b.iter_batched(|| &text_fixture, import_detached, BatchSize::SmallInput) + }); + b.bench_function("text_split_checkout_1024", |b| { + b.iter_batched( + || &text_fixture, + checkout_after_detached_import, + BatchSize::SmallInput, + ) + }); + b.bench_function("list_attached_import_4096x1024", |b| { + b.iter_batched(|| &list_fixture, import_attached, BatchSize::SmallInput) + }); + b.bench_function("list_checkout_4096x1024", |b| { + b.iter_batched( + || &list_fixture, + checkout_after_detached_import, + BatchSize::SmallInput, + ) + }); + } +} + +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!( + benches, + run::b4, + run::issue_992_shallow_snapshot_tail, + sync::b4, + import::causal_iter, + import::import_regression +); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/event.rs b/vendor/loro-internal/benches/event.rs new file mode 100644 index 00000000..892a191c --- /dev/null +++ b/vendor/loro-internal/benches/event.rs @@ -0,0 +1,48 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +#[cfg(feature = "test_utils")] +mod event { + use super::*; + + use loro_internal::{ListHandler, LoroDoc}; + use std::sync::Arc; + + fn create_sub_container(handler: ListHandler, children_num: usize) -> Vec { + let mut ans = vec![]; + for idx in 0..children_num { + let child_handler = handler + .insert_container(idx, ListHandler::new_detached()) + .unwrap(); + ans.push(child_handler); + } + ans + } + + pub fn resolved_container(c: &mut Criterion) { + let mut b = c.benchmark_group("resolved"); + b.sample_size(10); + b.bench_function("subContainer in event", |b| { + let children_num = 80; + let deep = 3; + b.iter(|| { + let loro = LoroDoc::default(); + loro.start_auto_commit(); + let _g = loro.subscribe_root(Arc::new(|_e| {})); + let mut handlers = vec![loro.get_list("list")]; + for _ in 0..deep { + handlers = handlers + .into_iter() + .flat_map(|h| create_sub_container(h, children_num)) + .collect(); + } + }) + }); + } +} + +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!(benches, event::resolved_container); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/jsonpath.rs b/vendor/loro-internal/benches/jsonpath.rs new file mode 100644 index 00000000..23696b06 --- /dev/null +++ b/vendor/loro-internal/benches/jsonpath.rs @@ -0,0 +1,317 @@ +use criterion::{criterion_group, criterion_main, Criterion}; + +#[cfg(feature = "test_utils")] +mod jsonpath { + use criterion::Criterion; + use loro_internal::{ListHandler, LoroDoc, MapHandler}; + use std::hint::black_box; + + // Sizes to test: small, medium, large + const SIZES: [usize; 3] = [100, 500, 2000]; + + /// Setup a large document with: + /// - books[0..size] with title, author, price, available + /// - '1984' by George Orwell every ~100 items + /// - Nested structure under $.catalog.fiction.books for recursive tests + fn setup_large_doc(size: usize) -> LoroDoc { + let doc = LoroDoc::new_auto_commit(); + let store = doc.get_map("store"); + let books = store + .insert_container("books", ListHandler::new_detached()) + .unwrap(); + + for i in 0..size { + let book = books + .insert_container(i, MapHandler::new_detached()) + .unwrap(); + let title = if i % 100 == 42 { + "1984" + } else { + &format!("Book {}", i) + }; + let author = if i % 100 == 42 { + "George Orwell" + } else { + &format!("Author {}", i % 10) + }; + let price = 5.0 + (i % 25) as f64; + let available = i % 4 != 0; + + book.insert("title", title).unwrap(); + book.insert("author", author).unwrap(); + book.insert("price", price).unwrap(); + book.insert("available", available).unwrap(); + } + + // Add nested structure for $..title and recursive filters + let catalog = doc.get_map("catalog"); + let fiction = catalog + .insert_container("fiction", MapHandler::new_detached()) + .unwrap(); + let nested_books = fiction + .insert_container("books", ListHandler::new_detached()) + .unwrap(); + for i in 0..10 { + let b = nested_books + .insert_container(i, MapHandler::new_detached()) + .unwrap(); + b.insert("title", format!("Nested {}", i)).unwrap(); + b.insert("price", 12.0 + i as f64).unwrap(); + } + + doc + } + + /// Benchmark a single JSONPath query across all sizes + fn bench_pattern(c: &mut Criterion, group_name: &str, query: &str, setup: F) + where + F: Fn(usize) -> LoroDoc, + { + let mut group = c.benchmark_group(group_name); + group.sample_size(if group_name.contains("large") { 20 } else { 50 }); + + for &size in &SIZES { + let doc = setup(size); + let name = format!("{} (size={})", query, size); + group.bench_function(&name, |b| { + b.iter(|| { + let result = doc.jsonpath(query).unwrap(); + black_box(result); + }) + }); + } + group.finish(); + } + + pub fn child_selector(c: &mut Criterion) { + bench_pattern( + c, + "Child Selector", + "$.store.books[0].title", + setup_large_doc, + ); + } + + pub fn wildcard(c: &mut Criterion) { + bench_pattern(c, "Wildcard", "$.store.books[*].title", setup_large_doc); + } + + pub fn recursive_descent(c: &mut Criterion) { + bench_pattern(c, "Recursive Descent", "$..title", setup_large_doc); + } + + pub fn quoted_keys(c: &mut Criterion) { + bench_pattern( + c, + "Quoted Keys", + "$.store['books'][0]['title']", + setup_large_doc, + ); + } + + pub fn string_filter(c: &mut Criterion) { + bench_pattern( + c, + "String Filter", + "$.store.books[?(@.title == '1984')].title", + setup_large_doc, + ); + } + + pub fn logical_operator(c: &mut Criterion) { + bench_pattern( + c, + "Logical Operator", + "$.store.books[?(@.author == 'George Orwell' && @.price < 10)].title", + setup_large_doc, + ); + } + + pub fn union_operation(c: &mut Criterion) { + bench_pattern( + c, + "Union Operation", + "$.store.books[0,2].title", + setup_large_doc, + ); + } + + pub fn slice_operation(c: &mut Criterion) { + let mut group = c.benchmark_group("Slice Operation"); + + for &size in &SIZES { + let doc = setup_large_doc(size); + + // Small slice + group.bench_function(format!("$.store.books[0:3].title (size={})", size), |b| { + b.iter(|| { + let result = doc.jsonpath("$.store.books[0:3].title").unwrap(); + black_box(result); + }) + }); + + // Large slice (half) + let half = size / 2; + group.bench_function( + format!("$.store.books[0:{}].title (size={})", half, size), + |b| { + b.iter(|| { + let result = doc + .jsonpath(&format!("$.store.books[0:{}].title", half)) + .unwrap(); + black_box(result); + }) + }, + ); + + // Negative slice + group.bench_function(format!("$.store.books[-10:].title (size={})", size), |b| { + b.iter(|| { + let result = doc.jsonpath("$.store.books[-10:].title").unwrap(); + black_box(result); + }) + }); + } + group.finish(); + } + + pub fn complex_filter(c: &mut Criterion) { + bench_pattern( + c, + "Complex Filter", + "$.store.books[?(@.price >= 10 && @.available == true && @.title contains '1984')].title", + setup_large_doc, + ); + } + + pub fn recursive_mapped_filter(c: &mut Criterion) { + bench_pattern( + c, + "Recursive Mapped Filter", + "$.store.books[?(@.price > 10)].title", + setup_large_doc, + ); + } + + pub fn recursive_filter(c: &mut Criterion) { + bench_pattern( + c, + "Recursive Filter", + "$..[?(@.price > 10)].title", + setup_large_doc, + ); + } + + // === Legacy / Extra: Keep your original ones (now size-scaled) === + + pub fn index_selector(c: &mut Criterion) { + let mut group = c.benchmark_group("Index Selector"); + group.sample_size(50); + + for &size in &SIZES { + let doc = setup_large_doc(size); + let idx = (size as i64 / 2).min(42); // Safe index + group.bench_function(format!("positive index {} (size={})", idx, size), |b| { + b.iter(|| { + let result = doc + .jsonpath(&format!("$.store.books[{}].title", idx)) + .unwrap(); + black_box(result); + }) + }); + + group.bench_function(format!("negative index -10 (size={})", size), |b| { + b.iter(|| { + let result = doc.jsonpath("$.store.books[-10].title").unwrap(); + black_box(result); + }) + }); + } + group.finish(); + } + + pub fn union_key_selector(c: &mut Criterion) { + let mut group = c.benchmark_group("Union Key Selector"); + group.sample_size(50); + + for &size in &SIZES { + let doc = setup_large_doc(size); + group.bench_function(format!("two keys single item (size={})", size), |b| { + b.iter(|| { + let result = doc.jsonpath("$.store.books[42]['title','author']").unwrap(); + black_box(result); + }) + }); + + group.bench_function(format!("three keys slice (size={})", size), |b| { + b.iter(|| { + let result = doc + .jsonpath("$.store.books[0:10]['title','author','price']") + .unwrap(); + black_box(result); + }) + }); + } + group.finish(); + } + + pub fn mixed_selectors(c: &mut Criterion) { + bench_pattern( + c, + "Mixed Selectors", + "$.store.books[0,5:15,20].title", + setup_large_doc, + ); + } + + pub fn real_world_patterns(c: &mut Criterion) { + let mut group = c.benchmark_group("Real World Patterns"); + group.sample_size(50); + + for &size in &SIZES { + let doc = setup_large_doc(size); + group.bench_function(format!("last 10 items (size={})", size), |b| { + b.iter(|| { + let result = doc.jsonpath("$.store.books[-10:].title").unwrap(); + black_box(result); + }) + }); + + group.bench_function(format!("pagination page 1 (size={})", size), |b| { + b.iter(|| { + let result = doc.jsonpath("$.store.books[0:20].title").unwrap(); + black_box(result); + }) + }); + } + group.finish(); + } + + // === Master Group === + pub fn all_benches(c: &mut Criterion) { + child_selector(c); + wildcard(c); + recursive_descent(c); + quoted_keys(c); + string_filter(c); + logical_operator(c); + union_operation(c); + slice_operation(c); + complex_filter(c); + recursive_mapped_filter(c); + recursive_filter(c); + index_selector(c); + union_key_selector(c); + mixed_selectors(c); + real_world_patterns(c); + } +} + +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!(benches, jsonpath::all_benches); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); + +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/list.rs b/vendor/loro-internal/benches/list.rs new file mode 100644 index 00000000..4e2a7cc6 --- /dev/null +++ b/vendor/loro-internal/benches/list.rs @@ -0,0 +1,119 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +#[cfg(feature = "test_utils")] +mod run { + use super::*; + use arbitrary::Arbitrary; + use arbitrary::Unstructured; + use loro_internal::cursor::PosType; + use loro_internal::loro::ExportMode; + use loro_internal::LoroDoc; + use loro_internal::LoroValue; + use rand::Rng; + use rand::SeedableRng; + + pub fn many_list_containers(c: &mut Criterion) { + #[derive(Arbitrary)] + struct Action { + container: u8, + actor: u8, + pos: u8, + value: u8, + sync: u8, + } + + let mut rgn = rand::rngs::StdRng::seed_from_u64(0); + let mut bytes = Vec::new(); + for _ in 0..10000 { + bytes.push(rgn.gen::()); + } + + let mut gen = Unstructured::new(&bytes); + let actions: [Action; 200] = gen.arbitrary().unwrap(); + let mut b = c.benchmark_group("10 list containers"); + b.sample_size(10); + b.bench_function("sync random inserts to 10 list containers", |b| { + b.iter(|| { + let mut actors: Vec<_> = (0..10).map(|_| LoroDoc::default()).collect(); + for action in actions.iter() { + let len = actors.len(); + let actor = &mut actors[action.actor as usize % len]; + let container = action.container % 10; + if container % 2 == 0 { + let text = actor.get_text(container.to_string().as_str()); + let mut txn = actor.txn().unwrap(); + text.insert_with_txn( + &mut txn, + (action.pos as usize) % text.len_unicode().max(1), + action.value.to_string().as_str(), + PosType::Unicode, + ) + .unwrap(); + } else { + let list = actor.get_list(container.to_string().as_str()); + let mut txn = actor.txn().unwrap(); + list.insert_with_txn( + &mut txn, + (action.pos as usize) % list.len().max(1), + action.value.to_string().as_str().into(), + ) + .unwrap(); + } + + let a = (action.actor as usize) % len; + let b = (action.sync as usize) % len; + if a != b { + let (a, b) = arref::array_mut_ref!(&mut actors, [a, b]); + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + } + } + + for i in 1..actors.len() { + let (a, b) = arref::array_mut_ref!(&mut actors, [0, i]); + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + } + for i in 1..actors.len() { + let (a, b) = arref::array_mut_ref!(&mut actors, [i, 0]); + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + } + }) + }); + } + + pub fn many_actors(c: &mut Criterion) { + let mut b = c.benchmark_group("many_actors"); + b.sample_size(10); + b.bench_function("100 actors", |b| { + b.iter(|| { + let mut actors: Vec<_> = (0..100).map(|_| LoroDoc::default()).collect(); + for (i, actor) in actors.iter_mut().enumerate() { + let list = actor.get_list("list"); + let value: LoroValue = i.to_string().into(); + let mut txn = actor.txn().unwrap(); + list.insert_with_txn(&mut txn, 0, value).unwrap(); + } + + for i in 1..actors.len() { + let (a, b) = arref::array_mut_ref!(&mut actors, [0, i]); + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + } + + for i in 1..actors.len() { + let (a, b) = arref::array_mut_ref!(&mut actors, [0, i]); + b.import(&a.export(ExportMode::updates(&b.oplog_vv())).unwrap()) + .unwrap(); + } + }) + }); + } +} +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!(benches, run::many_list_containers, run::many_actors); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/map.rs b/vendor/loro-internal/benches/map.rs new file mode 100644 index 00000000..18daac9e --- /dev/null +++ b/vendor/loro-internal/benches/map.rs @@ -0,0 +1,85 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +#[cfg(feature = "test_utils")] +mod map { + use super::*; + use loro_internal::{loro::ExportMode, LoroDoc}; + use rand::{rngs::StdRng, Rng}; + + pub fn tree_move(c: &mut Criterion) { + let mut b = c.benchmark_group("map crdt"); + b.sample_size(10); + b.bench_function("create 10^4 key", |b| { + let size = 10000; + b.iter(|| { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + for i in 0..size { + map.insert(&i.to_string(), i).unwrap(); + } + loro.commit_then_renew(); + }) + }); + + b.bench_function("map checkout 10^3", |b| { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + let mut versions = vec![]; + let size = 10000; + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + for i in 0..size { + versions.push(loro.oplog_frontiers()); + map.insert(&rng.gen::().to_string(), i).unwrap(); + loro.commit_then_renew(); + } + + b.iter(|| { + for _ in 0..1000 { + let i = rng.gen::() % 1000; + let f = &versions[i]; + loro.checkout(f).unwrap(); + } + }) + }); + + b.bench_function("realtime map set", |b| { + let doc_a = LoroDoc::default(); + let doc_b = LoroDoc::default(); + let map_a = doc_a.get_map("map"); + let map_b = doc_b.get_map("map"); + let n = 1000; + b.iter(|| { + for t in 0..n { + if t % 2 == 0 { + let mut txn = doc_a.txn().unwrap(); + map_a.insert_with_txn(&mut txn, "key", t.into()).unwrap(); + doc_b + .import( + &doc_a + .export(ExportMode::updates(&doc_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } else { + let mut txn = doc_b.txn().unwrap(); + map_b.insert_with_txn(&mut txn, "key", t.into()).unwrap(); + doc_a + .import( + &doc_b + .export(ExportMode::updates(&doc_a.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + } + }) + }); + } +} + +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!(benches, map::tree_move); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/pending.rs b/vendor/loro-internal/benches/pending.rs new file mode 100644 index 00000000..0cede19e --- /dev/null +++ b/vendor/loro-internal/benches/pending.rs @@ -0,0 +1,47 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +#[cfg(feature = "test_utils")] +mod pending { + use super::*; + use bench_utils::TextAction; + use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc, VersionVector}; + + pub fn b4(c: &mut Criterion) { + let mut b = c.benchmark_group("B4 pending decode"); + b.sample_size(10); + b.bench_function("detached mode", |b| { + let loro = LoroDoc::default(); + let mut latest_vv = VersionVector::default(); + let mut updates = vec![]; + let actions = bench_utils::get_automerge_actions(); + let action_length = actions.len(); + let text = loro.get_text("text"); + for chunks in actions.chunks(action_length / 5) { + for TextAction { pos, ins, del } in chunks { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + updates.push(loro.export(ExportMode::updates(&latest_vv)).unwrap()); + latest_vv = loro.oplog_vv(); + } + } + updates.reverse(); + b.iter(|| { + let store2 = LoroDoc::default(); + store2.detach(); + for update in updates.iter() { + store2.import(update).unwrap(); + } + }) + }); + } +} + +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!(benches, pending::b4); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/text_r.rs b/vendor/loro-internal/benches/text_r.rs new file mode 100644 index 00000000..96c2e653 --- /dev/null +++ b/vendor/loro-internal/benches/text_r.rs @@ -0,0 +1,432 @@ +use criterion::{criterion_group, criterion_main, Criterion}; + +#[cfg(feature = "test_utils")] +mod run { + use std::hint::black_box; + use std::sync::Arc; + + use super::*; + use bench_utils::TextAction; + use loro_common::LoroValue; + use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc}; + + pub fn b4(c: &mut Criterion) { + let actions = bench_utils::get_automerge_actions(); + let mut b = c.benchmark_group("refactored direct_apply"); + b.sample_size(10); + b.bench_function("B4", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + let mut txn = loro.txn().unwrap(); + + for TextAction { pos, ins, del } in actions.iter() { + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + }) + }); + + b.bench_function("B4 with 100K actors history", |b| { + let store = LoroDoc::default(); + for i in 0..100_000 { + store.set_peer_id(i).unwrap(); + let list = store.get_list("list"); + let value: LoroValue = i.to_string().into(); + let mut txn = store.txn().unwrap(); + list.insert_with_txn(&mut txn, 0, value).unwrap(); + txn.commit().unwrap(); + } + + let update = store.export(ExportMode::Snapshot).unwrap(); + drop(store); + b.iter_batched( + || { + let loro = LoroDoc::default(); + loro.import(&update).unwrap(); + loro + }, + |loro| { + let text = loro.get_text("text"); + let mut txn = loro.txn().unwrap(); + + for TextAction { pos, ins, del } in actions.iter() { + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + }, + criterion::BatchSize::SmallInput, + ) + }); + + b.bench_function("B4 Obs", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + let _g = loro.subscribe_root(Arc::new(move |event| { + black_box(event); + })); + let mut txn = loro.txn().unwrap(); + for TextAction { pos, ins, del } in actions.iter() { + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + }) + }); + + b.bench_function("B4 encode snapshot", |b| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + + let mut n = 0; + let mut txn = loro.txn().unwrap(); + for TextAction { pos, ins, del } in actions.iter() { + if n == 10 { + n = 0; + drop(txn); + txn = loro.txn().unwrap(); + } + n += 1; + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + + b.iter(|| { + loro.export(ExportMode::Snapshot).unwrap(); + }); + }); + + b.bench_function("B4 encode updates", |b| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + + let mut n = 0; + let mut txn = loro.txn().unwrap(); + for TextAction { pos, ins, del } in actions.iter() { + if n == 10 { + n = 0; + drop(txn); + txn = loro.txn().unwrap(); + } + n += 1; + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + + b.iter(|| { + loro.export(ExportMode::all_updates()).unwrap(); + }); + }); + + b.bench_function("B4 decode snapshot", |b| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + let mut n = 0; + let mut txn = loro.txn().unwrap(); + for TextAction { pos, ins, del } in actions.iter() { + if n == 10 { + n = 0; + drop(txn); + txn = loro.txn().unwrap(); + } + n += 1; + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + + let data = loro.export(ExportMode::Snapshot).unwrap(); + b.iter(|| { + let l = LoroDoc::new(); + l.import(&data).unwrap(); + }); + }); + + b.bench_function("B4 import updates", |b| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + + let mut n = 0; + let mut txn = loro.txn().unwrap(); + for TextAction { pos, ins, del } in actions.iter() { + if n == 10 { + n = 0; + drop(txn); + txn = loro.txn().unwrap(); + } + n += 1; + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + + let data = loro.export(ExportMode::all_updates()).unwrap(); + b.iter(|| { + let l = LoroDoc::new(); + l.import(&data).unwrap(); + }); + }); + + // b.bench_function("B4 utf16", |b| { + // b.iter(|| { + // let loro = LoroDoc::new(); + // let text = loro.get_text("text"); + // let mut txn = loro.txn().unwrap(); + + // for TextAction { pos, ins, del } in actions.iter() { + // text.delete_utf16(&mut txn, *pos, *del).unwrap(); + // text.insert_utf16(&mut txn, *pos, ins).unwrap(); + // } + // }) + // }); + + b.bench_function("B4_Per100_Txn", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + let mut n = 0; + let mut txn = loro.txn().unwrap(); + for TextAction { pos, ins, del } in actions.iter() { + if n == 100 { + n = 0; + drop(txn); + txn = loro.txn().unwrap(); + } + n += 1; + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + }) + }); + + b.bench_function("B4 One Op One Txn", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + { + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + } + }) + }); + + b.bench_function("B4 One Op One Txn Obs", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + let _g = loro.subscribe_root(Arc::new(move |event| { + black_box(event); + })); + { + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + } + }) + }); + + b.bench_function("B4DirectSync", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let loro_b = LoroDoc::default(); + let text = loro.get_text("text"); + for TextAction { pos, ins, del } in actions.iter() { + { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + + loro_b + .import( + &loro + .export(ExportMode::updates(&loro_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + }) + }); + + drop(b); + let mut b = c.benchmark_group("refactored-sync"); + b.bench_function("B4Parallel", |b| { + b.iter(|| { + let loro = LoroDoc::default(); + let loro_b = LoroDoc::default(); + let text = loro.get_text("text"); + let text2 = loro_b.get_text("text"); + let mut i = 0; + for TextAction { pos, ins, del } in actions.iter() { + let pos = *pos; + let del = *del; + i += 1; + if i > 1000 { + break; + } + + { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, pos, del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, pos, ins, PosType::Unicode) + .unwrap(); + } + + { + let mut txn = loro_b.txn().unwrap(); + text2 + .delete_with_txn(&mut txn, pos, del, PosType::Unicode) + .unwrap(); + text2 + .insert_with_txn(&mut txn, pos, ins, PosType::Unicode) + .unwrap(); + } + loro_b + .import( + &loro + .export(ExportMode::updates(&loro_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + loro.import( + &loro_b + .export(ExportMode::updates(&loro.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + }) + }); + + let b = b.sample_size(10); + b.bench_function("DecodeUpdates B4Parallel", |b| { + let loro = LoroDoc::default(); + let loro_b = LoroDoc::default(); + let text = loro.get_text("text"); + let text2 = loro_b.get_text("text"); + for TextAction { pos, ins, del } in actions.iter() { + let pos = *pos; + let del = *del; + { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, pos, del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, pos, ins, PosType::Unicode) + .unwrap(); + } + + { + let mut txn = loro_b.txn().unwrap(); + text2 + .delete_with_txn(&mut txn, pos, del, PosType::Unicode) + .unwrap(); + text2 + .insert_with_txn(&mut txn, pos, ins, PosType::Unicode) + .unwrap(); + } + loro_b + .import( + &loro + .export(ExportMode::updates(&loro_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + loro.import( + &loro_b + .export(ExportMode::updates(&loro.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + let data = loro.export(ExportMode::all_updates()).unwrap(); + b.iter(|| { + let loro = LoroDoc::default(); + loro.import(&data).unwrap(); + }) + }); + b.bench_function("DecodeSnapshot B4Parallel", |b| { + let loro = LoroDoc::default(); + let loro_b = LoroDoc::default(); + let text = loro.get_text("text"); + let text2 = loro_b.get_text("text"); + for TextAction { pos, ins, del } in actions.iter() { + let pos = *pos; + let del = *del; + { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, pos, del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, pos, ins, PosType::Unicode) + .unwrap(); + } + + { + let mut txn = loro_b.txn().unwrap(); + text2 + .delete_with_txn(&mut txn, pos, del, PosType::Unicode) + .unwrap(); + text2 + .insert_with_txn(&mut txn, pos, ins, PosType::Unicode) + .unwrap(); + } + loro_b + .import( + &loro + .export(ExportMode::updates(&loro_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + loro.import( + &loro_b + .export(ExportMode::updates(&loro.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + let data = loro.export(ExportMode::Snapshot).unwrap(); + b.iter(|| { + let loro = LoroDoc::default(); + loro.import(&data).unwrap(); + }) + }); + } +} +pub fn dumb(_c: &mut Criterion) {} + +#[cfg(feature = "test_utils")] +criterion_group!(benches, run::b4); +#[cfg(not(feature = "test_utils"))] +criterion_group!(benches, dumb); +criterion_main!(benches); diff --git a/vendor/loro-internal/benches/tree.rs b/vendor/loro-internal/benches/tree.rs new file mode 100644 index 00000000..a55f9768 --- /dev/null +++ b/vendor/loro-internal/benches/tree.rs @@ -0,0 +1,166 @@ +use criterion::{criterion_group, criterion_main, Criterion}; + +mod tree { + use super::*; + use criterion::{AxisScale, BenchmarkId, PlotConfiguration}; + use loro_internal::{encoding::ExportMode, LoroDoc, TreeParentId}; + use rand::{rngs::StdRng, Rng}; + + pub fn tree_move(c: &mut Criterion) { + let mut group = c.benchmark_group("movable tree"); + let plot_config = PlotConfiguration::default().summary_scale(AxisScale::Logarithmic); + group.plot_config(plot_config); + group.sample_size(10); + + for i in 3..=6 { + let input = 10u64.pow(i); + group.bench_with_input( + BenchmarkId::new("create node append", input), + &input, + |b, i| { + b.iter(|| { + let loro = LoroDoc::new_auto_commit(); + let tree = loro.get_tree("tree"); + for idx in 0..*i { + tree.create_at(TreeParentId::Root, idx as usize).unwrap(); + } + }) + }, + ); + + group.bench_with_input( + BenchmarkId::new("move node append", input), + &input, + |b, i| { + let loro = LoroDoc::new_auto_commit(); + let tree = loro.get_tree("tree"); + const SIZE: usize = 1000; + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + let mut ids = vec![]; + for _ in 0..SIZE { + let pos = rng.gen::() % (ids.len() + 1); + ids.push(tree.create_at(TreeParentId::Root, pos).unwrap()); + } + + b.iter(|| { + for _ in 0..*i { + tree.create_at(TreeParentId::Root, 0).unwrap(); + let i = rng.gen::() % SIZE; + let j = rng.gen::() % SIZE; + tree.mov(ids[i], TreeParentId::Node(ids[j])) + .unwrap_or_default(); + } + }) + }, + ); + } + + group.bench_function("1000 node checkout 10^3", |b| { + let loro = LoroDoc::default(); + loro.start_auto_commit(); + let tree = loro.get_tree("tree"); + let mut ids = vec![]; + let mut versions = vec![]; + let size = 1000; + for _ in 0..size { + ids.push(tree.create(TreeParentId::Root).unwrap()) + } + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + let mut n = 1000; + while n > 0 { + let i = rng.gen::() % size; + let j = rng.gen::() % size; + if tree.mov(ids[i], TreeParentId::Node(ids[j])).is_ok() { + versions.push(loro.oplog_frontiers()); + n -= 1; + }; + } + b.iter(|| { + for _ in 0..1000 { + let i = rng.gen::() % 1000; + let f = &versions[i]; + loro.checkout(f).unwrap(); + } + }) + }); + + group.bench_function("300 deep node random checkout 10^3", |b| { + let depth = 300; + let loro = LoroDoc::default(); + loro.start_auto_commit(); + let tree = loro.get_tree("tree"); + let mut ids = vec![]; + let mut versions = vec![]; + let id1 = tree.create(TreeParentId::Root).unwrap(); + ids.push(id1); + versions.push(loro.oplog_frontiers()); + for _ in 1..depth { + let id = tree + .create(TreeParentId::Node(*ids.last().unwrap())) + .unwrap(); + ids.push(id); + versions.push(loro.oplog_frontiers()); + } + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + b.iter(|| { + for _ in 0..1000 { + let i = rng.gen::() % depth; + let f = &versions[i]; + loro.checkout(f).unwrap(); + } + }) + }); + + group.bench_function("realtime tree move", |b| { + let doc_a = LoroDoc::default(); + let doc_b = LoroDoc::default(); + doc_a.start_auto_commit(); + doc_b.start_auto_commit(); + let tree_a = doc_a.get_tree("tree"); + let tree_b = doc_b.get_tree("tree"); + let mut ids = vec![]; + let size = 1000; + for _ in 0..size { + ids.push(tree_a.create(TreeParentId::Root).unwrap()) + } + doc_b + .import(&doc_a.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + let n = 1000; + b.iter(|| { + for t in 0..n { + let i = rng.gen::() % size; + let j = rng.gen::() % size; + if t % 2 == 0 { + tree_a + .mov(ids[i], TreeParentId::Node(ids[j])) + .unwrap_or_default(); + doc_b + .import( + &doc_a + .export(ExportMode::updates(&doc_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } else { + tree_b + .mov(ids[i], TreeParentId::Node(ids[j])) + .unwrap_or_default(); + doc_a + .import( + &doc_b + .export(ExportMode::updates(&doc_a.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + } + }) + }); + group.finish(); + } +} + +criterion_group!(benches, tree::tree_move); +criterion_main!(benches); diff --git a/vendor/loro-internal/deno.json b/vendor/loro-internal/deno.json new file mode 100644 index 00000000..6edf0b37 --- /dev/null +++ b/vendor/loro-internal/deno.json @@ -0,0 +1,10 @@ +{ + "tasks": { + "test": "cargo nextest run --features test_utils", + "deny": "cargo deny check", + "fuzz": "cargo +nightly fuzz run", + "mem": "deno run -A ./scripts/run_mem.ts", + "flame": "CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --example encoding_refactored --root", + "bench": "cargo bench --features test_utils" + } +} diff --git a/vendor/loro-internal/deno.lock b/vendor/loro-internal/deno.lock new file mode 100644 index 00000000..5a95611b --- /dev/null +++ b/vendor/loro-internal/deno.lock @@ -0,0 +1,134 @@ +{ + "version": "3", + "redirects": { + "https://deno.land/std/fmt/printf.ts": "https://deno.land/std@0.105.0/fmt/printf.ts", + "https://deno.land/std/path/mod.ts": "https://deno.land/std@0.105.0/path/mod.ts", + "https://deno.land/std/testing/asserts.ts": "https://deno.land/std@0.105.0/testing/asserts.ts", + "https://x.nest.land/std@0.73.0/path/mod.ts": "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/mod.ts" + }, + "remote": { + "https://deno.land/std@0.105.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.105.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", + "https://deno.land/std@0.105.0/fmt/colors.ts": "d2f8355f00a74404668fc5a1e4a92983ce1a9b0a6ac1d40efbd681cb8f519586", + "https://deno.land/std@0.105.0/fmt/printf.ts": "7ec612e9b89958b8f7710129f74f502327aad285a9e48ee5297f5882fbc3a078", + "https://deno.land/std@0.105.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", + "https://deno.land/std@0.105.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", + "https://deno.land/std@0.105.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", + "https://deno.land/std@0.105.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", + "https://deno.land/std@0.105.0/path/glob.ts": "3b84af55c53febacf6afe214c095624b22a56b6f57d7312157479cc783a0de65", + "https://deno.land/std@0.105.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", + "https://deno.land/std@0.105.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", + "https://deno.land/std@0.105.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", + "https://deno.land/std@0.105.0/path/win32.ts": "f4a3d4a3f2c9fe894da046d5eac48b5e789a0ebec5152b2c0985efe96a9f7ae1", + "https://deno.land/std@0.105.0/testing/_diff.ts": "5d3693155f561d1a5443ac751ac70aab9f5d67b4819a621d4b96b8a1a1c89620", + "https://deno.land/std@0.105.0/testing/asserts.ts": "e4311d45d956459d4423bc267208fe154b5294989da2ed93257b6a85cae0427e", + "https://deno.land/std@0.106.0/_util/assert.ts": "2f868145a042a11d5ad0a3c748dcf580add8a0dbc0e876eaa0026303a5488f58", + "https://deno.land/std@0.106.0/_util/os.ts": "dfb186cc4e968c770ab6cc3288bd65f4871be03b93beecae57d657232ecffcac", + "https://deno.land/std@0.106.0/fs/_util.ts": "f2ce811350236ea8c28450ed822a5f42a0892316515b1cd61321dec13569c56b", + "https://deno.land/std@0.106.0/fs/copy.ts": "631bbafbfe6cba282158abc8aeb7e8251cc69a7ec28ce12878ea1b75fec2add4", + "https://deno.land/std@0.106.0/fs/empty_dir.ts": "5f08b263dd064dc7917c4bbeb13de0f5505a664b9cdfe312fa86e7518cfaeb84", + "https://deno.land/std@0.106.0/fs/ensure_dir.ts": "b7c103dc41a3d1dbbb522bf183c519c37065fdc234831a4a0f7d671b1ed5fea7", + "https://deno.land/std@0.106.0/fs/ensure_file.ts": "c06031af24368e80c330897e4b8e9109efc8602ffabc8f3e2306be07529e1d13", + "https://deno.land/std@0.106.0/fs/ensure_link.ts": "26e54363508b822afd87a3f6e873bbbcd6b5993dd638f8170758c16262a75065", + "https://deno.land/std@0.106.0/fs/ensure_symlink.ts": "c07b6d19ef58b6f5c671ffa942e7f9be50315f4f78e2f9f511626fd2e13beccc", + "https://deno.land/std@0.106.0/fs/eol.ts": "afaebaaac36f48c423b920c836551997715672b80a0fee9aa7667c181a94f2df", + "https://deno.land/std@0.106.0/fs/exists.ts": "b0d2e31654819cc2a8d37df45d6b14686c0cc1d802e9ff09e902a63e98b85a00", + "https://deno.land/std@0.106.0/fs/expand_glob.ts": "73e7b13f01097b04ed782b3d63863379b718417417758ba622e282b1e5300b91", + "https://deno.land/std@0.106.0/fs/mod.ts": "26eee4b52a8c516e37d464094b080ff6822883e7f01ff0ba0a72b8dcd54b9927", + "https://deno.land/std@0.106.0/fs/move.ts": "4623058e39bbbeb3ad30aeff9c974c55d2d574ad7c480295c12b04c244686a99", + "https://deno.land/std@0.106.0/fs/walk.ts": "b91c655c60d048035f9cae0e6177991ab3245e786e3ab7d20a5b60012edf2126", + "https://deno.land/std@0.106.0/path/_constants.ts": "1247fee4a79b70c89f23499691ef169b41b6ccf01887a0abd131009c5581b853", + "https://deno.land/std@0.106.0/path/_interface.ts": "1fa73b02aaa24867e481a48492b44f2598cd9dfa513c7b34001437007d3642e4", + "https://deno.land/std@0.106.0/path/_util.ts": "2e06a3b9e79beaf62687196bd4b60a4c391d862cfa007a20fc3a39f778ba073b", + "https://deno.land/std@0.106.0/path/common.ts": "eaf03d08b569e8a87e674e4e265e099f237472b6fd135b3cbeae5827035ea14a", + "https://deno.land/std@0.106.0/path/glob.ts": "3b84af55c53febacf6afe214c095624b22a56b6f57d7312157479cc783a0de65", + "https://deno.land/std@0.106.0/path/mod.ts": "4465dc494f271b02569edbb4a18d727063b5dbd6ed84283ff906260970a15d12", + "https://deno.land/std@0.106.0/path/posix.ts": "b81974c768d298f8dcd2c720229639b3803ca4a241fa9a355c762fa2bc5ef0c1", + "https://deno.land/std@0.106.0/path/separator.ts": "8fdcf289b1b76fd726a508f57d3370ca029ae6976fcde5044007f062e643ff1c", + "https://deno.land/std@0.106.0/path/win32.ts": "f4a3d4a3f2c9fe894da046d5eac48b5e789a0ebec5152b2c0985efe96a9f7ae1", + "https://deno.land/std@0.198.0/_util/os.ts": "d932f56d41e4f6a6093d56044e29ce637f8dcc43c5a90af43504a889cf1775e3", + "https://deno.land/std@0.198.0/assert/assert.ts": "9a97dad6d98c238938e7540736b826440ad8c1c1e54430ca4c4e623e585607ee", + "https://deno.land/std@0.198.0/assert/assertion_error.ts": "4d0bde9b374dfbcbe8ac23f54f567b77024fb67dbb1906a852d67fe050d42f56", + "https://deno.land/std@0.198.0/path/_basename.ts": "057d420c9049821f983f784fd87fa73ac471901fb628920b67972b0f44319343", + "https://deno.land/std@0.198.0/path/_constants.ts": "e49961f6f4f48039c0dfed3c3f93e963ca3d92791c9d478ac5b43183413136e0", + "https://deno.land/std@0.198.0/path/_dirname.ts": "355e297236b2218600aee7a5301b937204c62e12da9db4b0b044993d9e658395", + "https://deno.land/std@0.198.0/path/_extname.ts": "eaaa5aae1acf1f03254d681bd6a8ce42a9cb5b7ff2213a9d4740e8ab31283664", + "https://deno.land/std@0.198.0/path/_format.ts": "4a99270d6810f082e614309164fad75d6f1a483b68eed97c830a506cc589f8b4", + "https://deno.land/std@0.198.0/path/_from_file_url.ts": "7e4e5626089785adddb061f1b9f4932d6b21c7df778e7449531a11e32048245c", + "https://deno.land/std@0.198.0/path/_interface.ts": "6471159dfbbc357e03882c2266d21ef9afdb1e4aa771b0545e90db58a0ba314b", + "https://deno.land/std@0.198.0/path/_is_absolute.ts": "05dac10b5e93c63198b92e3687baa2be178df5321c527dc555266c0f4f51558c", + "https://deno.land/std@0.198.0/path/_join.ts": "fd78555bc34d5f188918fc7018dfe8fe2df5bbad94a3b30a433666c03934d77f", + "https://deno.land/std@0.198.0/path/_normalize.ts": "a19ec8706b2707f9dd974662a5cd89fad438e62ab1857e08b314a8eb49a34d81", + "https://deno.land/std@0.198.0/path/_parse.ts": "0f9b0ff43682dd9964eb1c4398610c4e165d8db9d3ac9d594220217adf480cfa", + "https://deno.land/std@0.198.0/path/_relative.ts": "27bdeffb5311a47d85be26d37ad1969979359f7636c5cd9fcf05dcd0d5099dc5", + "https://deno.land/std@0.198.0/path/_resolve.ts": "3bf0287d62488cad08c3c219a9708c4a4c658c65d7b4400fd99afdc3ba10a64d", + "https://deno.land/std@0.198.0/path/_to_file_url.ts": "739bfda583598790b2e77ce227f2bb618f6ebdb939788cea47555b43970ec58c", + "https://deno.land/std@0.198.0/path/_to_namespaced_path.ts": "0d5f4caa2ed98ef7a8786286df6af804b50e38859ae897b5b5b4c8c5930a75c8", + "https://deno.land/std@0.198.0/path/_util.ts": "4e191b1bac6b3bf0c31aab42e5ca2e01a86ab5a0d2e08b75acf8585047a86221", + "https://deno.land/std@0.198.0/path/basename.ts": "6f08fbb90dbfcf320765b3abb01f995b1723f75e2534acfd5380e202c802a3aa", + "https://deno.land/std@0.198.0/path/common.ts": "ee7505ab01fd22de3963b64e46cff31f40de34f9f8de1fff6a1bd2fe79380000", + "https://deno.land/std@0.198.0/path/dirname.ts": "098996822a31b4c46e1eb52a19540d3c6f9f54b772fc8a197939eeabc29fca2f", + "https://deno.land/std@0.198.0/path/extname.ts": "9b83c62fd16505739541f7a3ab447d8972da39dbf668d47af2f93206c2480893", + "https://deno.land/std@0.198.0/path/format.ts": "cb22f95cc7853d590b87708cc9441785e760d711188facff3d225305a8213aca", + "https://deno.land/std@0.198.0/path/from_file_url.ts": "a6221cfc928928ec4d9786d767dfac98fa2ab746af0786446c9834a07b98817e", + "https://deno.land/std@0.198.0/path/glob.ts": "d479e0a695621c94d3fd7fe7abd4f9499caf32a8de13f25073451c6ef420a4e1", + "https://deno.land/std@0.198.0/path/is_absolute.ts": "6b3d36352eb7fa29edb53f9e7b09b1aeb022a3c5465764f6cc5b8c41f9736197", + "https://deno.land/std@0.198.0/path/join.ts": "4a2867ff2f3c81ffc9eb3d56dade16db6f8bd3854f269306d23dad4115089c84", + "https://deno.land/std@0.198.0/path/mod.ts": "7765507696cb321994cdacfc19ee3ba61e8e3ebf4bd98fa75a276cf5dc18ce2a", + "https://deno.land/std@0.198.0/path/normalize.ts": "7d992cd262b2deefa842d93a8ba2ed51f3949ba595b1d07f627ac2cddbc74808", + "https://deno.land/std@0.198.0/path/parse.ts": "031fe488b3497fb8312fc1dc3c3d6c2d80707edd9c661e18ee9fd20f95edf322", + "https://deno.land/std@0.198.0/path/posix.ts": "0a1c1952d132323a88736d03e92bd236f3ed5f9f079e5823fae07c8d978ee61b", + "https://deno.land/std@0.198.0/path/relative.ts": "7db80c5035016174267da16321a742d76e875215c317859a383b12f413c6f5d6", + "https://deno.land/std@0.198.0/path/resolve.ts": "103b62207726a27f28177f397008545804ecb20aaf00623af1f622b18cd80b9f", + "https://deno.land/std@0.198.0/path/separator.ts": "0fb679739d0d1d7bf45b68dacfb4ec7563597a902edbaf3c59b50d5bcadd93b1", + "https://deno.land/std@0.198.0/path/to_file_url.ts": "dd32f7a01bbf3b15b5df46796659984b372973d9b2d7d59bcf0eb990763a0cb5", + "https://deno.land/std@0.198.0/path/to_namespaced_path.ts": "4e643ab729bf49ccdc166ad48d2de262ff462938fcf2a44a4425588f4a0bd690", + "https://deno.land/std@0.198.0/path/win32.ts": "8b3f80ef7a462511d5e8020ff490edcaa0a0d118f1b1e9da50e2916bdd73f9dd", + "https://deno.land/x/cliui@v7.0.4-deno/build/lib/index.d.ts": "4f04923352ce24027ad6fe25c85249e0b0cc5c29fe18d62024d004c59d9e41ee", + "https://deno.land/x/cliui@v7.0.4-deno/build/lib/index.js": "fb6030c7b12602a4fca4d81de3ddafa301ba84fd9df73c53de6f3bdda7b482d5", + "https://deno.land/x/cliui@v7.0.4-deno/build/lib/string-utils.js": "b3eb9d2e054a43a3064af17332fb1839a7dadb205c5371af4789616afb1a117f", + "https://deno.land/x/cliui@v7.0.4-deno/deno.ts": "d07bc3338661f8011e3a5fd215061d17a52107a5383c29f40ce0c1ecb8bb8cc3", + "https://deno.land/x/dirname@1.1.2/types.ts": "c1ed1667545bc4b1d69bdb2fc26a5fa8edae3a56e3081209c16a408a322a2319", + "https://deno.land/x/escalade@v3.0.3/sync.ts": "493bc66563292c5c10c4a75a467a5933f24dad67d74b0f5a87e7b988fe97c104", + "https://deno.land/x/y18n@v5.0.0-deno/build/lib/index.d.ts": "11f40d97041eb271cc1a1c7b296c6e7a068d4843759575e7416f0d14ebf8239c", + "https://deno.land/x/y18n@v5.0.0-deno/build/lib/index.js": "92c4624714aa508d33c6d21c0b0ffa072369a8b306e5f8c7727662f570bbd026", + "https://deno.land/x/y18n@v5.0.0-deno/deno.ts": "80997f0709a0b43d29931e2b33946f2bbc32b13fd82f80a5409628455427e28d", + "https://deno.land/x/y18n@v5.0.0-deno/lib/platform-shims/deno.ts": "8fa2c96ac03734966260cfd2c5bc240e41725c913e5b64a0297aede09f52b39d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/argsert.js": "eb085555452eac3ff300935994a42f35d16e04cf698cb775cb5ad4f5653c0627", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/command.js": "499c95cecd5e93f627e0b5ce66a193c9a595adc10fbafe0581a9725e38324dee", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/completion-templates.js": "f84823b1daa0ed0189e4f823f6a4fd29ad58de6a05771004918368fd62bb2b3f", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/completion.js": "c91772b89907ebf1a462804305d12d3b9deade75cd1b319c06831ac0bf5abd27", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/middleware.js": "cef3f017d5ff61c340c65b8422f5ab9600ba381aa656df634d1a3edf0f967527", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/parse-command.js": "327242c0afae207b7aefa13133439e3b321d7db4229febc5b7bd5285770ac7f7", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/typings/common-types.js": "9618b81a86acb88a61fd9988e9bc3ec21c5250d94fc2231ba7d898e71500789d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/usage.js": "61071feb99ac220f1b27036406ae8e4f9ee606b373a5f3bcb60042c7bcfbd0d8", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/apply-extends.js": "64640dce92669705abead3bdbe2c46c8318c8623843a55e4726fb3c55ff9dd1d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/is-promise.js": "be45baa3090c5106dd4e442cceef6b357a268783a2ee28ec10fe131a8cd8db72", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/levenshtein.js": "d8638efc3376b5f794b1c8df6ef4f3d484b29d919127c7fdc242400e3cfded91", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/maybe-async-result.js": "31cf4026279e14c87d16faa14ac758f35c8cc5795d29393c5ce07120f5a3caf6", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/obj-filter.js": "5523fb2288d1e86ed48c460e176770b49587554df4ae2405b468c093786b040b", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/set-blocking.js": "6fa8ffc3299f456e42902736bae35fbc1f2dc96b3905a02ba9629f5bd9f80af1", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/utils/which-module.js": "9267633b2c9f8990b2c699101b641e59ae59932e0dee5270613c0508bfa13c5d", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/validation.js": "3dc366de2eb23bc9457ed3e120b69db9d801251bef3dc19f93e4c0380ac0198c", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/yargs-factory.js": "a3e629d7d063b5ac007b18a0d8e9ad2ca72ca4d702c5c46822fbbdfdd6c512df", + "https://deno.land/x/yargs@v17.1.1-deno/build/lib/yerror.js": "1d9dead374fe06c8f13f2e4adafc002b8a15682b7185abf29638f1be96fd9dfc", + "https://deno.land/x/yargs@v17.1.1-deno/deno-types.ts": "62f5c61899c6da491890c8c84fd9580cfbfa2a83f5a70f6dc74727bbfb148623", + "https://deno.land/x/yargs@v17.1.1-deno/deno.ts": "f3df0bfd08ba367ec36dc59ef6cab1a391ace49ad44387ec5fe5d76289af08af", + "https://deno.land/x/yargs@v17.1.1-deno/lib/platform-shims/deno.ts": "b5a48b40d5c64fe66f5a77f87ebaf4413eea828ccd8159feeac370b3eef9a356", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/string-utils.js": "12fc056b23703bc370aae5b179dc5abee53fca277abc30eaf76f78d2546d6413", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/tokenize-arg-string.js": "7e0875b11795b8e217386e45f14b24a6e501ebbc62e15aa469aa8829d4d0ee61", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/yargs-parser-types.d.ts": "434deb76c6632b3b6cbc4c6f153f8aca04e06055ae9c6b24b40218cbc42688d9", + "https://deno.land/x/yargs_parser@v20.2.4-deno/build/lib/yargs-parser.js": "453200a7dfbb002e605d8009b7dad30f2b1d93665e046ab89c073a4fe63dfd48", + "https://deno.land/x/yargs_parser@v20.2.4-deno/deno.ts": "ad53c0c82c3982c4fc5be9472384b259e0a32ce1f7ae0f68de7b2445df5642fc", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/_util/assert.ts": "e1f76e77c5ccb5a8e0dbbbe6cce3a56d2556c8cb5a9a8802fc9565af72462149", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/_constants.ts": "aba480c4a2c098b6374fdd5951fea13ecc8aaaf8b8aa4dae1871baa50243d676", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/_interface.ts": "67b276380d297a7cedc3c17f7a0bf122edcfc96a3e1f69de06f379d85ba0e2c0", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/_util.ts": "7820a788b35c26dfc27ff329df12507fc0553ae92727009597046f6cf856b4fa", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/common.ts": "e4ec66a7416d56f60331b66e27a8a4f08c7b1cf48e350271cb69754a01cf5c04", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/glob.ts": "54fea22f0f47dee07399d22e18a34a84eaf4b8a3771bbed96db43bbec71ef237", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/mod.ts": "6de8885c2534757097818e302becd1cefcbc4c28ac022cc279e612ee04e8cfd1", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/posix.ts": "7c2539b5d9e22b47f02ccc5b6ef11e3975a5c60f49d2241271eaa5936e6082a0", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/separator.ts": "9dd15d46ff84a16e13554f56af7fee1f85f8d0f379efbbe60ac066a60561f036", + "https://lra6z45nakk5lnu3yjchp7tftsdnwwikwr65ocha5eojfnlgu4sa.arweave.net/XEHs860CldW2m8JEd_5lnIbbWQq0fdcI4OkckrVmpyQ/path/win32.ts": "47114c941681ecbabab4ec355cb41d525fb5a14175cc47a5c76fdc5eaee2685a" + } +} diff --git a/vendor/loro-internal/docs/diff_calc.md b/vendor/loro-internal/docs/diff_calc.md new file mode 100644 index 00000000..22e7d78b --- /dev/null +++ b/vendor/loro-internal/docs/diff_calc.md @@ -0,0 +1,23 @@ +# Internal of Diff Calculation + +Diff calculation is the core of the `diff` command. It is responsible for calculating the difference between two versions of a container. + +# Three modes of diff calculation + +## 1. Checkout Mode + +This is the most general mode of diff calculation. It can be used whenever a user want to switch to a different version. +But it is also the slowest mode. It relies on the `ContainerHistoryCache`, which is expensive to build and maintain in memory. + +## 2. Import Mode + +This mode is used when the user imports new updates. It is faster than the checkout mode, but it is still slower than the linear mode. + +- The difference between the import mode and the checkout mode: in import mode, target version > current version. + So when calculating the `DiffCalculator` doesn't need to rely on `ContainerHistoryCache`, except for the Tree container. +- The difference between the import mode and the linear mode: in linear mode, all the imported updates are ordered, no concurrent update exists. + so there is no need to build CRDTs for the calculation + +## 3. Linear Mode + +This mode is used when we don't need to build CRDTs to calculate the difference. It is the fastest mode. diff --git a/vendor/loro-internal/docs/mergeable-container-id.md b/vendor/loro-internal/docs/mergeable-container-id.md new file mode 100644 index 00000000..c741beec --- /dev/null +++ b/vendor/loro-internal/docs/mergeable-container-id.md @@ -0,0 +1,87 @@ +# Mergeable Container ID Encoding + +Mergeable child containers are represented as synthetic `ContainerID::Root` +values. The root name is internal and starts with the reserved namespace prefix: + +```text +🤝: +``` + +The child container type is not encoded in ``. It is carried by +`ContainerID::Root.container_type`, exactly like ordinary root containers. + +## Payload + +`` is a flattened map path: + +```text +payload = base-parent ">" key-1 ">" key-2 ... +``` + +There is no version byte in this format. It replaces the previous unpublished +encoding. + +`` is the nearest non-mergeable map ancestor: + +```text +$ +@: +``` + +`$` means the base parent is a root map. The rest of the segment is the escaped +root name. `@` means the base parent is a normal op-created map. The peer id and +counter are canonical lowercase base36 encoded to keep the payload compact. +Leading zeroes, uppercase digits, and `-0` are rejected by the parser because +`new_mergeable` never emits them. + +Every following segment is one escaped map key. Intermediate mergeable parents +are always maps, so their type is omitted. The final container's type is the +`Root.container_type` field. + +For example: + +```text +Root map "state", key "note-1", child map: +🤝:$state>note-1 with Root.container_type = Map + +Nested key "body" under that mergeable map, child text: +🤝:$state>note-1>body with Root.container_type = Text +``` + +Parsing `🤝:$state>note-1>body` as a `Text` returns: + +```text +parent = Root("🤝:$state>note-1", Map) +key = "body" +container_type = Text +``` + +This keeps nesting growth linear in the total path length. It does not embed the +full serialized parent cid at each level. + +## Escaping + +Segments are escaped before being placed in the synthetic root name: + +```text +\ -> \\ +> -> \> +/ -> \s +NUL -> \0 +``` + +`>` is the only structural delimiter. `\` introduces an escape. `/` and NUL are +also escaped so synthetic root names keep the same safety property as the old +hex encoding: they do not contain raw slash or raw NUL bytes. + +The parser rejects dangling backslashes, unknown escapes, raw slash, and raw NUL. +This prevents malformed synthetic roots from being treated as mergeable ids. + +## Marker Relationship + +The parent map slot still stores the binary mergeable marker. The marker binds +`(parent container id, key, child type)` through its digest and is not changed by +this encoding. + +The root name encoding only controls the deterministic identity of the child +container. Visibility remains controlled by the parent map slot marker. diff --git a/vendor/loro-internal/examples/automerge_x100.rs b/vendor/loro-internal/examples/automerge_x100.rs new file mode 100644 index 00000000..c8468f10 --- /dev/null +++ b/vendor/loro-internal/examples/automerge_x100.rs @@ -0,0 +1,41 @@ +use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc, VersionVector}; +use std::hint::black_box; + +fn main() { + use bench_utils::TextAction; + use std::time::Instant; + + let actions = bench_utils::get_automerge_actions(); + let loro = LoroDoc::default(); + loro.start_auto_commit(); + let start = Instant::now(); + // loro.subscribe_deep(Box::new(|_| ())); + let text = loro.get_text("text"); + let n = 100; + let mut v = VersionVector::new(); + for _ in 0..n { + for TextAction { del, ins, pos } in actions.iter() { + text.delete(*pos, *del, PosType::Unicode).unwrap(); + text.insert(*pos, ins, PosType::Unicode).unwrap(); + } + loro.commit_then_renew(); + black_box(loro.export(ExportMode::updates(&v))).unwrap(); + v = loro.oplog_vv(); + } + println!("Apply time {:?}", start.elapsed()); + loro.diagnose_size(); + drop(actions); + let start = Instant::now(); + let snapshot = loro.export(ExportMode::Snapshot).unwrap(); + println!("Snapshot encoding time {}", start.elapsed().as_millis()); + let compressed = zstd::encode_all(&mut snapshot.as_slice(), 0).unwrap(); + println!( + "Snapshot encoding time including compression {}", + start.elapsed().as_millis() + ); + println!("Snapshot size {}", snapshot.len()); + println!("Snapshot size after compression {}", compressed.len()); + let start = Instant::now(); + let _doc = LoroDoc::from_snapshot(&snapshot); + println!("Snapshot importing time {:?}", start.elapsed()); +} diff --git a/vendor/loro-internal/examples/encoding.rs b/vendor/loro-internal/examples/encoding.rs new file mode 100644 index 00000000..2ac5f309 --- /dev/null +++ b/vendor/loro-internal/examples/encoding.rs @@ -0,0 +1,155 @@ +use std::time::Instant; + +use bench_utils::TextAction; +use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc}; + +fn main() { + let actions = bench_utils::get_automerge_actions(); + + // let loro = LoroDoc::default(); + // let loro_b = LoroDoc::default(); + // let text = loro.get_text("text"); + // let mut i = 0; + // let start = Instant::now(); + // for TextAction { pos, ins, del } in actions.iter() { + // { + // let mut txn = loro.txn().unwrap(); + // text.delete(&mut txn, *pos, *del).unwrap(); + // text.insert(&mut txn, *pos, ins).unwrap(); + // } + + // loro_b + // .import(&loro.export(ExportMode::updates(&loro_b.oplog_vv())).unwrap()) + // .unwrap(); + // i += 1; + // if i == 30000 { + // break; + // } + // } + + // println!("{}ms", start.elapsed().as_millis()); + + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + + let start = Instant::now(); + let snapshot = loro.export(ExportMode::Snapshot).unwrap(); + println!("Snapshot time {}ms", start.elapsed().as_millis()); + let output = miniz_oxide::deflate::compress_to_vec(&snapshot, 6); + println!( + "Snapshot+compression time {}ms", + start.elapsed().as_millis() + ); + + println!( + "snapshot size {} after compression {}", + snapshot.len(), + output.len(), + ); + + let start = Instant::now(); + let shallow_snapshot = loro + .export(ExportMode::shallow_snapshot(&loro.oplog_frontiers())) + .unwrap(); + println!("Shallow Snapshot time {}ms", start.elapsed().as_millis()); + let output = miniz_oxide::deflate::compress_to_vec(&shallow_snapshot, 6); + println!( + "Shallow Snapshot+compression time {}ms", + start.elapsed().as_millis() + ); + + println!( + "Shallow snapshot size {} after compression {}", + shallow_snapshot.len(), + output.len(), + ); + + let updates = loro.export(ExportMode::all_updates()).unwrap(); + let output = miniz_oxide::deflate::compress_to_vec(&updates, 6); + println!( + "updates size {} after compression {}", + updates.len(), + output.len(), + ); + + let json_updates = serde_json::to_string(&loro.export_json_updates( + &Default::default(), + &loro.oplog_vv(), + true, + )) + .unwrap(); + let output = miniz_oxide::deflate::compress_to_vec(json_updates.as_bytes(), 6); + println!( + "json updates size {} after compression {}", + json_updates.len(), + output.len(), + ); + + // { + // // Delta encoding + + { + // Delta encoding + + let start = Instant::now(); + for _ in 0..10 { + loro.export(ExportMode::all_updates()).unwrap(); + } + + println!("Avg encode {}ms", start.elapsed().as_millis() as f64 / 10.0); + + let data = loro.export(ExportMode::all_updates()).unwrap(); + let start = Instant::now(); + let n = 5; + for _ in 0..n { + let b = LoroDoc::default(); + b.detach(); + b.import(&data).unwrap(); + } + + println!( + "Avg normal decode {}ms (without applying)", + start.elapsed().as_millis() as f64 / (n as f64) + ); + println!("size len={}", data.len()); + let d = miniz_oxide::deflate::compress_to_vec(&data, 10); + println!("size after compress len={}", d.len()); + } + + // { + // // Snapshot encoding + // // println!("\n=======================\nSnapshot Encoding:"); + + // // let start = Instant::now(); + // // for _ in 0..10 { + // // loro.export(ExportMode::Snapshot); + // // } + + // // println!("Avg encode {}ms", start.elapsed().as_millis() as f64 / 10.0); + + // // let data = loro.export(ExportMode::Snapshot); + // // let start = Instant::now(); + // // let times = 300; + // // for _ in 0..times { + // // let b = LoroDoc::default(); + // // b.import(&data).unwrap(); + // // } + + // // println!( + // // "Avg decode {}ms", + // // start.elapsed().as_millis() as f64 / times as f64 + // // ); + // // println!("size len={}", data.len()); + // // let d = miniz_oxide::deflate::compress_to_vec(&data, 10); + // // println!("size after compress len={}", d.len()); + // } +} diff --git a/vendor/loro-internal/examples/encoding_parallel.rs b/vendor/loro-internal/examples/encoding_parallel.rs new file mode 100644 index 00000000..79224d91 --- /dev/null +++ b/vendor/loro-internal/examples/encoding_parallel.rs @@ -0,0 +1,90 @@ +use bench_utils::{get_automerge_actions, TextAction}; +use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc}; + +// #[allow(dead_code)] +// fn parallel() { +// let mut rng: StdRng = SeedableRng::seed_from_u64(1); + +// let actions = get_automerge_actions(); +// let mut loro = LoroCore::new(Default::default(), Some(1)); +// let mut loro_b = LoroCore::new(Default::default(), Some(2)); +// let mut i = 0; +// for TextAction { pos, ins, del } in actions.iter() { +// i += 1; +// if i > 1000 { +// break; +// } +// let pos = *pos; +// let del = *del; +// let mut text = loro.get_text("text"); +// text.delete(&loro, pos, del).unwrap(); +// text.insert(&loro, pos, ins).unwrap(); +// let mut text = loro_b.get_text("text"); +// let r = rng.gen_range(1..11); +// for _ in 0..r { +// text.delete(&loro_b, pos, del).unwrap(); +// text.insert(&loro_b, pos, ins).unwrap(); +// } +// loro_b.import(loro.export(loro_b.vv_cloned())); +// loro.import(loro_b.export(loro.vv_cloned())); +// } +// let encoded = loro.encode_with_cfg(EncodeMode::RleUpdates(VersionVector::new())); +// println!("parallel doc size {} bytes", encoded.len()); +// let encoded = loro.encode_all(); +// println!("parallel doc size {} bytes", encoded.len()); +// } + +// #[allow(dead_code)] +// fn real_time() { +// let actions = get_automerge_actions(); +// let mut c1 = LoroCore::new(Default::default(), Some(0)); +// let mut c2 = LoroCore::new(Default::default(), Some(1)); +// let mut t1 = c1.get_text("text"); +// let mut t2 = c2.get_text("text"); +// for (i, action) in actions.iter().enumerate() { +// if i > 2000 { +// break; +// } +// let TextAction { pos, ins, del } = action; +// if i % 2 == 0 { +// t1.delete(&c1, *pos, *del).unwrap(); +// t1.insert(&c1, *pos, ins).unwrap(); +// let update = c1.encode_with_cfg(EncodeMode::Updates(c2.vv_cloned())); +// c2.decode(&update).unwrap(); +// } else { +// t2.delete(&c2, *pos, *del).unwrap(); +// t2.insert(&c2, *pos, ins).unwrap(); +// let update = c2.encode_with_cfg(EncodeMode::Updates(c1.vv_cloned())); +// c1.decode(&update).unwrap(); +// } +// } +// } + +fn main() { + let actions = get_automerge_actions(); + let loro = LoroDoc::default(); + let loro_b = LoroDoc::default(); + let text = loro.get_text("text"); + let mut count = 0; + for TextAction { pos, ins, del } in actions.iter() { + { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + + loro_b + .import( + &loro + .export(ExportMode::updates(&loro_b.oplog_vv())) + .unwrap(), + ) + .unwrap(); + count += 1; + if count % 1000 == 0 { + println!("{count}"); + } + } +} diff --git a/vendor/loro-internal/examples/encoding_refactored.rs b/vendor/loro-internal/examples/encoding_refactored.rs new file mode 100644 index 00000000..709c426c --- /dev/null +++ b/vendor/loro-internal/examples/encoding_refactored.rs @@ -0,0 +1,115 @@ +use bench_utils::TextAction; +use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc}; +use std::hint::black_box; + +fn main() { + log_size(); + // bench_decode(); + // bench_decode_updates(); +} + +#[allow(unused)] +fn log_size() { + let actions = bench_utils::get_automerge_actions(); + { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + let mut txn = loro.txn().unwrap(); + + for TextAction { pos, ins, del } in actions.iter() { + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + txn.commit().unwrap(); + let snapshot = loro.export(ExportMode::Snapshot).unwrap(); + let updates = loro.export(ExportMode::all_updates()).unwrap(); + let json_updates = serde_json::to_string(&loro.export_json_updates( + &Default::default(), + &loro.oplog_vv(), + true, + )) + .unwrap(); + println!("\n"); + println!("Snapshot size={}", snapshot.len()); + println!("Updates size={}", updates.len()); + println!("Json Updates size={}", json_updates.len()); + println!("\n"); + loro.diagnose_size(); + } + println!("\n"); + println!("\n"); + println!("\n"); + { + println!("One Transaction Per Action"); + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + let snapshot = loro.export(ExportMode::Snapshot).unwrap(); + let updates = loro.export(ExportMode::all_updates()).unwrap(); + println!("\n"); + println!("Snapshot size={}", snapshot.len()); + println!("Updates size={}", updates.len()); + println!("\n"); + loro.diagnose_size(); + } +} + +#[allow(unused)] +fn bench_decode() { + println!("Bench decode"); + let actions = bench_utils::get_automerge_actions(); + { + let mut loro = LoroDoc::default(); + let text = loro.get_text("text"); + loro.start_auto_commit(); + + for _ in 0..10 { + for TextAction { pos, ins, del } in actions.iter() { + text.delete(*pos, *del, PosType::Unicode).unwrap(); + text.insert(*pos, ins, PosType::Unicode).unwrap(); + } + } + let snapshot = loro.export(ExportMode::Snapshot).unwrap(); + // for _ in 0..100 { + // black_box(loro.export(ExportMode::Snapshot)); + // } + + for _ in 0..100 { + let loro = LoroDoc::new(); + loro.import(black_box(&snapshot)).unwrap(); + } + } +} + +#[allow(unused)] +fn bench_decode_updates() { + let actions = bench_utils::get_automerge_actions(); + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + + #[allow(warnings)] + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + + let updates = loro.export(ExportMode::all_updates()).unwrap(); + for _ in 0..10 { + let loro = LoroDoc::new(); + loro.import(black_box(&updates)).unwrap(); + } +} diff --git a/vendor/loro-internal/examples/event.rs b/vendor/loro-internal/examples/event.rs new file mode 100644 index 00000000..b1fb85e4 --- /dev/null +++ b/vendor/loro-internal/examples/event.rs @@ -0,0 +1,66 @@ +use std::sync::Arc; + +use loro_internal::{ + cursor::PosType, + event::Diff, + handler::{Handler, ValueOrHandler}, + ListHandler, LoroDoc, MapHandler, TextHandler, ToJson, TreeHandler, +}; + +fn main() { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let list = doc.get_list("list"); + let _g = doc.subscribe_root(Arc::new(|e| { + for container_diff in e.events { + match &container_diff.diff { + Diff::List(list) => { + for item in list.iter() { + if let loro_delta::DeltaItem::Replace { value, .. } = item { + for v in value.iter() { + match v { + ValueOrHandler::Handler(h) => { + // You can directly obtain the handler and perform some operations. + if matches!(h, Handler::Map(_)) { + let text = h + .as_map() + .unwrap() + .insert_container( + "text", + TextHandler::new_detached(), + ) + .unwrap(); + text.insert(0, "created from event", PosType::Unicode) + .unwrap(); + } + } + ValueOrHandler::Value(value) => { + println!("insert value {value:?}"); + } + } + } + } + } + } + Diff::Map(map) => { + println!("map container updates {:?}", map.updated); + } + _ => {} + } + } + })); + list.insert(0, "abc").unwrap(); + list.insert_container(1, ListHandler::new_detached()) + .unwrap(); + list.insert_container(2, MapHandler::new_detached()) + .unwrap(); + list.insert_container(3, TextHandler::new_detached()) + .unwrap(); + list.insert_container(4, TreeHandler::new_detached()) + .unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.get_deep_value().to_json(), + r#"{"list":["abc",[],{"text":"created from event"},"",[]]}"# + ); +} diff --git a/vendor/loro-internal/examples/list.rs b/vendor/loro-internal/examples/list.rs new file mode 100644 index 00000000..b31373e4 --- /dev/null +++ b/vendor/loro-internal/examples/list.rs @@ -0,0 +1,23 @@ +use std::time::Instant; +// #[global_allocator] +// static ALLOC: dhat::Alloc = dhat::Alloc; + +use loro_internal::{encoding::ExportMode, LoroDoc}; + +fn main() { + // let p = dhat::Profiler::builder().trim_backtraces(None).build(); + let start = Instant::now(); + let actor = LoroDoc::default(); + let mut output = Vec::new(); + let list = actor.get_list("list"); + let mut last_vv = actor.oplog_vv(); + for i in 0..10000 { + let mut txn = actor.txn().unwrap(); + list.insert_with_txn(&mut txn, i, i.to_string().into()) + .unwrap(); + output.push(actor.export(ExportMode::updates(&last_vv.clone())).unwrap()); + last_vv = actor.oplog_vv(); + } + println!("{} ms", start.elapsed().as_millis()); + // drop(p) +} diff --git a/vendor/loro-internal/examples/many_actors.rs b/vendor/loro-internal/examples/many_actors.rs new file mode 100644 index 00000000..fd36b1f7 --- /dev/null +++ b/vendor/loro-internal/examples/many_actors.rs @@ -0,0 +1,60 @@ +use std::time::Instant; + +use loro_internal::{loro::ExportMode, LoroDoc, LoroValue}; +// #[global_allocator] +// static ALLOC: dhat::Alloc = dhat::Alloc; + +fn main() { + // with_100k_actors_then_action(); + import_with_many_actors(); +} + +#[allow(unused)] +fn import_with_many_actors() { + let store = LoroDoc::default(); + for i in 0..10000 { + store.set_peer_id(i); + let list = store.get_list("list"); + let value: LoroValue = i.to_string().into(); + let mut txn = store.txn().unwrap(); + list.insert_with_txn(&mut txn, 0, value).unwrap(); + txn.commit().unwrap(); + } + + { + let start = Instant::now(); + let bytes = store.export(ExportMode::Snapshot).unwrap(); + LoroDoc::default().import(&bytes).unwrap(); + println!("{} ms", start.elapsed().as_millis()); + } + + // let profiler = dhat::Profiler::builder().trim_backtraces(None).build(); + // let start = Instant::now(); + // let mut actor = LoroDoc::default(); + // actor.import_batch(&updates).unwrap(); + // println!("{} bytes", updates.iter().map(|x| x.len()).sum::()); + // // dbg!(actor.get_state_deep_value()); + // println!("{} ms", start.elapsed().as_millis()); + // drop(profiler); +} + +#[allow(unused)] +fn with_100k_actors_then_action() { + let store = LoroDoc::default(); + for i in 0..100_000 { + store.set_peer_id(i); + let list = store.get_list("list"); + let value: LoroValue = i.to_string().into(); + let mut txn = store.txn().unwrap(); + list.insert_with_txn(&mut txn, 0, value).unwrap(); + txn.commit().unwrap(); + } + + for i in 0..200_000 { + let list = store.get_list("list"); + let value: LoroValue = i.to_string().into(); + let mut txn = store.txn().unwrap(); + list.insert_with_txn(&mut txn, 0, value).unwrap(); + txn.commit().unwrap(); + } +} diff --git a/vendor/loro-internal/examples/mem.rs b/vendor/loro-internal/examples/mem.rs new file mode 100644 index 00000000..b7bda998 --- /dev/null +++ b/vendor/loro-internal/examples/mem.rs @@ -0,0 +1,110 @@ +// #[global_allocator] +// static ALLOC: dhat::Alloc = dhat::Alloc; + +// use std::time::Instant; + +// use bench_utils::TextAction; +// use loro_internal::LoroDoc; + +// fn apply_automerge(times: usize) { +// let actions = bench_utils::get_automerge_actions(); +// let start = Instant::now(); +// let profiler = dhat::Profiler::builder().trim_backtraces(None).build(); +// let mut loro = LoroCore::default(); +// let mut text = loro.get_text("text"); +// println!("Apply Automerge Dataset 1X"); +// for _i in 0..times { +// for TextAction { pos, ins, del } in actions.iter() { +// text.delete(&loro, *pos, *del).unwrap(); +// text.insert(&loro, *pos, ins).unwrap(); +// } +// } +// drop(profiler); +// println!("Used: {} ms", start.elapsed().as_millis()); +// } + +// fn concurrent_actors(actor_num: usize) { +// let mut actors: Vec = Vec::new(); +// for _ in 0..actor_num { +// actors.push(LoroCore::default()); +// } + +// let mut updates = Vec::new(); +// for actor in actors.iter_mut() { +// let mut list = actor.get_list("list"); +// list.insert(actor, 0, 1).unwrap(); +// updates.push(actor.encode_all()); +// } + +// let mut a = actors.drain(0..1).next().unwrap(); +// drop(actors); +// let profiler = dhat::Profiler::builder().trim_backtraces(None).build(); +// for update in updates { +// a.decode(&update).unwrap(); +// } +// drop(profiler); +// } + +// fn realtime_sync(actor_num: usize, action_num: usize) { +// let actions = bench_utils::gen_realtime_actions(action_num, actor_num, 100); +// let profiler = dhat::Profiler::builder().trim_backtraces(None).build(); +// let mut actors = Vec::new(); +// for _ in 0..actor_num { +// actors.push(LoroDoc::default()); +// } + +// for action in actions { +// match action { +// bench_utils::Action::Text { client, action } => { +// let mut text = actors[client].get_text("text"); +// let bench_utils::TextAction { pos, ins, del } = action; +// let pos = pos % (text.len() + 1); +// let del = del.min(text.len() - pos); +// text.delete(&actors[client], pos, del).unwrap(); +// text.insert(&actors[client], pos, &ins).unwrap(); +// } +// bench_utils::Action::SyncAll => { +// let mut updates = Vec::new(); +// for i in 1..actor_num { +// let (a, b) = arref::array_mut_ref!(&mut actors, [0, i]); +// updates.push(b.encode_from(a.vv_cloned())); +// } +// for update in updates { +// // TODO: use import batch here +// actors[0].decode(&update).unwrap(); +// } +// for i in 1..actor_num { +// let (a, b) = arref::array_mut_ref!(&mut actors, [0, i]); +// b.decode(&a.encode_from(b.vv_cloned())).unwrap(); +// } +// } +// } +// } +// drop(profiler); +// } + +pub fn main() { + // let args: Vec<_> = std::env::args().collect(); + // if args.len() < 2 { + // apply_automerge(1); + // return; + // } + + // match args[1].as_str() { + // "automerge" => { + // apply_automerge(1); + // } + // "100_concurrent" => { + // concurrent_actors(100); + // } + // "200_concurrent" => { + // concurrent_actors(200); + // } + // "10_actor_sync_1000_actions" => realtime_sync(10, 1000), + // "20_actor_sync_1000_actions" => realtime_sync(20, 1000), + // "10_actor_sync_2000_actions" => realtime_sync(10, 2000), + // _ => { + // panic!("Unknown command `{}`", args.join(" ")); + // } + // } +} diff --git a/vendor/loro-internal/examples/obs.rs b/vendor/loro-internal/examples/obs.rs new file mode 100644 index 00000000..fee6d7d1 --- /dev/null +++ b/vendor/loro-internal/examples/obs.rs @@ -0,0 +1,26 @@ +use bench_utils::TextAction; + +use loro_internal::{cursor::PosType, LoroDoc}; + +fn main() { + let actions = bench_utils::get_automerge_actions(); + let start = std::time::Instant::now(); + for _ in 0..10 { + let loro = LoroDoc::default(); + let text = loro.get_text("text"); + // loro.subscribe_deep(Arc::new(move |event| { + // black_box(event); + // })); + for TextAction { pos, ins, del } in actions.iter() { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + } + + text.diagnose(); + } + + println!("time: {:?}", start.elapsed()); +} diff --git a/vendor/loro-internal/examples/pending.rs b/vendor/loro-internal/examples/pending.rs new file mode 100644 index 00000000..a2e5bae8 --- /dev/null +++ b/vendor/loro-internal/examples/pending.rs @@ -0,0 +1,33 @@ +use bench_utils::TextAction; +use loro_internal::{cursor::PosType, encoding::ExportMode, LoroDoc, VersionVector}; + +pub fn main() { + let loro = LoroDoc::default(); + let mut latest_vv = VersionVector::default(); + let mut updates = vec![]; + let actions = bench_utils::get_automerge_actions(); + let action_length = actions.len(); + let text = loro.get_text("text"); + for chunks in actions.chunks(action_length / 10) { + for TextAction { pos, ins, del } in chunks { + let mut txn = loro.txn().unwrap(); + text.delete_with_txn(&mut txn, *pos, *del, PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, *pos, ins, PosType::Unicode) + .unwrap(); + let update = loro.export(ExportMode::updates(&latest_vv)).unwrap(); + updates.push(update); + latest_vv = loro.oplog_vv(); + } + } + + println!("done encoding"); + updates.reverse(); + let start = std::time::Instant::now(); + let store2 = LoroDoc::default(); + store2.detach(); + for update in updates.iter() { + store2.import(update).unwrap(); + } + println!("Elapsed {}", start.elapsed().as_millis()); +} diff --git a/vendor/loro-internal/examples/state_size.rs b/vendor/loro-internal/examples/state_size.rs new file mode 100644 index 00000000..099025de --- /dev/null +++ b/vendor/loro-internal/examples/state_size.rs @@ -0,0 +1,127 @@ +use loro_common::{LoroResult, LoroValue}; +use loro_internal::{loro::ExportMode, LoroDoc}; +use rand::{thread_rng, Rng}; + +#[derive(Debug, Clone, Copy)] +enum Kind { + Map, + List, + MovableList, + Text, + Tree, + Counter, +} + +fn empty_state() -> Vec { + let doc = LoroDoc::new_auto_commit(); + doc.export(ExportMode::StateOnly(None)).unwrap() +} + +fn create_elem_sequentially(n: usize, kind: Kind) -> LoroResult> { + let doc = LoroDoc::new_auto_commit(); + for i in 0..n { + match kind { + Kind::Map => { + doc.get_map("map").insert(&i.to_string(), i as u32)?; + } + Kind::List => { + doc.get_list("list").push(i as u32)?; + } + Kind::MovableList => { + doc.get_movable_list("movable_list") + .push(LoroValue::from(i as u32))?; + } + Kind::Text => { + let text = doc.get_text("text"); + text.insert_unicode(text.len_unicode(), &i.to_string())?; + } + Kind::Tree => { + doc.get_tree("tree").enable_fractional_index(0); + doc.get_tree("tree") + .create(loro_internal::TreeParentId::Root)?; + } + Kind::Counter => { + #[cfg(feature = "counter")] + { + doc.get_counter("counter").increment(i as f64)?; + } + } + } + } + let bytes = doc.export(ExportMode::StateOnly(None)).unwrap(); + Ok(bytes) +} + +fn create_elem_randomly(n: usize, kind: Kind) -> LoroResult> { + let doc = LoroDoc::new_auto_commit(); + let mut rng = thread_rng(); + for _ in 0..n / 5 { + for _ in 0..5 { + match kind { + Kind::Map => { + let i = rng.gen::() % 1000; + doc.get_map("map").insert(&i.to_string(), i as u32)?; + } + Kind::List => { + let list = doc.get_list("list"); + let i = rng.gen::() % (list.len() + 1); + list.insert(i, i as u32)?; + } + Kind::MovableList => { + let list = doc.get_movable_list("movable_list"); + let i = rng.gen::() % (list.len() + 1); + list.insert(i, i as u32)?; + } + Kind::Text => { + let text = doc.get_text("text"); + let i = rng.gen::() % (text.len_unicode() + 1); + text.insert_unicode(i, &i.to_string())?; + } + Kind::Tree => { + doc.get_tree("tree").enable_fractional_index(0); + doc.get_tree("tree") + .create(loro_internal::TreeParentId::Root)?; + } + Kind::Counter => { + #[cfg(feature = "counter")] + { + doc.get_counter("counter").increment(rng.gen::())?; + } + } + } + } + doc.set_peer_id(rng.gen::()).unwrap(); + } + let bytes = doc.export(ExportMode::StateOnly(None)).unwrap(); + Ok(bytes) +} + +fn main() -> LoroResult<()> { + for kind in [ + Kind::Map, + Kind::List, + Kind::MovableList, + Kind::Text, + #[allow(unused)] + Kind::Tree, + #[cfg(feature = "counter")] + Kind::Counter, + ] { + println!("============================="); + println!("{kind:?}"); + println!("============================="); + + for (title, bytes) in [ + ("empty", empty_state()), + ("10 items", create_elem_sequentially(10, kind)?), + ( + "1000 items sequentially", + create_elem_sequentially(1000, kind)?, + ), + ("1000 items randomly", create_elem_randomly(1000, kind)?), + ] { + println!("{}: {} bytes", title, bytes.len()) + } + } + Ok(()) +} diff --git a/vendor/loro-internal/examples/tree.rs b/vendor/loro-internal/examples/tree.rs new file mode 100644 index 00000000..bc2c2d66 --- /dev/null +++ b/vendor/loro-internal/examples/tree.rs @@ -0,0 +1,83 @@ +use std::time::Instant; + +use loro_internal::{encoding::ExportMode, LoroDoc, TreeParentId}; +use rand::{rngs::StdRng, Rng}; + +#[allow(unused)] +fn checkout() { + let depth = 300; + let loro = LoroDoc::new_auto_commit(); + let tree = loro.get_tree("tree"); + let mut ids = vec![]; + let mut versions = vec![]; + let id1 = tree.create_at(TreeParentId::Root, 0).unwrap(); + ids.push(id1); + versions.push(loro.oplog_frontiers()); + for _ in 1..depth { + let id = tree + .create_at(TreeParentId::Node(*ids.last().unwrap()), 0) + .unwrap(); + ids.push(id); + versions.push(loro.oplog_frontiers()); + } + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + + for _ in 0..1000 { + let i = rng.gen::() % depth; + let f = &versions[i]; + loro.checkout(f).unwrap(); + } +} + +#[allow(unused)] +fn mov() { + let loro = LoroDoc::new_auto_commit(); + let tree = loro.get_tree("tree"); + let mut ids = vec![]; + let size = 10000; + for _ in 0..size { + ids.push(tree.create_at(TreeParentId::Root, 0).unwrap()) + } + let mut rng: StdRng = rand::SeedableRng::seed_from_u64(0); + let n = 100000; + + for _ in 0..n { + let i = rng.gen::() % size; + let j = rng.gen::() % size; + let children_num = tree.children_num(&TreeParentId::Node(ids[j])).unwrap_or(0); + tree.move_to(ids[i], TreeParentId::Node(ids[j]), children_num) + .unwrap_or_default(); + } + println!( + "encode snapshot size {:?}", + loro.export(ExportMode::Snapshot).unwrap().len() + ); + println!( + "encode updates size {:?}", + loro.export(ExportMode::all_updates()).unwrap().len() + ); +} + +#[allow(unused)] +fn create() { + let size = 100000; + let loro = LoroDoc::default(); + let tree = loro.get_tree("tree"); + for _ in 0..size { + tree.create_at(TreeParentId::Root, 0).unwrap(); + } + println!( + "encode snapshot size {:?}\n", + loro.export(ExportMode::Snapshot).unwrap().len() + ); + println!( + "encode updates size {:?}", + loro.export(ExportMode::all_updates()).unwrap().len() + ); +} + +fn main() { + let s = Instant::now(); + mov(); + println!("{} ms", s.elapsed().as_millis()); +} diff --git a/vendor/loro-internal/proptest-regressions/dag/test.txt b/vendor/loro-internal/proptest-regressions/dag/test.txt new file mode 100644 index 00000000..54efd860 --- /dev/null +++ b/vendor/loro-internal/proptest-regressions/dag/test.txt @@ -0,0 +1,10 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 6fa5e507d2f2f20cb7f5d8a40762b5dfbe9832884ba1ec4fdd98afb3618a1495 # shrinks to before_merged_insertions = [], after_merged_insertions = [] +cc 33ec71c1f62792545519d525dcd8cdd8a4a666640aada85fc78bfe2d2b92591d # shrinks to mut interactions = [Interaction { dag_idx: 0, merge_with: None, len: 1 }, Interaction { dag_idx: 1, merge_with: None, len: 2 }, Interaction { dag_idx: 1, merge_with: Some(0), len: 1 }] +cc cbbf51fd3f0bad25aede701c73321a1ead8738e3052655595bd29aa378bc0655 # shrinks to interactions = [Interaction { dag_idx: 1, merge_with: None, len: 3 }, Interaction { dag_idx: 1, merge_with: None, len: 3 }, Interaction { dag_idx: 4, merge_with: Some(1), len: 1 }, Interaction { dag_idx: 0, merge_with: None, len: 1 }, Interaction { dag_idx: 2, merge_with: Some(0), len: 1 }, Interaction { dag_idx: 0, merge_with: None, len: 1 }, Interaction { dag_idx: 2, merge_with: Some(0), len: 1 }, Interaction { dag_idx: 3, merge_with: Some(0), len: 1 }] +cc eee5bb7b82641d1eab5da8ab36320aafb72768c9cafb531c0b16f37061d56527 # shrinks to before_merged_insertions = [Interaction { dag_idx: 0, merge_with: None, len: 1 }], after_merged_insertions = [Interaction { dag_idx: 0, merge_with: None, len: 1 }] diff --git a/vendor/loro-internal/scripts/fuzz.ts b/vendor/loro-internal/scripts/fuzz.ts new file mode 100644 index 00000000..86f7f100 --- /dev/null +++ b/vendor/loro-internal/scripts/fuzz.ts @@ -0,0 +1,36 @@ +import * as path from "https://deno.land/std@0.105.0/path/mod.ts"; +const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); +import { resolve } from "https://deno.land/std@0.198.0/path/mod.ts"; + +const validTargets = Array.from( + Deno.readDirSync(resolve(__dirname, "../fuzz/fuzz_targets")), +).map((x) => x.name.replace(/.rs$/, "")); + +const targets = + Deno.args.length === 0 + ? validTargets + : Deno.args.filter((x) => validTargets.includes(x)); + +const promises = []; +for (const target of targets) { + const cmd = [ + "cargo", + "+nightly", + "fuzz", + "run", + target, + "--", + "-max_total_time=1", + ]; + console.log("🔨" + cmd.join(" ")); + promises.push( + Deno.run({ + cmd, + stdout: "inherit", + stderr: "inherit", + cwd: resolve(__dirname, ".."), + }).status(), + ); +} + +await Promise.allSettled(promises); diff --git a/vendor/loro-internal/scripts/mem.ts b/vendor/loro-internal/scripts/mem.ts new file mode 100644 index 00000000..384ba2b5 --- /dev/null +++ b/vendor/loro-internal/scripts/mem.ts @@ -0,0 +1,49 @@ +import * as path from "https://deno.land/std@0.105.0/path/mod.ts"; +const __dirname = path.dirname(path.fromFileUrl(import.meta.url)); +import { resolve } from "https://deno.land/std@0.105.0/path/mod.ts"; + +export const Tasks = [ + "100_concurrent", + "200_concurrent", + "automerge", + "10_actor_sync_1000_actions", + "20_actor_sync_1000_actions", + "10_actor_sync_2000_actions", +]; + +export interface Result { + task: string; + maxBytes: number; + endBytes: number; +} + +// run `cargo run --example mem -r -- ${task}` +export async function run(task: string): Promise { + const cmd = `cargo run --example mem -r -- ${task}`; + const process = Deno.run({ + cmd: cmd.split(" "), + cwd: resolve(__dirname, ".."), + stdout: "piped", + stderr: "piped", + }); + + const output = new TextDecoder().decode(await process.stderrOutput()); + try { + // extract "2,555,555" from `dhat: At t-gmax: 2,555,555 bytes` + const maxBytes = parseInt( + output.match(/dhat: At t-gmax:\s+((\d+,?)+) bytes/)![1].replace(/,/g, ""), + ); + const endBytes = parseInt( + output.match(/dhat: At t-end:\s+((\d+,?)+) bytes/)![1].replace(/,/g, ""), + ); + return { + task, + maxBytes, + endBytes, + }; + } catch (e) { + console.error(e); + console.log(output); + throw e; + } +} diff --git a/vendor/loro-internal/scripts/run_mem.ts b/vendor/loro-internal/scripts/run_mem.ts new file mode 100644 index 00000000..c1476b93 --- /dev/null +++ b/vendor/loro-internal/scripts/run_mem.ts @@ -0,0 +1,9 @@ +import { run, Tasks } from "./mem.ts"; + +const output = []; +for (const task of Tasks) { + const result = await run(task); + output.push(result); +} + +console.log(output); diff --git a/vendor/loro-internal/src/allocation/bfs.rs b/vendor/loro-internal/src/allocation/bfs.rs new file mode 100644 index 00000000..47fae081 --- /dev/null +++ b/vendor/loro-internal/src/allocation/bfs.rs @@ -0,0 +1,386 @@ +#![allow(dead_code)] +use crate::{ + dag::{Dag, DagNode}, + id::ID, + version::Frontiers, +}; + +use rustc_hash::FxHashSet; +use std::collections::BinaryHeap; + +#[derive(Debug, PartialEq, Eq)] +struct SortBase { + id: ID, + lamport: u32, +} + +impl PartialOrd for SortBase { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SortBase { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport.cmp(&other.lamport) + } +} + +pub struct BfsBody { + queue: BinaryHeap, + visited: FxHashSet, +} + +pub fn calc_critical_version_bfs>( + graph: &D, + start_list: &Frontiers, +) -> Vec { + let mut runner = BfsBody::new(); + runner.run(graph, start_list) +} + +impl BfsBody { + fn new() -> Self { + Self { + queue: BinaryHeap::new(), + visited: FxHashSet::default(), + } + } + + fn run>(&mut self, graph: &D, start_list: &Frontiers) -> Vec { + let mut start_end_set: FxHashSet = start_list.iter().collect(); + for start in start_list.iter() { + self.queue.push(SortBase { + id: start, + lamport: graph.get(start).unwrap().lamport(), + }); + } + let mut result: Vec = Vec::new(); + while let Some(SortBase { id, lamport: _ }) = self.queue.pop() { + if self.queue.is_empty() { + result.push(id); + } + let node = graph.get(id).unwrap(); + if node.deps().is_empty() { + start_end_set.insert(id); + } else { + for to_id in node.deps().iter() { + if self.visited.contains(&to_id) { + continue; + } + self.visited.insert(to_id); + self.queue.push(SortBase { + id: to_id, + lamport: graph.get(to_id).unwrap().lamport(), + }); + } + } + } + result + .iter() + .filter(|id| !start_end_set.contains(id)) + .cloned() + .collect() + } +} + +#[cfg(test)] +mod additional_tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rle::{HasLength, Sliceable}; + + use super::*; + use crate::{ + change::Lamport, + span::{HasLamport, HasLamportSpan}, + version::VersionVector, + }; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + 1 + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes.get(&id).cloned() + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.nodes.contains_key(&id) + } + } + + fn node(peer: u64, counter: i32, lamport: Lamport, deps: Frontiers) -> TestNode { + TestNode { + id: ID::new(peer, counter), + lamport, + deps, + } + } + + #[test] + fn sort_base_prioritizes_larger_lamport_in_binary_heap() { + let mut heap = BinaryHeap::new(); + heap.push(SortBase { + id: ID::new(1, 0), + lamport: 10, + }); + heap.push(SortBase { + id: ID::new(2, 0), + lamport: 20, + }); + + assert_eq!(heap.pop().unwrap().id, ID::new(2, 0)); + assert_eq!(heap.pop().unwrap().id, ID::new(1, 0)); + } + + #[test] + fn bfs_critical_versions_include_linear_cut_points() { + let root = node(1, 0, 0, Frontiers::default()); + let middle = node(1, 1, 1, root.id.into()); + let head = node(1, 2, 2, middle.id.into()); + let dag = TestDag::new( + vec![root.clone(), middle.clone(), head.clone()], + head.id.into(), + ); + + let critical = calc_critical_version_bfs(&dag, &head.id.into()); + assert_eq!(critical, vec![middle.id]); + } + + #[test] + fn bfs_critical_versions_skip_start_and_leaf_nodes_in_diamond_graphs() { + let root = node(1, 0, 0, Frontiers::default()); + let left = node(2, 0, 1, root.id.into()); + let right = node(3, 0, 2, root.id.into()); + let merge = node(4, 0, 3, Frontiers::from([left.id, right.id])); + let dag = TestDag::new( + vec![root.clone(), left.clone(), right.clone(), merge.clone()], + merge.id.into(), + ); + + let critical = calc_critical_version_bfs(&dag, &merge.id.into()); + assert!(critical.is_empty()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + sync::Arc, + }; + + use crate::{ + change::Lamport, + id::{Counter, PeerID}, + span::{HasId, HasLamport}, + }; + use rle::{HasLength, Sliceable}; + + #[derive(Debug, Clone, PartialEq, Eq)] + struct TestNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Arc, + } + + impl TestNode { + fn new(id: ID, lamport: Lamport, deps: Frontiers) -> Self { + Self { + id, + lamport, + len: 1, + deps: Arc::new(deps), + } + } + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + #[derive(Debug)] + struct TestDag { + nodes: HashMap>, + version_vec: crate::version::VersionVector, + } + + impl TestDag { + fn new(nodes: Vec) -> Self { + let mut map: HashMap> = HashMap::new(); + let mut vv = crate::version::VersionVector::new(); + for node in nodes { + vv.insert(node.id.peer, node.id.counter + node.len as Counter); + map.entry(node.id.peer).or_default().push(node); + } + for nodes in map.values_mut() { + nodes.sort_by(|a, b| match a.id.counter.cmp(&b.id.counter) { + Ordering::Equal => a.len.cmp(&b.len), + other => other, + }); + } + Self { + nodes: map, + version_vec: vv, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + let arr = self.nodes.get(&id.peer)?; + arr.binary_search_by(|node| { + if node.id.counter > id.counter { + Ordering::Greater + } else if node.id.counter + node.len as i32 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + .ok() + .map(|idx| arr[idx].clone()) + } + + fn frontier(&self) -> &Frontiers { + panic!("frontier is not used in bfs tests") + } + + fn vv(&self) -> &crate::version::VersionVector { + &self.version_vec + } + + fn contains(&self, id: ID) -> bool { + self.version_vec.includes_id(id) + } + } + + fn id(peer: PeerID, counter: Counter) -> ID { + ID::new(peer, counter) + } + + fn frontier(ids: &[ID]) -> Frontiers { + let mut frontier = Frontiers::new(); + for id in ids { + frontier.push(*id); + } + frontier + } + + #[test] + fn dedupes_shared_dependencies_and_filters_terminal_nodes() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 8, frontier(&[id(4, 0)])), + TestNode::new(id(3, 0), 7, frontier(&[id(4, 0)])), + TestNode::new(id(4, 0), 1, Frontiers::new()), + ]); + + let result = calc_critical_version_bfs(&graph, &frontier(&[id(1, 0)])); + + assert!(result.is_empty()); + assert_eq!(graph.get(id(4, 0)).unwrap().deps().len(), 0); + } +} diff --git a/vendor/loro-internal/src/allocation/dfs.rs b/vendor/loro-internal/src/allocation/dfs.rs new file mode 100644 index 00000000..da41c5ca --- /dev/null +++ b/vendor/loro-internal/src/allocation/dfs.rs @@ -0,0 +1,416 @@ +use std::collections::HashSet; + +use crate::{ + dag::{Dag, DagNode}, + id::ID, + version::Frontiers, +}; + +fn get_all_points>(graph: &D, points: &mut HashSet, current: &ID) { + points.insert(*current); + for to_id in graph.get(*current).unwrap().deps().iter() { + get_all_points(graph, points, &to_id); + } +} + +pub fn get_end_list>(graph: &D, start_list: &Frontiers) -> Frontiers { + let mut end_set: HashSet = HashSet::new(); + for start_id in start_list.iter() { + end_dfs(graph, &start_id, &mut end_set); + } + end_set.into_iter().collect() +} + +fn end_dfs>(graph: &D, current: &ID, end_set: &mut HashSet) { + let binding = graph.get(*current).unwrap(); + let deps = binding.deps(); + if deps.is_empty() { + end_set.insert(*current); + } + for to_id in deps.iter() { + end_dfs(graph, &to_id, end_set); + } +} + +pub fn calc_critical_version_dfs>( + graph: &D, + start_list: &Frontiers, + end_list: &Frontiers, +) -> Vec { + let mut result: Vec = vec![]; + let mut points: HashSet = HashSet::new(); + let start_list_set: HashSet = HashSet::from_iter(start_list.iter()); + let end_list_set: HashSet = HashSet::from_iter(end_list.iter()); + for start_id in start_list.iter() { + get_all_points(graph, &mut points, &start_id); + } + for escape in points { + let mut flag = false; + for start_id in start_list.iter() { + if dfs(graph, &start_id, &escape, &end_list_set) { + flag = true; + break; + } + } + if flag { + continue; + } + if !end_list_set.contains(&escape) && !start_list_set.contains(&escape) { + result.push(escape); + } + } + result +} + +fn dfs>( + graph: &D, + current: &ID, + escape: &ID, + end_list_set: &HashSet, +) -> bool { + if current == escape { + return false; + } + if end_list_set.contains(current) { + return true; + } + for to_id in graph.get(*current).unwrap().deps().iter() { + if dfs(graph, &to_id, escape, end_list_set) { + return true; + } + } + false +} + +#[cfg(test)] +mod additional_tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rle::{HasLength, Sliceable}; + + use super::*; + use crate::{ + change::Lamport, + span::{HasLamport, HasLamportSpan}, + version::VersionVector, + }; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + 1 + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes.get(&id).cloned() + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.nodes.contains_key(&id) + } + } + + fn node(peer: u64, counter: i32, lamport: Lamport, deps: Frontiers) -> TestNode { + TestNode { + id: ID::new(peer, counter), + lamport, + deps, + } + } + + #[test] + fn end_list_collects_dependency_leaves_from_all_start_frontiers() { + let a = node(1, 0, 0, Frontiers::default()); + let b = node(1, 1, 1, a.id.into()); + let c = node(2, 0, 2, a.id.into()); + let d = node(3, 0, 3, Frontiers::from([b.id, c.id])); + let e = node(4, 0, 4, c.id.into()); + let dag = TestDag::new( + vec![a.clone(), b.clone(), c.clone(), d.clone(), e.clone()], + Frontiers::from([d.id, e.id]), + ); + + let ends = get_end_list(&dag, &Frontiers::from([d.id, e.id])); + assert_eq!(ends, Frontiers::from(a.id)); + } + + #[test] + fn dfs_critical_versions_include_linear_cut_points() { + let root = node(1, 0, 0, Frontiers::default()); + let middle = node(1, 1, 1, root.id.into()); + let head = node(1, 2, 2, middle.id.into()); + let dag = TestDag::new( + vec![root.clone(), middle.clone(), head.clone()], + head.id.into(), + ); + + let critical = calc_critical_version_dfs(&dag, &head.id.into(), &root.id.into()); + assert_eq!(critical, vec![middle.id]); + } + + #[test] + fn dfs_critical_versions_exclude_diamond_branches_with_alternate_paths() { + let root = node(1, 0, 0, Frontiers::default()); + let left = node(2, 0, 1, root.id.into()); + let right = node(3, 0, 2, root.id.into()); + let merge = node(4, 0, 3, Frontiers::from([left.id, right.id])); + let dag = TestDag::new( + vec![root.clone(), left.clone(), right.clone(), merge.clone()], + merge.id.into(), + ); + + let critical = calc_critical_version_dfs(&dag, &merge.id.into(), &root.id.into()); + assert!(critical.is_empty()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + cmp::Ordering, + collections::{HashMap, HashSet}, + sync::Arc, + }; + + use crate::{ + change::Lamport, + id::{Counter, PeerID}, + span::{HasId, HasLamport}, + }; + use rle::{HasLength, Sliceable}; + + #[derive(Debug, Clone, PartialEq, Eq)] + struct TestNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Arc, + } + + impl TestNode { + fn new(id: ID, lamport: Lamport, deps: Frontiers) -> Self { + Self { + id, + lamport, + len: 1, + deps: Arc::new(deps), + } + } + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + #[derive(Debug)] + struct TestDag { + nodes: HashMap>, + version_vec: crate::version::VersionVector, + } + + impl TestDag { + fn new(nodes: Vec) -> Self { + let mut map: HashMap> = HashMap::new(); + let mut vv = crate::version::VersionVector::new(); + for node in nodes { + vv.insert(node.id.peer, node.id.counter + node.len as Counter); + map.entry(node.id.peer).or_default().push(node); + } + for nodes in map.values_mut() { + nodes.sort_by(|a, b| match a.id.counter.cmp(&b.id.counter) { + Ordering::Equal => a.len.cmp(&b.len), + other => other, + }); + } + Self { + nodes: map, + version_vec: vv, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + let arr = self.nodes.get(&id.peer)?; + arr.binary_search_by(|node| { + if node.id.counter > id.counter { + Ordering::Greater + } else if node.id.counter + node.len as i32 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + .ok() + .map(|idx| arr[idx].clone()) + } + + fn frontier(&self) -> &Frontiers { + panic!("frontier is not used in dfs tests") + } + + fn vv(&self) -> &crate::version::VersionVector { + &self.version_vec + } + + fn contains(&self, id: ID) -> bool { + self.version_vec.includes_id(id) + } + } + + fn id(peer: PeerID, counter: Counter) -> ID { + ID::new(peer, counter) + } + + fn frontier(ids: &[ID]) -> Frontiers { + let mut frontier = Frontiers::new(); + for id in ids { + frontier.push(*id); + } + frontier + } + + fn as_set(ids: Vec) -> HashSet { + ids.into_iter().collect() + } + + #[test] + fn get_end_list_collects_all_leaf_nodes_reachable_from_start() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 7, Frontiers::new()), + TestNode::new(id(3, 0), 8, Frontiers::new()), + ]); + + let ends = get_end_list(&graph, &frontier(&[id(1, 0)])); + + assert_eq!(ends.len(), 2); + assert!(ends.contains(&id(2, 0))); + assert!(ends.contains(&id(3, 0))); + assert!(!ends.contains(&id(1, 0))); + } + + #[test] + fn calc_critical_version_dfs_returns_non_start_nodes_when_no_end_is_present() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 7, Frontiers::new()), + TestNode::new(id(3, 0), 8, Frontiers::new()), + ]); + + let result = calc_critical_version_dfs(&graph, &frontier(&[id(1, 0)]), &Frontiers::new()); + + assert_eq!(as_set(result), as_set(vec![id(2, 0), id(3, 0)])); + } + + #[test] + fn calc_critical_version_dfs_skips_candidates_when_an_end_is_on_every_start_path() { + let graph = TestDag::new(vec![ + TestNode::new(id(1, 0), 10, frontier(&[id(2, 0), id(3, 0)])), + TestNode::new(id(2, 0), 7, Frontiers::new()), + TestNode::new(id(3, 0), 8, Frontiers::new()), + ]); + + let result = + calc_critical_version_dfs(&graph, &frontier(&[id(1, 0)]), &frontier(&[id(2, 0)])); + + assert!(result.is_empty()); + } +} diff --git a/vendor/loro-internal/src/allocation/mod.rs b/vendor/loro-internal/src/allocation/mod.rs new file mode 100644 index 00000000..be34ae5e --- /dev/null +++ b/vendor/loro-internal/src/allocation/mod.rs @@ -0,0 +1,10 @@ +#![allow(dead_code)] +#![allow(unused)] + +mod bfs; +pub(crate) use bfs::calc_critical_version_bfs as calc_critical_version; + +// Only for testing +mod dfs; +pub(crate) use dfs::calc_critical_version_dfs; +pub(crate) use dfs::get_end_list; diff --git a/vendor/loro-internal/src/arena.rs b/vendor/loro-internal/src/arena.rs new file mode 100644 index 00000000..8f1d36c6 --- /dev/null +++ b/vendor/loro-internal/src/arena.rs @@ -0,0 +1,827 @@ +mod str_arena; +use self::str_arena::{StrArena, StrArenaCheckpoint}; +use crate::sync::{Mutex, MutexGuard, RwLock, RwLockWriteGuard}; +use crate::{ + change::Lamport, + container::{ + idx::ContainerIdx, + list::list_op::{InnerListOp, ListOp}, + map::MapSet, + ContainerID, + }, + id::Counter, + op::{InnerContent, ListSlice, Op, RawOp, RawOpContent, SliceRange}, + LoroValue, +}; +use append_only_bytes::BytesSlice; +use loro_common::PeerID; +use rustc_hash::FxHashMap; +use std::fmt; +use std::{ + num::NonZeroU16, + ops::{Range, RangeBounds}, + sync::Arc, +}; + +pub(crate) struct LoadAllFlag; +type ParentResolver = dyn Fn(ContainerID) -> Option + Send + Sync + 'static; + +#[derive(Default)] +struct ArenaContainers { + container_idx_to_id: Vec, + // 0 stands for unknown, 1 stands for root containers + depth: Vec>, + container_id_to_idx: FxHashMap, + /// The parent of each container. + parents: FxHashMap>, + /// All retention roots: top-level user roots **and** mergeable cids. Used by + /// alive-container / shallow-snapshot retention walks that must see both. + root_c_idx: Vec, + /// Subset of `root_c_idx` containing only top-level (non-mergeable) roots. This is the + /// list user-facing APIs enumerate (`preferred_root_containers`, `get_value`, + /// `get_deep_value`, jsonpath, ...). Keeping it pre-filtered means those paths do + /// not pay a per-mergeable `is_mergeable()` parse on every call. + top_level_root_c_idx: Vec, + /// Optional resolver used when querying parent for a container that has not been registered yet. + /// If set, `get_parent` will try this resolver to lazily fetch and register the parent. + parent_resolver: Option>, +} + +#[derive(Default)] +struct InnerSharedArena { + // Container metadata is a single consistency domain. Keep it under one + // mutex so container id/index/parent/depth updates cannot acquire locks in + // inconsistent orders. + containers: RwLock, + values: Mutex>, + str: Arc>, +} + +impl fmt::Debug for InnerSharedArena { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InnerSharedArena") + .field("containers", &">") + .field("values", &">") + .field("str", &">>") + .finish() + } +} + +/// This is shared between [OpLog] and [AppState]. +/// +#[derive(Debug, Clone)] +pub struct SharedArena { + inner: Arc, +} + +pub(crate) struct SharedArenaRollback { + container_len: usize, + root_len: usize, + top_level_root_len: usize, + values_len: usize, + str: StrArenaCheckpoint, +} + +#[derive(Debug)] +pub struct StrAllocResult { + /// unicode start + pub start: usize, + /// unicode end + pub end: usize, +} + +impl ArenaContainers { + /// Add a freshly-registered cid to the retention-root tracking vectors. + /// + /// Centralizes the `root_c_idx ⊇ top_level_root_c_idx` invariant: `root_c_idx` is the set of + /// retention roots (top-level user roots **and** mergeable cids) that seed the alive-walk for + /// shallow snapshot; `top_level_root_c_idx` is the user-visible subset enumerated by + /// `preferred_root_containers` etc. Every push to either vector should go through this method, + /// so the invariant is maintained by construction rather than by remembering to push to two + /// vectors at every call site. + fn push_root(&mut self, idx: ContainerIdx, is_mergeable: bool) { + self.root_c_idx.push(idx); + if !is_mergeable { + self.top_level_root_c_idx.push(idx); + } + } + + fn register_container(&mut self, id: &ContainerID) -> ContainerIdx { + if let Some(&idx) = self.container_id_to_idx.get(id) { + return idx; + } + + let idx = self.container_idx_to_id.len(); + self.container_idx_to_id.push(id.clone()); + let idx = ContainerIdx::from_index_and_type(idx as u32, id.container_type()); + self.container_id_to_idx.insert(id.clone(), idx); + // Resolve the cid's kind once. `is_mergeable` is non-trivial for mergeable names, + // so we avoid calling it twice on the same id. + let mergeable_parts = if id.is_root() { + id.parse_mergeable() + } else { + None + }; + match (id.is_root(), mergeable_parts) { + (true, None) => { + self.push_root(idx, false); + self.parents.insert(idx, None); + self.depth.push(NonZeroU16::new(1)); + } + (true, Some((parent_id, _key, _kind))) => { + // Mergeable Roots are retention roots AND logical children. Push to `root_c_idx` + // so shallow snapshot's `retain_keys` does not GC the loser of a concurrent-kind + // conflict (whose state is reachable only by its deterministic cid, not through + // the parent marker); then `set_parent` for path/event resolution. They are + // deliberately absent from `top_level_root_c_idx` so user-facing root + // enumeration does not have to filter them out. + self.push_root(idx, true); + self.depth.push(None); + let parent_idx = self.register_container(&parent_id); + self.set_parent(idx, Some(parent_idx)); + } + (false, _) => { + self.depth.push(None); + } + } + idx + } + + fn set_parent(&mut self, child: ContainerIdx, parent: Option) { + self.parents.insert(child, parent); + + match parent { + Some(p) => { + if let Some(d) = self.get_depth(p) { + self.depth[child.to_index() as usize] = NonZeroU16::new(d.get() + 1); + } else { + self.depth[child.to_index() as usize] = None; + } + } + None => { + self.depth[child.to_index() as usize] = NonZeroU16::new(1); + } + } + } + + fn get_depth(&mut self, target: ContainerIdx) -> Option { + if let Some(d) = self.depth[target.to_index() as usize] { + return Some(d); + } + + let parent: Option = if let Some(p) = self.parents.get(&target) { + *p + } else { + let id = self + .container_idx_to_id + .get(target.to_index() as usize) + .unwrap() + .clone(); + if id.is_root() && !id.is_mergeable() { + None + } else { + // Mergeable Roots get registered via the main `register_container` path, which + // already wires their parent edge — they should never reach here with a missing + // `parents` entry. But for ordinary children whose parent isn't in the arena + // yet, fall back to the resolver. + let resolver = self.parent_resolver.clone()?; + let parent_id = resolver(id)?; + // Route through `register_container` so a freshly-discovered parent (especially + // a mergeable cid) lands in `root_c_idx`, gets its own parent edge wired up, + // and has `parents` initialized — instead of being half-registered with just + // `idx_to_id` / `id_to_idx` / `depth` and silently missing from retention walks. + let parent_idx = if let Some(idx) = + self.container_id_to_idx.get(&parent_id).copied() + { + // Already in the arena. For a top-level root that was hand-registered + // somewhere else without a `parents` entry, ensure it has one. + if parent_id.is_root() && !parent_id.is_mergeable() { + self.parents.entry(idx).or_insert(None); + if self.depth[idx.to_index() as usize].is_none() { + self.depth[idx.to_index() as usize] = NonZeroU16::new(1); + } + } + idx + } else { + self.register_container(&parent_id) + }; + + Some(parent_idx) + } + }; + + let d = match parent { + Some(p) => NonZeroU16::new(self.get_depth(p)?.get() + 1), + None => NonZeroU16::new(1), + }; + self.depth[target.to_index() as usize] = d; + d + } + + fn container_id(&self, idx: ContainerIdx) -> Option { + self.container_idx_to_id + .get(idx.to_index() as usize) + .cloned() + } +} + +pub(crate) struct ArenaGuards<'a> { + containers: RwLockWriteGuard<'a, ArenaContainers>, +} + +impl ArenaGuards<'_> { + pub fn register_container(&mut self, id: &ContainerID) -> ContainerIdx { + self.containers.register_container(id) + } + + pub fn set_parent(&mut self, child: ContainerIdx, parent: Option) { + self.containers.set_parent(child, parent); + } +} + +impl SharedArena { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self { + inner: Arc::new(InnerSharedArena::default()), + } + } + + pub fn fork(&self) -> Self { + Self { + inner: Arc::new(InnerSharedArena { + containers: RwLock::new({ + let containers = self.inner.containers.read(); + ArenaContainers { + container_idx_to_id: containers.container_idx_to_id.clone(), + depth: containers.depth.clone(), + container_id_to_idx: containers.container_id_to_idx.clone(), + parents: containers.parents.clone(), + root_c_idx: containers.root_c_idx.clone(), + top_level_root_c_idx: containers.top_level_root_c_idx.clone(), + parent_resolver: containers.parent_resolver.clone(), + } + }), + values: Mutex::new(self.inner.values.lock().clone()), + str: self.inner.str.clone(), + }), + } + } + + pub(crate) fn checkpoint_for_rollback(&self) -> SharedArenaRollback { + let containers = self.inner.containers.read(); + let container_len = containers.container_idx_to_id.len(); + let root_len = containers.root_c_idx.len(); + let top_level_root_len = containers.top_level_root_c_idx.len(); + drop(containers); + let values_len = self.inner.values.lock().len(); + let str = self.inner.str.lock().checkpoint(); + SharedArenaRollback { + container_len, + root_len, + top_level_root_len, + values_len, + str, + } + } + + pub(crate) fn rollback(&self, checkpoint: SharedArenaRollback) { + let mut containers = self.inner.containers.write(); + let removed_ids = containers + .container_idx_to_id + .split_off(checkpoint.container_len); + for id in removed_ids { + containers.container_id_to_idx.remove(&id); + } + containers.depth.truncate(checkpoint.container_len); + containers.root_c_idx.truncate(checkpoint.root_len); + containers + .top_level_root_c_idx + .truncate(checkpoint.top_level_root_len); + containers.parents.retain(|child, parent| { + let child_is_kept = (child.to_index() as usize) < checkpoint.container_len; + let parent_is_kept = parent + .map(|p| (p.to_index() as usize) < checkpoint.container_len) + .unwrap_or(true); + child_is_kept && parent_is_kept + }); + drop(containers); + + self.inner.values.lock().truncate(checkpoint.values_len); + self.inner.str.lock().rollback(checkpoint.str); + } + + pub(crate) fn with_guards(&self, f: impl FnOnce(&mut ArenaGuards)) { + let mut guards = self.get_arena_guards(); + f(&mut guards); + } + + fn get_arena_guards(&self) -> ArenaGuards<'_> { + ArenaGuards { + containers: self.inner.containers.write(), + } + } + + pub fn register_container(&self, id: &ContainerID) -> ContainerIdx { + self.inner.containers.write().register_container(id) + } + + pub fn get_container_id(&self, idx: ContainerIdx) -> Option { + self.inner.containers.read().container_id(idx) + } + + /// Fast map from `ContainerID` to `ContainerIdx` for containers already registered + /// in the arena. + /// + /// Important: This is not an existence check. Absence here does not imply that a + /// container does not exist, since registration can be lazy and containers may + /// be persisted only in the state KV store until first use. + /// + /// For existence-aware lookup that consults persisted state and performs lazy + /// registration, prefer `DocState::resolve_idx`. + pub fn id_to_idx(&self, id: &ContainerID) -> Option { + self.inner + .containers + .read() + .container_id_to_idx + .get(id) + .copied() + } + + #[inline] + pub fn idx_to_id(&self, id: ContainerIdx) -> Option { + self.inner.containers.read().container_id(id) + } + + #[inline] + pub fn with_idx_to_id(&self, f: impl FnOnce(&Vec) -> R) -> R { + let containers = self.inner.containers.read(); + f(&containers.container_idx_to_id) + } + + pub fn alloc_str(&self, str: &str) -> StrAllocResult { + let mut text_lock = self.inner.str.lock(); + _alloc_str(&mut text_lock, str) + } + + /// return slice and unicode index + pub fn alloc_str_with_slice(&self, str: &str) -> (BytesSlice, StrAllocResult) { + let mut text_lock = self.inner.str.lock(); + _alloc_str_with_slice(&mut text_lock, str) + } + + /// alloc str without extra info + pub fn alloc_str_fast(&self, bytes: &[u8]) { + let mut text_lock = self.inner.str.lock(); + text_lock.alloc(std::str::from_utf8(bytes).unwrap()); + } + + #[inline] + pub fn utf16_len(&self) -> usize { + self.inner.str.lock().len_utf16() + } + + #[inline] + pub fn alloc_value(&self, value: LoroValue) -> usize { + let mut values_lock = self.inner.values.lock(); + _alloc_value(&mut values_lock, value) + } + + #[inline] + pub fn alloc_values(&self, values: impl Iterator) -> std::ops::Range { + let mut values_lock = self.inner.values.lock(); + _alloc_values(&mut values_lock, values) + } + + #[inline] + pub fn set_parent(&self, child: ContainerIdx, parent: Option) { + self.inner.containers.write().set_parent(child, parent); + } + + pub fn log_hierarchy(&self) { + if cfg!(debug_assertions) { + let containers = self.inner.containers.read(); + for (c, p) in containers.parents.iter() { + tracing::info!( + "container {:?} {:?} {:?}", + c, + containers.container_id(*c), + p.and_then(|x| containers.container_id(x)) + ); + } + } + } + + pub fn log_all_containers(&self) { + let containers = self.inner.containers.read(); + containers.container_id_to_idx.iter().for_each(|(id, idx)| { + tracing::info!("container {:?} {:?}", id, idx); + }); + containers + .container_idx_to_id + .iter() + .enumerate() + .for_each(|(i, id)| { + tracing::info!("container {} {:?}", i, id); + }); + } + + pub fn get_parent(&self, child: ContainerIdx) -> Option { + let (child_id, resolver) = { + let containers = self.inner.containers.read(); + let child_id = containers.container_id(child).unwrap(); + if child_id.is_root() && !child_id.is_mergeable() { + // TODO: PERF: we can speed this up by use a special bit in ContainerIdx to indicate + // whether the target is a root container + return None; + } + + // Try fast path first + if let Some(p) = containers.parents.get(&child).copied() { + return p; + } + + // Fallback: try to resolve parent lazily via the resolver if provided. + (child_id, containers.parent_resolver.clone()) + }; + if let Some(resolver) = resolver { + if let Some(parent_id) = resolver(child_id.clone()) { + let parent_idx = self.register_container(&parent_id); + self.set_parent(child, Some(parent_idx)); + return Some(parent_idx); + } + } + + panic!("InternalError: Parent is not registered") + } + + /// Call `f` on each ancestor of `container`, including `container` itself. + /// + /// f(ContainerIdx, is_first) + pub fn with_ancestors(&self, container: ContainerIdx, mut f: impl FnMut(ContainerIdx, bool)) { + let mut container = Some(container); + let mut is_first = true; + while let Some(c) = container { + f(c, is_first); + is_first = false; + container = self.get_parent(c) + } + } + + #[inline] + pub fn slice_by_unicode(&self, range: impl RangeBounds) -> BytesSlice { + self.inner.str.lock().slice_by_unicode(range) + } + + #[inline] + pub fn slice_by_utf8(&self, range: impl RangeBounds) -> BytesSlice { + self.inner.str.lock().slice_bytes(range) + } + + #[inline] + pub fn slice_str_by_unicode_range(&self, range: Range) -> String { + let mut s = self.inner.str.lock(); + let s: &mut StrArena = &mut s; + let mut ans = String::with_capacity(range.len()); + ans.push_str(s.slice_str_by_unicode(range)); + ans + } + + #[inline] + pub fn with_text_slice(&self, range: Range, mut f: impl FnMut(&str)) { + f(self.inner.str.lock().slice_str_by_unicode(range)) + } + + #[inline] + pub fn get_value(&self, idx: usize) -> Option { + self.inner.values.lock().get(idx).cloned() + } + + #[inline] + pub fn get_values(&self, range: Range) -> Vec { + (self.inner.values.lock()[range]).to_vec() + } + + pub fn convert_single_op( + &self, + container: &ContainerID, + peer: PeerID, + counter: Counter, + lamport: Lamport, + content: RawOpContent, + ) -> Op { + let container = self.register_container(container); + self.inner_convert_op(content, peer, counter, lamport, container) + } + + pub fn can_import_snapshot(&self) -> bool { + let str_empty = self.inner.str.lock().is_empty(); + let values_empty = self.inner.values.lock().is_empty(); + str_empty && values_empty + } + + fn inner_convert_op( + &self, + content: RawOpContent<'_>, + _peer: PeerID, + counter: i32, + _lamport: Lamport, + container: ContainerIdx, + ) -> Op { + match content { + crate::op::RawOpContent::Map(MapSet { key, value }) => Op { + counter, + container, + content: crate::op::InnerContent::Map(MapSet { key, value }), + }, + crate::op::RawOpContent::List(list) => match list { + ListOp::Insert { slice, pos } => match slice { + ListSlice::RawData(values) => { + let range = self.alloc_values(values.iter().cloned()); + Op { + counter, + container, + content: crate::op::InnerContent::List(InnerListOp::Insert { + slice: SliceRange::from(range.start as u32..range.end as u32), + pos, + }), + } + } + ListSlice::RawStr { str, unicode_len } => { + let (slice, info) = self.alloc_str_with_slice(&str); + Op { + counter, + container, + content: crate::op::InnerContent::List(InnerListOp::InsertText { + slice, + unicode_start: info.start as u32, + unicode_len: unicode_len as u32, + pos: pos as u32, + }), + } + } + }, + ListOp::Delete(span) => Op { + counter, + container, + content: crate::op::InnerContent::List(InnerListOp::Delete(span)), + }, + ListOp::StyleStart { + start, + end, + info, + key, + value, + } => Op { + counter, + container, + content: InnerContent::List(InnerListOp::StyleStart { + start, + end, + key, + info, + value, + }), + }, + ListOp::StyleEnd => Op { + counter, + container, + content: InnerContent::List(InnerListOp::StyleEnd), + }, + ListOp::Move { + from, + to, + elem_id: from_id, + } => Op { + counter, + container, + content: InnerContent::List(InnerListOp::Move { + from, + to, + elem_id: from_id, + }), + }, + ListOp::Set { elem_id, value } => Op { + counter, + container, + content: InnerContent::List(InnerListOp::Set { elem_id, value }), + }, + }, + crate::op::RawOpContent::Tree(tree) => Op { + counter, + container, + content: crate::op::InnerContent::Tree(tree.clone()), + }, + #[cfg(feature = "counter")] + crate::op::RawOpContent::Counter(c) => Op { + counter, + container, + content: crate::op::InnerContent::Future(crate::op::FutureInnerContent::Counter(c)), + }, + crate::op::RawOpContent::Unknown { prop, value } => Op { + counter, + container, + content: crate::op::InnerContent::Future(crate::op::FutureInnerContent::Unknown { + prop, + value: Box::new(value), + }), + }, + } + } + + #[inline] + pub fn convert_raw_op(&self, op: &RawOp) -> Op { + self.inner_convert_op( + op.content.clone(), + op.id.peer, + op.id.counter, + op.lamport, + op.container, + ) + } + + #[inline] + pub fn export_containers(&self) -> Vec { + self.inner.containers.read().container_idx_to_id.clone() + } + + pub fn export_parents(&self) -> Vec> { + let containers = self.inner.containers.read(); + containers + .container_idx_to_id + .iter() + .enumerate() + .map(|(x, id)| { + let idx = ContainerIdx::from_index_and_type(x as u32, id.container_type()); + let parent_idx = containers.parents.get(&idx)?; + *parent_idx + }) + .collect() + } + + /// Returns all the possible root containers of the docs + /// + /// We need to load all the cached kv in DocState before we can ensure all root contains are covered. + /// So we need the flag type here. + /// + /// This includes mergeable cids (which are also retention roots). Callers that only want + /// user-visible top-level roots should use [`Self::top_level_root_containers`]. + #[inline] + pub(crate) fn root_containers(&self, _f: LoadAllFlag) -> Vec { + self.inner.containers.read().root_c_idx.clone() + } + + /// Returns only the user-visible top-level root containers (excludes mergeable cids). + /// + /// Used by `preferred_root_containers`, `get_value` / `get_deep_value`, jsonpath, etc. — + /// any path that enumerates the doc's top-level roots. Pre-filtering at registration time + /// keeps these calls O(top_level_roots) instead of O(top_level_roots + mergeable_cids). + #[inline] + pub(crate) fn top_level_root_containers(&self, _f: LoadAllFlag) -> Vec { + self.inner.containers.read().top_level_root_c_idx.clone() + } + + // TODO: this can return a u16 directly now, since the depths are always valid + pub(crate) fn get_depth(&self, container: ContainerIdx) -> Option { + self.inner.containers.write().get_depth(container) + } + + pub(crate) fn iter_value_slice( + &self, + range: Range, + ) -> impl Iterator + '_ { + let values = self.inner.values.lock(); + range + .into_iter() + .map(move |i| values.get(i).unwrap().clone()) + } + + #[allow(unused)] + pub(crate) fn log_all_values(&self) { + let values = self.inner.values.lock(); + for (i, v) in values.iter().enumerate() { + loro_common::debug!("value {} {:?}", i, v); + } + } +} + +fn _alloc_str_with_slice( + text_lock: &mut MutexGuard<'_, StrArena>, + str: &str, +) -> (BytesSlice, StrAllocResult) { + let start = text_lock.len_bytes(); + let ans = _alloc_str(text_lock, str); + (text_lock.slice_bytes(start..), ans) +} + +fn _alloc_values( + values_lock: &mut MutexGuard<'_, Vec>, + values: impl Iterator, +) -> Range { + values_lock.reserve(values.size_hint().0); + let start = values_lock.len(); + for value in values { + values_lock.push(value); + } + + start..values_lock.len() +} + +fn _alloc_value(values_lock: &mut MutexGuard<'_, Vec>, value: LoroValue) -> usize { + values_lock.push(value); + values_lock.len() - 1 +} + +fn _alloc_str(text_lock: &mut MutexGuard<'_, StrArena>, str: &str) -> StrAllocResult { + let start = text_lock.len_unicode(); + text_lock.alloc(str); + StrAllocResult { + start, + end: text_lock.len_unicode(), + } +} + +fn _slice_str(range: Range, s: &mut StrArena) -> String { + let mut ans = String::with_capacity(range.len()); + ans.push_str(s.slice_str_by_unicode(range)); + ans +} + +impl SharedArena { + /// Register or clear a resolver to lazily determine a container's parent when missing. + /// + /// - The resolver receives the child `ContainerIdx` and returns an optional `ContainerID` of its parent. + /// - If the resolver returns `Some`, `SharedArena` will register the parent in the arena and link it. + /// - If the resolver is `None` or returns `None`, `get_parent` will panic for non-root containers as before. + pub fn set_parent_resolver(&self, resolver: Option) + where + F: Fn(ContainerID) -> Option + Send + Sync + 'static, + { + self.inner.containers.write().parent_resolver = + resolver.map(|f| Arc::new(f) as Arc); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use loro_common::ContainerType; + + /// When a non-mergeable child is registered, then later asks for its depth and the parent + /// resolver yields a mergeable cid, the mergeable parent must be registered through the + /// retention-root path (push to `root_c_idx`, recursively link its own parent) — exactly the + /// same way the main `register_container` path would handle a freshly-encountered mergeable + /// cid. Without this, shallow snapshot's alive-walk would silently GC the mergeable parent's + /// state. + #[test] + fn get_depth_resolver_registers_mergeable_parent_as_retention_root() { + let arena = SharedArena::new(); + + let top_root = ContainerID::new_root("state", ContainerType::Map); + let mergeable_parent = + ContainerID::new_mergeable(&top_root, "profile", ContainerType::Map); + let child = ContainerID::new_normal( + loro_common::ID::new(1, 0), + ContainerType::Counter, + ); + + let mergeable_parent_for_resolver = mergeable_parent.clone(); + let child_for_resolver = child.clone(); + arena.set_parent_resolver(Some(move |q: ContainerID| { + if q == child_for_resolver { + Some(mergeable_parent_for_resolver.clone()) + } else { + None + } + })); + + let child_idx = arena.register_container(&child); + let _ = arena.get_depth(child_idx); + + let flag = LoadAllFlag; + let mergeable_idx = arena + .id_to_idx(&mergeable_parent) + .expect("resolver should have registered the mergeable parent"); + assert!( + arena.root_containers(flag).contains(&mergeable_idx), + "mergeable parent discovered via resolver must be a retention root" + ); + + let flag = LoadAllFlag; + assert!( + !arena.top_level_root_containers(flag).contains(&mergeable_idx), + "mergeable parent must not appear in user-facing top-level enumeration" + ); + + let flag = LoadAllFlag; + let top_root_idx = arena + .id_to_idx(&top_root) + .expect("mergeable parent's own grandparent must also be registered"); + assert!( + arena + .top_level_root_containers(flag) + .contains(&top_root_idx), + "the mergeable parent's grandparent (a top-level root) must be in the top-level list" + ); + } +} diff --git a/vendor/loro-internal/src/arena/str_arena.rs b/vendor/loro-internal/src/arena/str_arena.rs new file mode 100644 index 00000000..db7de2a4 --- /dev/null +++ b/vendor/loro-internal/src/arena/str_arena.rs @@ -0,0 +1,248 @@ +use std::ops::{Bound, RangeBounds}; + +use append_only_bytes::{AppendOnlyBytes, BytesSlice}; + +use crate::container::richtext::richtext_state::unicode_to_utf8_index; +const INDEX_INTERVAL: u32 = 128; + +#[derive(Default, Debug, Clone)] +pub(crate) struct StrArena { + bytes: AppendOnlyBytes, + unicode_indexes: Vec, + len: Index, +} + +#[derive(Debug, Clone)] +pub(crate) struct StrArenaCheckpoint { + bytes_len: usize, + unicode_indexes_len: usize, + len: Index, +} + +#[derive(Debug, Default, Clone, Copy)] +struct Index { + bytes: u32, + utf16: u32, + unicode: u32, +} + +impl StrArena { + #[inline] + pub fn is_empty(&self) -> bool { + self.len.bytes == 0 + } + + #[inline] + #[allow(dead_code)] + pub fn len_bytes(&self) -> usize { + self.len.bytes as usize + } + + #[inline] + pub fn len_utf16(&self) -> usize { + self.len.utf16 as usize + } + + #[inline] + pub fn len_unicode(&self) -> usize { + self.len.unicode as usize + } + + pub fn alloc(&mut self, input: &str) { + let mut utf16 = 0; + let mut unicode_len = 0; + let mut last_save_index = 0; + for (byte_index, c) in input.char_indices() { + let byte_index = byte_index + c.len_utf8(); + utf16 += c.len_utf16() as u32; + unicode_len += 1; + if byte_index - last_save_index > INDEX_INTERVAL as usize { + self._alloc(&input[last_save_index..byte_index], utf16, unicode_len); + last_save_index = byte_index; + utf16 = 0; + unicode_len = 0; + } + } + + if last_save_index != input.len() { + self._alloc(&input[last_save_index..], utf16, unicode_len); + } + } + + fn _alloc(&mut self, input: &str, utf16: u32, unicode_len: i32) { + let s = input; + self.len.bytes += s.len() as u32; + self.len.utf16 += utf16; + self.len.unicode += unicode_len as u32; + self.bytes.push_str(s); + let cur_len = self.len; + + let index = &mut self.unicode_indexes; + if index.is_empty() { + index.push(Index { + bytes: 0, + utf16: 0, + unicode: 0, + }); + } + + let last = index.last().unwrap(); + if cur_len.bytes - last.bytes > INDEX_INTERVAL { + self.unicode_indexes.push(cur_len); + } + } + + #[inline] + pub fn slice_by_unicode(&mut self, range: impl RangeBounds) -> BytesSlice { + let (start, end) = self.unicode_range_to_utf8_range(range); + self.bytes.slice(start..end) + } + + #[inline] + pub fn slice_str_by_unicode(&mut self, range: impl RangeBounds) -> &str { + let (start, end) = self.unicode_range_to_utf8_range(range); + // SAFETY: we know that the range is valid + unsafe { std::str::from_utf8_unchecked(&self.bytes[start..end]) } + } + + fn unicode_range_to_utf8_range(&mut self, range: impl RangeBounds) -> (usize, usize) { + if self.is_empty() { + return (0, 0); + } + + let start = match range.start_bound() { + Bound::Included(&i) => { + unicode_to_byte_index(&self.unicode_indexes, i as u32, &self.bytes) + } + Bound::Excluded(&_i) => unreachable!(), + Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + Bound::Included(&i) => { + unicode_to_byte_index(&self.unicode_indexes, i as u32 + 1, &self.bytes) + } + Bound::Excluded(&i) => { + unicode_to_byte_index(&self.unicode_indexes, i as u32, &self.bytes) + } + Bound::Unbounded => self.len.bytes as usize, + }; + + (start, end) + } + + #[inline] + pub fn slice_bytes(&self, range: impl RangeBounds) -> BytesSlice { + self.bytes.slice(range) + } + + #[inline] + pub(crate) fn checkpoint(&self) -> StrArenaCheckpoint { + StrArenaCheckpoint { + bytes_len: self.bytes.len(), + unicode_indexes_len: self.unicode_indexes.len(), + len: self.len, + } + } + + pub(crate) fn rollback(&mut self, checkpoint: StrArenaCheckpoint) { + if checkpoint.bytes_len == 0 { + *self = Self::default(); + return; + } + + if checkpoint.bytes_len < self.bytes.len() { + let mut bytes = AppendOnlyBytes::with_capacity(checkpoint.bytes_len); + bytes.push_slice(&self.bytes[..checkpoint.bytes_len]); + self.bytes = bytes; + } + self.unicode_indexes + .truncate(checkpoint.unicode_indexes_len); + self.len = checkpoint.len; + } +} + +fn unicode_to_byte_index(index: &[Index], unicode_index: u32, bytes: &AppendOnlyBytes) -> usize { + let i = match index.binary_search_by_key(&unicode_index, |x| x.unicode) { + Ok(i) => i, + Err(i) => i - 1, + }; + + let index = index[i]; + if index.unicode == unicode_index { + return index.bytes as usize; + } + + // SAFETY: we know that the index must be valid, because we record and calculate the valid index + let s = unsafe { std::str::from_utf8_unchecked(&bytes[index.bytes as usize..]) }; + unicode_to_utf8_index(s, (unicode_index - index.unicode) as usize).unwrap() + + index.bytes as usize +} + +#[cfg(test)] +mod test { + use std::ops::Deref; + + use super::*; + + #[test] + fn test() { + let mut arena = StrArena::default(); + arena.alloc("Hello"); + let slice = arena.slice_by_unicode(0..5); + assert_eq!(slice.deref(), b"Hello"); + arena.alloc("World"); + let slice = arena.slice_by_unicode(5..10); + assert_eq!(slice.deref(), b"World"); + } + + #[test] + fn parse_unicode_correctly() { + let mut arena = StrArena::default(); + arena.alloc("Hello"); + arena.alloc("World"); + arena.alloc("你好"); + arena.alloc("世界"); + let slice = arena.slice_by_unicode(0..4); + assert_eq!(slice.deref(), b"Hell"); + + let slice = arena.slice_by_unicode(4..8); + assert_eq!(slice.deref(), b"oWor"); + + let slice = arena.slice_by_unicode(8..10); + assert_eq!(slice.deref(), b"ld"); + + let slice = arena.slice_by_unicode(10..12); + assert_eq!(slice.deref(), "你好".as_bytes()); + + let slice = arena.slice_by_unicode(12..14); + assert_eq!(slice.deref(), "世界".as_bytes()); + } + + #[test] + fn slice_long_unicode_correctly() { + let mut arena = StrArena::default(); + let src = "一二34567八九零"; + for s in std::iter::repeat_n(src, 100) { + arena.alloc(s); + } + + let slice = arena.slice_by_unicode(110..120); + assert_eq!(slice.deref(), src.as_bytes()); + let slice = arena.slice_by_unicode(111..121); + assert_eq!(slice.deref(), "二34567八九零一".as_bytes()); + } + + #[test] + fn rollback_truncates_bytes_without_losing_prefix() { + let mut arena = StrArena::default(); + arena.alloc("hello"); + let checkpoint = arena.checkpoint(); + arena.alloc(" transient"); + arena.rollback(checkpoint); + arena.alloc("!"); + + let slice = arena.slice_by_unicode(..); + assert_eq!(slice.deref(), b"hello!"); + } +} diff --git a/vendor/loro-internal/src/awareness.rs b/vendor/loro-internal/src/awareness.rs new file mode 100644 index 00000000..581b2783 --- /dev/null +++ b/vendor/loro-internal/src/awareness.rs @@ -0,0 +1,873 @@ +//! Ephemeral presence utilities. +//! +//! `EphemeralStore` is the recommended API: a timestamped, last-write-wins key-value +//! store for transient presence data (cursors, selections, etc.). It supports: +//! - Per-key timeouts: expired entries are skipped by `encode`/`encode_all` and removed +//! by `remove_outdated`. +//! - Import/export via `encode`/`apply` for syncing between peers. +//! - Subscriptions for both local updates (raw bytes to send) and merged updates. +//! +//! The legacy `Awareness` type remains for backward compatibility but is deprecated in +//! favor of `EphemeralStore`. +use crate::sync::Mutex; +use std::sync::atomic::AtomicI64; +use std::sync::Arc; + +use loro_common::{LoroValue, PeerID}; +use rustc_hash::FxHashMap; +use serde::de::{DeserializeSeed, EnumAccess, MapAccess, SeqAccess, VariantAccess}; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::change::{get_sys_timestamp, Timestamp}; +use crate::{SubscriberSetWithQueue, Subscription}; + +/// `Awareness` is a structure that tracks the ephemeral state of peers. +/// +/// It can be used to synchronize cursor positions, selections, and the names of the peers. +/// +/// The state of a specific peer is expected to be removed after a specified timeout. Use +/// `remove_outdated` to eliminate outdated states. +#[derive(Debug, Clone)] +#[deprecated(since = "1.4.6", note = "Use `EphemeralStore` instead.")] +pub struct Awareness { + peer: PeerID, + peers: FxHashMap, + timeout: i64, +} + +#[derive(Debug, Clone)] +pub struct PeerInfo { + pub state: LoroValue, + pub counter: i32, + // This field is generated locally + pub timestamp: i64, +} + +#[derive(Serialize, Deserialize)] +struct EncodedPeerInfo { + peer: PeerID, + counter: i32, + record: LoroValue, +} + +#[derive(Deserialize)] +struct DecodedPeerInfo { + peer: PeerID, + counter: i32, + #[serde(deserialize_with = "deserialize_depth_limited_loro_value")] + record: LoroValue, +} + +#[allow(deprecated)] +impl Awareness { + pub fn new(peer: PeerID, timeout: i64) -> Awareness { + Awareness { + peer, + timeout, + peers: FxHashMap::default(), + } + } + + pub fn encode(&self, peers: &[PeerID]) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + for peer in peers { + if let Some(peer_info) = self.peers.get(peer) { + if now - peer_info.timestamp > self.timeout { + continue; + } + + let encoded_peer_info = EncodedPeerInfo { + peer: *peer, + record: peer_info.state.clone(), + counter: peer_info.counter, + }; + peers_info.push(encoded_peer_info); + } + } + + postcard::to_allocvec(&peers_info).unwrap() + } + + pub fn encode_all(&self) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + for (peer, peer_info) in self.peers.iter() { + if now - peer_info.timestamp > self.timeout { + continue; + } + + let encoded_peer_info = EncodedPeerInfo { + peer: *peer, + record: peer_info.state.clone(), + counter: peer_info.counter, + }; + peers_info.push(encoded_peer_info); + } + + postcard::to_allocvec(&peers_info).unwrap() + } + + /// Try to apply encoded updates imported from another peer/process. + /// + /// Returns (updated, added). + pub fn try_apply( + &mut self, + encoded_peers_info: &[u8], + ) -> Result<(Vec, Vec), Box> { + let peers_info: Vec = postcard::from_bytes(encoded_peers_info) + .map_err(|err| format!("Failed to decode awareness data: {err}").into_boxed_str())?; + let mut changed_peers = Vec::new(); + let mut added_peers = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + for peer_info in peers_info { + match self.peers.get(&peer_info.peer) { + Some(x) if x.counter >= peer_info.counter || peer_info.peer == self.peer => { + // do nothing + } + _ => { + let old = self.peers.insert( + peer_info.peer, + PeerInfo { + counter: peer_info.counter, + state: peer_info.record, + timestamp: now, + }, + ); + if old.is_some() { + changed_peers.push(peer_info.peer); + } else { + added_peers.push(peer_info.peer); + } + } + } + } + + Ok((changed_peers, added_peers)) + } + + /// Returns (updated, added) + pub fn apply(&mut self, encoded_peers_info: &[u8]) -> (Vec, Vec) { + self.try_apply(encoded_peers_info) + .expect("Failed to decode awareness data") + } + + pub fn set_local_state(&mut self, value: impl Into) { + self._set_local_state(value.into()); + } + + fn _set_local_state(&mut self, value: LoroValue) { + let peer = self.peers.entry(self.peer).or_insert_with(|| PeerInfo { + state: Default::default(), + counter: 0, + timestamp: 0, + }); + + peer.state = value; + peer.counter += 1; + peer.timestamp = get_sys_timestamp() as Timestamp; + } + + pub fn get_local_state(&self) -> Option { + self.peers.get(&self.peer).map(|x| x.state.clone()) + } + + pub fn remove_outdated(&mut self) -> Vec { + let now = get_sys_timestamp() as Timestamp; + let mut removed = Vec::new(); + self.peers.retain(|id, v| { + if now - v.timestamp > self.timeout { + removed.push(*id); + false + } else { + true + } + }); + + removed + } + + pub fn get_all_states(&self) -> &FxHashMap { + &self.peers + } + + pub fn peer(&self) -> PeerID { + self.peer + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EphemeralEventTrigger { + Local, + Import, + Timeout, +} + +#[derive(Debug, Clone)] +pub struct EphemeralStoreEvent { + pub by: EphemeralEventTrigger, + pub added: Arc>, + pub updated: Arc>, + pub removed: Arc>, +} + +pub type LocalEphemeralCallback = Box) -> bool + Send + Sync + 'static>; +pub type EphemeralSubscriber = Box bool + Send + Sync + 'static>; + +/// `EphemeralStore` is a structure that tracks ephemeral key-value state across peers. +/// +/// - Use it for syncing lightweight presence/state like cursors, selections, and UI hints. +/// - Each key uses timestamp-based LWW (Last-Write-Wins) conflict resolution. +/// - Timeout unit: milliseconds. +/// - After timeout: entries are considered expired. They are omitted from +/// `encode`/`encode_all`, and calling [`remove_outdated`] will purge them and +/// notify subscribers with `EphemeralEventTrigger::Timeout`. +/// +/// In Rust, you are responsible for periodically calling [`remove_outdated`] +/// if you want timed-out entries to be removed and corresponding events to be +/// emitted. In the WASM/TS wrapper, a timer runs automatically while the store +/// is non-empty. +/// +/// See: https://loro.dev/docs/tutorial/ephemeral +/// +/// # Example +/// +/// ```rust +/// use loro_internal::awareness::EphemeralStore; +/// +/// let mut store = EphemeralStore::new(1000); +/// store.set("key", "value"); +/// let encoded = store.encode("key"); +/// let mut store2 = EphemeralStore::new(1000); +/// store.subscribe_local_updates(Box::new(|data| { +/// println!("local update: {:?}", data); +/// true +/// })); +/// store2.apply(&encoded); +/// assert_eq!(store2.get("key"), Some("value".into())); +/// ``` +#[derive(Debug, Clone)] +pub struct EphemeralStore { + inner: Arc, +} + +impl EphemeralStore { + /// Create a new `EphemeralStore`. + /// + /// - `timeout`: inactivity timeout in milliseconds. If a key does not + /// receive updates within this duration, it is considered expired. It + /// will be skipped by `encode`/`encode_all`, and removed when + /// [`remove_outdated`] is called (triggering a `Timeout` event to + /// subscribers). + pub fn new(timeout: i64) -> Self { + Self { + inner: Arc::new(EphemeralStoreInner::new(timeout)), + } + } + + /// Encode the latest value of `key`. + /// + /// Expired keys (past timeout) are omitted and produce an empty payload. + pub fn encode(&self, key: &str) -> Vec { + self.inner.encode(key) + } + + /// Encode all non-expired keys. + /// + /// Entries that have exceeded the timeout are not included in the payload. + pub fn encode_all(&self) -> Vec { + self.inner.encode_all() + } + + /// Apply encoded updates imported from another peer/process. + /// + /// Subscribers receive an event with `by = Import` and the lists of + /// `added`/`updated`/`removed` keys as appropriate. + pub fn apply(&self, data: &[u8]) -> Result<(), Box> { + self.inner.apply(data) + } + + pub fn set(&self, key: &str, value: impl Into) { + self.inner.set(key, value) + } + + pub fn delete(&self, key: &str) { + self.inner.delete(key) + } + + pub fn get(&self, key: &str) -> Option { + self.inner.get(key) + } + + /// Remove entries whose last update timestamp has exceeded the timeout. + /// + /// If any keys are removed, subscribers receive an event with + /// `by = Timeout` and the `removed` keys. This does not run automatically + /// in Rust; call it on your own schedule. + pub fn remove_outdated(&self) { + self.inner.remove_outdated() + } + + pub fn get_all_states(&self) -> FxHashMap { + self.inner.get_all_states() + } + + pub fn keys(&self) -> Vec { + self.inner.keys() + } + + /// Subscribe to local ephemeral/awareness updates. + /// + /// The callback receives encoded update bytes whenever local ephemeral state changes. + /// This is useful for syncing awareness information like cursor positions, selections, + /// or presence data to other peers in real-time. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, providing a convenient way to implement one-time or conditional + /// subscriptions in Rust. + /// + /// # Parameters + /// - `callback`: Function that receives `&Vec` (encoded ephemeral updates) and returns `bool` + /// - Return `true` to keep the subscription active + /// - Return `false` to automatically unsubscribe + /// + /// # Example + /// ```rust + /// use loro_internal::awareness::EphemeralStore; + /// use std::sync::{Arc, Mutex}; + /// + /// let store = EphemeralStore::new(30000); + /// let update_count = Arc::new(Mutex::new(0)); + /// let count_clone = update_count.clone(); + /// + /// // Subscribe and collect first 3 updates, then auto-unsubscribe + /// let sub = store.subscribe_local_updates(Box::new(move |bytes| { + /// println!("Received {} bytes of ephemeral data", bytes.len()); + /// let mut count = count_clone.lock().unwrap(); + /// *count += 1; + /// *count < 3 // Auto-unsubscribe after 3 updates + /// })); + /// + /// store.set("cursor", 42); + /// ``` + pub fn subscribe_local_updates(&self, callback: LocalEphemeralCallback) -> Subscription { + self.inner.subscribe_local_updates(callback) + } + + pub fn subscribe(&self, callback: EphemeralSubscriber) -> Subscription { + self.inner.subscribe(callback) + } +} + +struct EphemeralStoreInner { + states: Mutex>, + local_subs: SubscriberSetWithQueue<(), LocalEphemeralCallback, Vec>, + subscribers: SubscriberSetWithQueue<(), EphemeralSubscriber, EphemeralStoreEvent>, + timeout: AtomicI64, +} + +impl std::fmt::Debug for EphemeralStoreInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "AwarenessV2 {{ states: {:?}, timeout: {:?} }}", + self.states, self.timeout + ) + } +} + +#[derive(Serialize, Deserialize)] +struct EncodedState<'a> { + #[serde(borrow)] + key: &'a str, + value: Option, + timestamp: i64, +} + +#[derive(Deserialize)] +struct DecodedState<'a> { + #[serde(borrow)] + key: &'a str, + #[serde(deserialize_with = "deserialize_optional_depth_limited_loro_value")] + value: Option, + timestamp: i64, +} + +const EPHEMERAL_VALUE_MAX_DEPTH: usize = 512; + +fn deserialize_optional_depth_limited_loro_value<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + Option::::deserialize(deserializer) + .map(|value| value.map(DepthLimitedLoroValue::into_inner)) +} + +fn deserialize_depth_limited_loro_value<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + LoroValueSeed { + remaining_depth: EPHEMERAL_VALUE_MAX_DEPTH, + } + .deserialize(deserializer) +} + +struct DepthLimitedLoroValue(LoroValue); + +impl DepthLimitedLoroValue { + fn into_inner(self) -> LoroValue { + self.0 + } +} + +impl<'de> Deserialize<'de> for DepthLimitedLoroValue { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserialize_depth_limited_loro_value(deserializer).map(Self) + } +} + +#[derive(Clone, Copy)] +struct LoroValueSeed { + remaining_depth: usize, +} + +impl LoroValueSeed { + fn nested(self) -> Result + where + E: serde::de::Error, + { + let remaining_depth = self + .remaining_depth + .checked_sub(1) + .ok_or_else(|| E::custom("LoroValue nesting depth exceeded"))?; + Ok(Self { remaining_depth }) + } +} + +impl<'de> DeserializeSeed<'de> for LoroValueSeed { + type Value = LoroValue; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + if deserializer.is_human_readable() { + return Err(serde::de::Error::custom( + "human-readable LoroValue is not supported by the awareness decoder", + )); + } + + deserializer.deserialize_enum( + "LoroValue", + &[ + "Null", + "Bool", + "Double", + "I32", + "String", + "List", + "Map", + "Container", + "Binary", + ], + LoroValueSeedVisitor { seed: self }, + ) + } +} + +struct LoroValueSeedVisitor { + seed: LoroValueSeed, +} + +impl<'de> serde::de::Visitor<'de> for LoroValueSeedVisitor { + type Value = LoroValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a depth-limited LoroValue") + } + + fn visit_enum(self, data: A) -> Result + where + A: EnumAccess<'de>, + { + match data.variant()? { + (DepthLimitedLoroValueField::Null, v) => { + v.unit_variant()?; + Ok(LoroValue::Null) + } + (DepthLimitedLoroValueField::Bool, v) => v.newtype_variant().map(LoroValue::Bool), + (DepthLimitedLoroValueField::Double, v) => v.newtype_variant().map(LoroValue::Double), + (DepthLimitedLoroValueField::I32, v) => v.newtype_variant().map(LoroValue::I64), + (DepthLimitedLoroValueField::String, v) => v + .newtype_variant() + .map(|value: String| LoroValue::String(value.into())), + (DepthLimitedLoroValueField::List, v) => v + .newtype_variant_seed(LoroListSeed { + value_seed: self.seed.nested()?, + }) + .map(|value| LoroValue::List(value.into())), + (DepthLimitedLoroValueField::Map, v) => v + .newtype_variant_seed(LoroMapSeed { + value_seed: self.seed.nested()?, + }) + .map(|value| LoroValue::Map(value.into())), + (DepthLimitedLoroValueField::Container, v) => { + v.newtype_variant().map(LoroValue::Container) + } + (DepthLimitedLoroValueField::Binary, v) => v + .newtype_variant() + .map(|value: Vec| LoroValue::Binary(value.into())), + } + } +} + +#[derive(Deserialize)] +enum DepthLimitedLoroValueField { + Null, + Bool, + Double, + I32, + String, + List, + Map, + Container, + Binary, +} + +struct LoroListSeed { + value_seed: LoroValueSeed, +} + +impl<'de> DeserializeSeed<'de> for LoroListSeed { + type Value = Vec; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_seq(LoroListVisitor { + value_seed: self.value_seed, + }) + } +} + +struct LoroListVisitor { + value_seed: LoroValueSeed, +} + +impl<'de> serde::de::Visitor<'de> for LoroListVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a depth-limited LoroValue list") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut list = Vec::new(); + while let Some(value) = seq.next_element_seed(self.value_seed)? { + list.push(value); + } + Ok(list) + } +} + +struct LoroMapSeed { + value_seed: LoroValueSeed, +} + +impl<'de> DeserializeSeed<'de> for LoroMapSeed { + type Value = FxHashMap; + + fn deserialize(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_map(LoroMapVisitor { + value_seed: self.value_seed, + }) + } +} + +struct LoroMapVisitor { + value_seed: LoroValueSeed, +} + +impl<'de> serde::de::Visitor<'de> for LoroMapVisitor { + type Value = FxHashMap; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a depth-limited LoroValue map") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut value = FxHashMap::default(); + while let Some(key) = map.next_key()? { + let entry = map.next_value_seed(self.value_seed)?; + value.insert(key, entry); + } + Ok(value) + } +} + +#[derive(Debug, Clone)] +struct State { + state: Option, + timestamp: i64, +} + +impl EphemeralStoreInner { + pub fn new(timeout: i64) -> EphemeralStoreInner { + EphemeralStoreInner { + timeout: AtomicI64::new(timeout), + states: Mutex::new(FxHashMap::default()), + local_subs: SubscriberSetWithQueue::new(), + subscribers: SubscriberSetWithQueue::new(), + } + } + + pub fn encode(&self, key: &str) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + let states = self.states.lock(); + if let Some(peer_state) = states.get(key) { + if now - peer_state.timestamp > self.timeout.load(std::sync::atomic::Ordering::Relaxed) + { + return vec![]; + } + let encoded_peer_info = EncodedState { + key, + value: peer_state.state.clone(), + timestamp: peer_state.timestamp, + }; + peers_info.push(encoded_peer_info); + } + + postcard::to_allocvec(&peers_info).unwrap() + } + + pub fn encode_all(&self) -> Vec { + let mut peers_info = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + let states = self.states.lock(); + for (key, peer_state) in states.iter() { + if now - peer_state.timestamp > self.timeout.load(std::sync::atomic::Ordering::Relaxed) + { + continue; + } + let encoded_peer_info = EncodedState { + key, + value: peer_state.state.clone(), + timestamp: peer_state.timestamp, + }; + peers_info.push(encoded_peer_info); + } + postcard::to_allocvec(&peers_info).unwrap() + } + + pub fn apply(&self, data: &[u8]) -> Result<(), Box> { + let peers_info = match postcard::from_bytes::>(data) { + Ok(ans) => ans, + Err(err) => return Err(format!("Failed to decode data: {}", err).into()), + }; + + let mut updated_keys = Vec::new(); + let mut added_keys = Vec::new(); + let mut removed_keys = Vec::new(); + let now = get_sys_timestamp() as Timestamp; + let timeout = self.timeout.load(std::sync::atomic::Ordering::Relaxed); + let mut states = self.states.lock(); + for DecodedState { + key, + value: record, + timestamp, + } in peers_info + { + if now - timestamp > timeout { + continue; + } + + match states.get_mut(key) { + Some(peer_info) if peer_info.timestamp >= timestamp => { + // do nothing + } + _ => { + let old = states.insert( + key.to_string(), + State { + state: record.clone(), + timestamp, + }, + ); + match (old, record) { + (Some(_), Some(_)) => updated_keys.push(key.to_string()), + (None, Some(_)) => added_keys.push(key.to_string()), + (Some(_), None) => removed_keys.push(key.to_string()), + (None, None) => {} + } + } + } + } + + drop(states); + if !self.subscribers.inner().is_empty() { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Import, + added: Arc::new(added_keys), + updated: Arc::new(updated_keys), + removed: Arc::new(removed_keys), + }, + ); + } + + Ok(()) + } + + pub fn set(&self, key: &str, value: impl Into) { + self._set_local_state(key, Some(value.into())); + } + + pub fn delete(&self, key: &str) { + self._set_local_state(key, None); + } + + pub fn get(&self, key: &str) -> Option { + let states = self.states.lock(); + states.get(key).and_then(|x| x.state.clone()) + } + + pub fn remove_outdated(&self) { + let now = get_sys_timestamp() as Timestamp; + let mut removed = Vec::new(); + let mut states = self.states.lock(); + states.retain(|key, state| { + if now - state.timestamp > self.timeout.load(std::sync::atomic::Ordering::Relaxed) { + if state.state.is_some() { + removed.push(key.clone()); + } + false + } else { + true + } + }); + drop(states); + if !self.subscribers.inner().is_empty() { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Timeout, + added: Arc::new(Vec::new()), + updated: Arc::new(Vec::new()), + removed: Arc::new(removed), + }, + ); + } + } + + pub fn get_all_states(&self) -> FxHashMap { + let states = self.states.lock(); + states + .iter() + .filter(|(_, v)| v.state.is_some()) + .map(|(k, v)| (k.clone(), v.state.clone().unwrap())) + .collect() + } + + pub fn keys(&self) -> Vec { + let states = self.states.lock(); + states + .keys() + .filter(|&k| states.get(k).unwrap().state.is_some()) + .map(|s| s.to_string()) + .collect() + } + + /// Subscribe to local ephemeral state updates. + /// + /// The callback receives encoded update bytes whenever the local peer's ephemeral state + /// is modified. This enables real-time synchronization of awareness data like cursor + /// positions, user presence, or temporary collaborative state. + /// + /// **Auto-unsubscription**: If the callback returns `false`, the subscription will be + /// automatically removed, making it easy to implement conditional or temporary subscriptions. + /// + /// # Parameters + /// - `callback`: Function that receives `&Vec` and returns `bool` + /// - Return `true` to maintain the subscription + /// - Return `false` to automatically unsubscribe + /// + /// # Returns + /// A `Subscription` that can be used to manually unsubscribe if needed. + pub fn subscribe_local_updates(&self, callback: LocalEphemeralCallback) -> Subscription { + let (sub, activate) = self.local_subs.inner().insert((), callback); + activate(); + sub + } + + pub fn subscribe(&self, callback: EphemeralSubscriber) -> Subscription { + let (sub, activate) = self.subscribers.inner().insert((), callback); + activate(); + sub + } + + fn _set_local_state(&self, key: &str, value: Option) { + let is_delete = value.is_none(); + let mut states = self.states.lock(); + let old = states.insert( + key.to_string(), + State { + state: value, + timestamp: get_sys_timestamp() as Timestamp, + }, + ); + + drop(states); + if !self.local_subs.inner().is_empty() { + self.local_subs.emit(&(), self.encode(key)); + } + if !self.subscribers.inner().is_empty() { + if old.is_some() { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Local, + added: Arc::new(Vec::new()), + updated: if !is_delete { + Arc::new(vec![key.to_string()]) + } else { + Arc::new(Vec::new()) + }, + removed: if !is_delete { + Arc::new(Vec::new()) + } else { + Arc::new(vec![key.to_string()]) + }, + }, + ); + } else if !is_delete { + self.subscribers.emit( + &(), + EphemeralStoreEvent { + by: EphemeralEventTrigger::Local, + added: Arc::new(vec![key.to_string()]), + updated: Arc::new(Vec::new()), + removed: Arc::new(Vec::new()), + }, + ); + } + } + } +} diff --git a/vendor/loro-internal/src/change.rs b/vendor/loro-internal/src/change.rs new file mode 100644 index 00000000..72edc4a7 --- /dev/null +++ b/vendor/loro-internal/src/change.rs @@ -0,0 +1,329 @@ +//! [Change]s are merged ops. +//! +//! Every [Change] has deps on other [Change]s. All [Change]s in the document thus form a DAG. +//! +//! Note: `dep` can only point to the end of the other [Change]. This is the invariant of [Change]s. + +use crate::{ + dag::DagNode, + estimated_size::EstimatedSize, + id::{Counter, ID}, + op::Op, + span::{HasId, HasLamport}, + version::Frontiers, +}; +use loro_common::{HasCounter, HasCounterSpan, PeerID}; +use num::traits::AsPrimitive; +use rle::{HasIndex, HasLength, Mergable, RleVec, Sliceable}; +use smallvec::SmallVec; + +pub type Timestamp = i64; +pub type Lamport = u32; + +/// A `Change` contains a list of [Op]s. +/// +/// When undo/redo we should always undo/redo a whole [Change]. +// PERF change slice and getting length is kinda slow I guess +#[derive(Debug, Clone, PartialEq)] +pub struct Change { + /// id of the first op in the change + pub(crate) id: ID, + /// Lamport timestamp of the change. It can be calculated from deps + pub(crate) lamport: Lamport, + pub(crate) deps: Frontiers, + /// [Unix time](https://en.wikipedia.org/wiki/Unix_time) + /// It is the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970. + pub(crate) timestamp: Timestamp, + pub(crate) commit_msg: Option>, + pub(crate) ops: RleVec<[O; 1]>, +} + +pub(crate) struct ChangeRef<'a, O = Op> { + pub(crate) id: &'a ID, + pub(crate) lamport: &'a Lamport, + pub(crate) deps: &'a Frontiers, + pub(crate) timestamp: &'a Timestamp, + pub(crate) commit_msg: &'a Option>, + pub(crate) ops: &'a RleVec<[O; 1]>, +} + +impl<'a, O> ChangeRef<'a, O> { + pub fn from_change(change: &'a Change) -> Self { + Self { + id: &change.id, + lamport: &change.lamport, + deps: &change.deps, + timestamp: &change.timestamp, + commit_msg: &change.commit_msg, + ops: &change.ops, + } + } +} + +impl Change { + pub fn new( + ops: RleVec<[O; 1]>, + deps: Frontiers, + id: ID, + lamport: Lamport, + timestamp: Timestamp, + ) -> Self { + Change { + ops, + deps, + id, + lamport, + timestamp, + commit_msg: None, + } + } + + #[inline] + pub fn ops(&self) -> &RleVec<[O; 1]> { + &self.ops + } + + #[inline] + pub fn deps(&self) -> &Frontiers { + &self.deps + } + + #[inline] + pub fn peer(&self) -> PeerID { + self.id.peer + } + + #[inline] + pub fn lamport(&self) -> Lamport { + self.lamport + } + + #[inline] + pub fn timestamp(&self) -> Timestamp { + self.timestamp + } + + #[inline] + pub fn id(&self) -> ID { + self.id + } + + #[inline] + pub fn deps_on_self(&self) -> bool { + if let Some(id) = self.deps.as_single() { + id.peer == self.id.peer + } else { + false + } + } + + pub fn message(&self) -> Option<&Arc> { + self.commit_msg.as_ref() + } +} + +impl EstimatedSize for Change { + /// Estimate the storage size of the change in bytes + #[inline] + fn estimate_storage_size(&self) -> usize { + let id_size = 2; + let lamport_size = 1; + let timestamp_size = 1; + let deps_size = (self.deps.len().max(1) - 1) * 4; + let ops_size = self + .ops + .iter() + .map(|op| op.estimate_storage_size()) + .sum::(); + id_size + lamport_size + timestamp_size + ops_size + deps_size + } +} + +impl HasIndex for Change { + type Int = Counter; + + fn get_start_index(&self) -> Self::Int { + self.id.counter + } +} + +impl HasId for Change { + fn id_start(&self) -> ID { + self.id + } +} + +impl HasCounter for Change { + fn ctr_start(&self) -> Counter { + self.id.counter + } +} + +impl HasLamport for Change { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl Mergable for Change { + fn is_mergable(&self, _other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + false + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + unreachable!() + } +} + +impl Change { + pub fn len(&self) -> usize { + self.ops.span().as_() + } + + pub fn is_empty(&self) -> bool { + self.ops.is_empty() + } +} + +use std::{fmt::Debug, sync::Arc}; +impl HasLength for Change { + fn content_len(&self) -> usize { + self.ops.span().as_() + } +} + +impl Sliceable for Change { + // TODO: feels slow, need to confirm whether this affects performance + fn slice(&self, from: usize, to: usize) -> Self { + assert!(from < to); + assert!(to <= self.atom_len()); + let from_counter = self.id.counter + from as Counter; + let to_counter = self.id.counter + to as Counter; + let ops = { + if from >= to { + RleVec::new() + } else { + let mut ans: SmallVec<[_; 1]> = SmallVec::new(); + let mut start_index = 0; + if self.ops.len() >= 8 { + let result = self + .ops + .binary_search_by(|op| op.ctr_end().cmp(&from_counter)); + start_index = match result { + Ok(i) => i, + Err(i) => i, + }; + } + + for i in start_index..self.ops.len() { + let op = &self.ops[i]; + if op.ctr_start() >= to_counter { + break; + } + if op.ctr_end() <= from_counter { + continue; + } + + let start_offset = + ((from_counter - op.ctr_start()).max(0) as usize).min(op.atom_len()); + let end_offset = + ((to_counter - op.ctr_start()).max(0) as usize).min(op.atom_len()); + assert_ne!(start_offset, end_offset); + ans.push(op.slice(start_offset, end_offset)) + } + + RleVec::from(ans) + } + }; + assert_eq!(ops.first().unwrap().ctr_start(), from_counter); + assert_eq!(ops.last().unwrap().ctr_end(), to_counter); + Self { + ops, + deps: if from > 0 { + Frontiers::from_id(self.id.inc(from as Counter - 1)) + } else { + self.deps.clone() + }, + id: self.id.inc(from as Counter), + lamport: self.lamport + from as Lamport, + timestamp: self.timestamp, + commit_msg: self.commit_msg.clone(), + } + } +} + +impl DagNode for Change { + fn deps(&self) -> &Frontiers { + &self.deps + } +} + +impl Change { + pub fn can_merge_right(&self, other: &Self, merge_interval: i64) -> bool { + if other.id.peer == self.id.peer + && other.id.counter == self.id.counter + self.content_len() as Counter + && other.deps.len() == 1 + && other.deps.as_single().unwrap().peer == self.id.peer + && other.timestamp - self.timestamp <= merge_interval + && self.commit_msg == other.commit_msg + { + debug_assert!(other.timestamp >= self.timestamp); + debug_assert!(other.lamport == self.lamport + self.len() as Lamport); + true + } else { + false + } + } +} + +/// [Unix time](https://en.wikipedia.org/wiki/Unix_time) +/// It is the number of milliseconds that have elapsed since 00:00:00 UTC on 1 January 1970. +#[cfg(any( + not(target_arch = "wasm32"), + target_os = "emscripten", + target_os = "wasi" +))] +pub(crate) fn get_sys_timestamp() -> f64 { + use std::time::{SystemTime, UNIX_EPOCH}; + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .as_() +} + +/// [Unix time](https://en.wikipedia.org/wiki/Unix_time) +/// It is the number of milliseconds that have elapsed since 00:00:00 UTC on 1 January 1970. +#[cfg(all( + target_arch = "wasm32", + not(target_os = "emscripten"), + not(target_os = "wasi") +))] +pub fn get_sys_timestamp() -> f64 { + use wasm_bindgen::prelude::wasm_bindgen; + #[wasm_bindgen] + extern "C" { + // Use `js_namespace` here to bind `console.log(..)` instead of just + // `log(..)` + #[wasm_bindgen(js_namespace = Date)] + pub fn now() -> f64; + } + + now() +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn size_of_change() { + let size = std::mem::size_of::(); + println!("{}", size); + } +} diff --git a/vendor/loro-internal/src/change_meta.rs b/vendor/loro-internal/src/change_meta.rs new file mode 100644 index 00000000..93e9cbd9 --- /dev/null +++ b/vendor/loro-internal/src/change_meta.rs @@ -0,0 +1,98 @@ +use std::{cmp::Ordering, sync::Arc}; + +use loro_common::HasLamport; +use rle::HasLength; + +use crate::{ + change::{Change, Lamport, Timestamp}, + id::ID, + version::Frontiers, +}; + +/// `Change` is a grouped continuous operations that share the same id, timestamp, commit message. +/// +/// - The id of the `Change` is the id of its first op. +/// - The second op's id is `{ peer: change.id.peer, counter: change.id.counter + 1 }` +/// +/// The same applies on `Lamport`: +/// +/// - The lamport of the `Change` is the lamport of its first op. +/// - The second op's lamport is `change.lamport + 1` +/// +/// The length of the `Change` is how many operations it contains +#[derive(Debug, Clone)] +pub struct ChangeMeta { + /// Lamport timestamp of the Change + pub lamport: Lamport, + /// The first Op id of the Change + pub id: ID, + /// [Unix time](https://en.wikipedia.org/wiki/Unix_time) + /// It is the number of seconds that have elapsed since 00:00:00 UTC on 1 January 1970. + pub timestamp: Timestamp, + /// The commit message of the change + pub message: Option>, + /// The dependencies of the first op of the change + pub deps: Frontiers, + /// The total op num inside this change + pub len: usize, +} + +impl PartialEq for ChangeMeta { + fn eq(&self, other: &Self) -> bool { + self.lamport == other.lamport && self.id == other.id + } +} + +impl Eq for ChangeMeta {} + +impl PartialOrd for ChangeMeta { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for ChangeMeta { + fn cmp(&self, other: &Self) -> Ordering { + (self.lamport + self.len as Lamport) + .cmp(&(other.lamport + other.len as Lamport)) + .then(self.id.peer.cmp(&other.id.peer)) + } +} + +impl HasLamport for ChangeMeta { + fn lamport(&self) -> loro_common::Lamport { + self.lamport + } +} + +impl HasLength for ChangeMeta { + fn content_len(&self) -> usize { + self.len + } +} + +impl ChangeMeta { + pub fn from_change(c: &Change) -> Self { + Self { + id: c.id(), + lamport: c.lamport(), + timestamp: c.timestamp(), + message: c.message().cloned(), + deps: c.deps().clone(), + len: c.len(), + } + } + + /// Get the commit message in &str + pub fn message(&self) -> &str { + match self.message.as_ref() { + Some(m) => m, + None => "", + } + } + + /// Get the commit timestamp in seconds since Unix epoch. + pub fn timestamp(&self) -> crate::change::Timestamp { + self.timestamp + } +} diff --git a/vendor/loro-internal/src/configure.rs b/vendor/loro-internal/src/configure.rs new file mode 100644 index 00000000..e4fe5e86 --- /dev/null +++ b/vendor/loro-internal/src/configure.rs @@ -0,0 +1,291 @@ +use crate::sync::{Mutex, RwLock}; +use loro_common::ContainerID; +use rustc_hash::FxHashSet; + +pub use crate::container::richtext::config::{StyleConfig, StyleConfigMap}; +use crate::LoroDoc; +use std::sync::atomic::{AtomicBool, AtomicI64}; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct Configure { + pub(crate) text_style_config: Arc>, + record_timestamp: Arc, + pub(crate) merge_interval_in_s: Arc, + pub(crate) editable_detached_mode: Arc, + pub(crate) deleted_root_containers: Arc>>, + pub(crate) hide_empty_root_containers: Arc, +} + +impl LoroDoc { + pub(crate) fn set_config(&self, config: &Configure) { + self.config_text_style(config.text_style_config.read().clone()); + self.set_record_timestamp(config.record_timestamp()); + self.set_change_merge_interval(config.merge_interval()); + self.set_detached_editing(config.detached_editing()); + } +} + +impl Default for Configure { + fn default() -> Self { + Self { + text_style_config: Arc::new(RwLock::new(StyleConfigMap::default_rich_text_config())), + record_timestamp: Arc::new(AtomicBool::new(false)), + editable_detached_mode: Arc::new(AtomicBool::new(false)), + merge_interval_in_s: Arc::new(AtomicI64::new(1000)), + deleted_root_containers: Arc::new(Mutex::new(Default::default())), + hide_empty_root_containers: Arc::new(AtomicBool::new(false)), + } + } +} + +impl Configure { + pub fn fork(&self) -> Self { + Self { + text_style_config: Arc::new(RwLock::new(self.text_style_config.read().clone())), + record_timestamp: Arc::new(AtomicBool::new( + self.record_timestamp + .load(std::sync::atomic::Ordering::Relaxed), + )), + merge_interval_in_s: Arc::new(AtomicI64::new( + self.merge_interval_in_s + .load(std::sync::atomic::Ordering::Relaxed), + )), + editable_detached_mode: Arc::new(AtomicBool::new( + self.editable_detached_mode + .load(std::sync::atomic::Ordering::Relaxed), + )), + deleted_root_containers: Arc::new(Mutex::new( + self.deleted_root_containers.lock().clone(), + )), + hide_empty_root_containers: Arc::new(AtomicBool::new( + self.hide_empty_root_containers + .load(std::sync::atomic::Ordering::Relaxed), + )), + } + } + + pub fn text_style_config(&self) -> &Arc> { + &self.text_style_config + } + + pub fn record_timestamp(&self) -> bool { + self.record_timestamp + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_record_timestamp(&self, record: bool) { + self.record_timestamp + .store(record, std::sync::atomic::Ordering::Relaxed); + } + + pub fn detached_editing(&self) -> bool { + self.editable_detached_mode + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_detached_editing(&self, mode: bool) { + self.editable_detached_mode + .store(mode, std::sync::atomic::Ordering::Relaxed); + } + + pub fn merge_interval(&self) -> i64 { + self.merge_interval_in_s + .load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_merge_interval(&self, interval: i64) { + self.merge_interval_in_s + .store(interval, std::sync::atomic::Ordering::Relaxed); + } + + pub fn set_hide_empty_root_containers(&self, hide: bool) { + self.hide_empty_root_containers + .store(hide, std::sync::atomic::Ordering::Relaxed); + } +} + +#[derive(Debug)] +pub struct DefaultRandom; + +#[cfg(test)] +use std::sync::atomic::AtomicU64; +#[cfg(test)] +static mut TEST_RANDOM: AtomicU64 = AtomicU64::new(0); + +impl SecureRandomGenerator for DefaultRandom { + fn fill_byte(&self, dest: &mut [u8]) { + #[cfg(not(test))] + getrandom::getrandom(dest).unwrap(); + + #[cfg(test)] + // SAFETY: this is only used in test + unsafe { + #[allow(static_mut_refs)] + let bytes = TEST_RANDOM + .fetch_add(1, std::sync::atomic::Ordering::Release) + .to_le_bytes(); + dest.copy_from_slice(&bytes[..dest.len()]); + } + } +} + +pub trait SecureRandomGenerator: Send + Sync { + fn fill_byte(&self, dest: &mut [u8]); + fn next_u64(&self) -> u64 { + let mut buf = [0u8; 8]; + self.fill_byte(&mut buf); + u64::from_le_bytes(buf) + } + + fn next_u32(&self) -> u32 { + let mut buf = [0u8; 4]; + self.fill_byte(&mut buf); + u32::from_le_bytes(buf) + } + + fn next_i64(&self) -> i64 { + let mut buf = [0u8; 8]; + self.fill_byte(&mut buf); + i64::from_le_bytes(buf) + } + + fn next_i32(&self) -> i32 { + let mut buf = [0u8; 4]; + self.fill_byte(&mut buf); + i32::from_le_bytes(buf) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{atomic::Ordering, Mutex, OnceLock}; + + use loro_common::{ContainerID, ContainerType, InternalString}; + + use crate::container::richtext::ExpandType; + + use super::*; + + fn random_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + #[test] + fn configure_default_values_and_setters_match_the_public_contract() { + let config = Configure::default(); + + assert!(!config.record_timestamp()); + assert!(!config.detached_editing()); + assert_eq!(config.merge_interval(), 1000); + assert!(!config.hide_empty_root_containers.load(Ordering::Relaxed)); + assert!(config.deleted_root_containers.lock().is_empty()); + + let styles = config.text_style_config.read(); + assert_eq!( + styles.get(&InternalString::from("bold")), + Some(StyleConfig { + expand: ExpandType::After, + }) + ); + assert_eq!( + styles.get(&InternalString::from("italic")), + Some(StyleConfig { + expand: ExpandType::After, + }) + ); + assert_eq!( + styles.get(&InternalString::from("link")), + Some(StyleConfig { + expand: ExpandType::None, + }) + ); + assert_eq!(styles.get(&InternalString::from("missing")), None); + + config.set_record_timestamp(true); + config.set_detached_editing(true); + config.set_merge_interval(42); + config.set_hide_empty_root_containers(true); + + assert!(config.record_timestamp()); + assert!(config.detached_editing()); + assert_eq!(config.merge_interval(), 42); + assert!(config.hide_empty_root_containers.load(Ordering::Relaxed)); + } + + #[test] + fn configure_fork_copies_current_state_and_then_diverges() { + let config = Configure::default(); + config.set_record_timestamp(true); + config.set_detached_editing(true); + config.set_merge_interval(25); + config.set_hide_empty_root_containers(true); + config + .deleted_root_containers + .lock() + .insert(ContainerID::Root { + name: InternalString::from("root"), + container_type: ContainerType::Map, + }); + config.text_style_config.write().insert( + InternalString::from("custom"), + StyleConfig { + expand: ExpandType::None, + }, + ); + + let forked = config.fork(); + + assert!(forked.record_timestamp()); + assert!(forked.detached_editing()); + assert_eq!(forked.merge_interval(), 25); + assert!(forked.hide_empty_root_containers.load(Ordering::Relaxed)); + assert_eq!(forked.deleted_root_containers.lock().len(), 1); + assert_eq!( + forked + .text_style_config + .read() + .get(&InternalString::from("custom")), + Some(StyleConfig { + expand: ExpandType::None, + }) + ); + + config.set_record_timestamp(false); + config.set_detached_editing(false); + config.set_merge_interval(99); + config.set_hide_empty_root_containers(false); + config.deleted_root_containers.lock().clear(); + config + .text_style_config + .write() + .insert(InternalString::from("fork-only"), StyleConfig::default()); + + assert!(forked.record_timestamp()); + assert!(forked.detached_editing()); + assert_eq!(forked.merge_interval(), 25); + assert!(forked.hide_empty_root_containers.load(Ordering::Relaxed)); + assert_eq!(forked.deleted_root_containers.lock().len(), 1); + assert!(forked + .text_style_config + .read() + .get(&InternalString::from("fork-only")) + .is_none()); + } + + #[test] + fn default_random_test_mode_uses_the_incrementing_counter_for_integer_helpers() { + let _guard = random_lock().lock().unwrap(); + let random = DefaultRandom; + + let a = random.next_u64(); + let b = random.next_u32(); + let c = random.next_i64(); + let d = random.next_i32(); + + assert_eq!(b as u64, (a + 1) as u32 as u64); + assert_eq!(c as u64, a + 2); + assert_eq!(d as u64, (a + 3) as u32 as u64); + } +} diff --git a/vendor/loro-internal/src/container.rs b/vendor/loro-internal/src/container.rs new file mode 100644 index 00000000..3e8ea7df --- /dev/null +++ b/vendor/loro-internal/src/container.rs @@ -0,0 +1,314 @@ +//! CRDT [Container]. Each container may have different CRDT type [ContainerType]. +//! Each [Op] has an associated container. It's the [Container]'s responsibility to +//! calculate the state from the [Op]s. +//! +//! Every [Container] can take a [Snapshot], which contains [crate::LoroValue] that describes the state. +//! +use crate::{arena::SharedArena, InternalString, ID}; + +pub mod list; +pub mod map; +pub mod richtext; +pub mod tree; +pub mod idx { + use super::super::ContainerType; + + /// Inner representation for ContainerID. + /// It contains the unique index for the container and the type of the container. + /// It uses top 4 bits to represent the type of the container. + /// + /// It's only used inside this crate and should not be exposed to the user. + /// + /// TODO: make this type private in this crate only + /// + /// + // During a transaction, we may create some containers which are deleted later. And these containers also need a unique ContainerIdx. + // So when we encode snapshot, we need to sort the containers by ContainerIdx and change the `container` of ops to the index of containers. + // An empty store decodes the snapshot, it will create these containers in a sequence of natural numbers so that containers and ops can correspond one-to-one + // + // TODO: PERF: use NonZeroU32 to save memory + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] + pub struct ContainerIdx(u32); + + impl std::fmt::Debug for ContainerIdx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ContainerIdx({} {})", self.get_type(), self.to_index()) + } + } + + impl std::fmt::Display for ContainerIdx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ContainerIdx({} {})", self.get_type(), self.to_index()) + } + } + + impl ContainerIdx { + pub(crate) const TYPE_MASK: u32 = 0b11111 << 27; + pub(crate) const INDEX_MASK: u32 = !Self::TYPE_MASK; + + #[allow(unused)] + pub(crate) fn get_type(self) -> ContainerType { + let a = (self.0 & Self::TYPE_MASK) >> 27; + if self.is_unknown() { + Self::parse_unknown_type(a) + } else { + ContainerType::try_from_u8(a as u8).unwrap() + } + } + + #[allow(unused)] + pub(crate) fn to_index(self) -> u32 { + self.0 & Self::INDEX_MASK + } + + pub(crate) fn from_index_and_type(index: u32, container_type: ContainerType) -> Self { + let prefix: u32 = if matches!(container_type, ContainerType::Unknown(_)) { + Self::unknown_to_prefix(container_type) + } else { + container_type.to_u8() as u32 + } << 27; + + Self(prefix | index) + } + + pub(crate) fn is_unknown(&self) -> bool { + self.0 >> 31 == 1 + } + + // The type_value is >>27 first, so it's 5 bits. + // we want to get the last 4 bits. so we use 0b1111 to get the last 4 bits. + fn parse_unknown_type(type_value: u32) -> ContainerType { + ContainerType::Unknown((type_value & 0b1111) as u8) + } + + // we use the top 5 bits to represent the type of the container. + // the first bit is whether it's an unknown type. + // So when we convert an unknown type to a prefix, we need to set the first bit to 1. + fn unknown_to_prefix(c: ContainerType) -> u32 { + if let ContainerType::Unknown(c) = c { + (0b10000 | c) as u32 + } else { + unreachable!() + } + } + } +} +use idx::ContainerIdx; + +pub use loro_common::ContainerType; + +pub use loro_common::ContainerID; + +#[derive(Debug)] +pub enum ContainerIdRaw { + Root { name: InternalString }, + Normal { id: ID }, +} + +pub trait IntoContainerId { + fn into_container_id(self, arena: &SharedArena, kind: ContainerType) -> ContainerID; +} + +impl IntoContainerId for String { + fn into_container_id(self, _arena: &SharedArena, kind: ContainerType) -> ContainerID { + ContainerID::Root { + name: InternalString::from(self.as_str()), + container_type: kind, + } + } +} + +impl IntoContainerId for &str { + fn into_container_id(self, _arena: &SharedArena, kind: ContainerType) -> ContainerID { + ContainerID::Root { + name: InternalString::from(self), + container_type: kind, + } + } +} + +impl IntoContainerId for ContainerID { + fn into_container_id(self, _arena: &SharedArena, _kind: ContainerType) -> ContainerID { + self + } +} + +impl IntoContainerId for &ContainerID { + fn into_container_id(self, _arena: &SharedArena, _kind: ContainerType) -> ContainerID { + self.clone() + } +} + +impl IntoContainerId for ContainerIdx { + fn into_container_id(self, arena: &SharedArena, kind: ContainerType) -> ContainerID { + assert_eq!(self.get_type(), kind); + arena.get_container_id(self).unwrap() + } +} + +impl IntoContainerId for &ContainerIdx { + fn into_container_id(self, arena: &SharedArena, kind: ContainerType) -> ContainerID { + assert_eq!(self.get_type(), kind); + arena.get_container_id(*self).unwrap() + } +} + +impl From for ContainerIdRaw { + fn from(value: String) -> Self { + ContainerIdRaw::Root { name: value.into() } + } +} + +impl<'a> From<&'a str> for ContainerIdRaw { + fn from(value: &'a str) -> Self { + ContainerIdRaw::Root { name: value.into() } + } +} + +impl From<&ContainerID> for ContainerIdRaw { + fn from(id: &ContainerID) -> Self { + match id { + ContainerID::Root { name, .. } => ContainerIdRaw::Root { name: name.clone() }, + ContainerID::Normal { peer, counter, .. } => ContainerIdRaw::Normal { + id: ID::new(*peer, *counter), + }, + } + } +} + +impl From for ContainerIdRaw { + fn from(id: ContainerID) -> Self { + match id { + ContainerID::Root { name, .. } => ContainerIdRaw::Root { name }, + ContainerID::Normal { peer, counter, .. } => ContainerIdRaw::Normal { + id: ID::new(peer, counter), + }, + } + } +} + +impl ContainerIdRaw { + pub fn with_type(self, container_type: ContainerType) -> ContainerID { + match self { + ContainerIdRaw::Root { name } => ContainerID::Root { + name, + container_type, + }, + ContainerIdRaw::Normal { id } => ContainerID::Normal { + peer: id.peer, + counter: id.counter, + container_type, + }, + } + } +} + +#[cfg(test)] +mod test { + use super::idx::ContainerIdx; + use super::*; + + #[test] + fn container_id_convert() { + let container_id = ContainerID::new_normal(ID::new(12, 12), ContainerType::List); + let s = container_id.to_string(); + assert_eq!(s, "cid:12@12:List"); + let actual = ContainerID::try_from(s.as_str()).unwrap(); + assert_eq!(actual, container_id); + + let container_id = ContainerID::new_root("123", ContainerType::Map); + let s = container_id.to_string(); + assert_eq!(s, "cid:root-123:Map"); + let actual = ContainerID::try_from(s.as_str()).unwrap(); + assert_eq!(actual, container_id); + + let container_id = ContainerID::new_root("kkk", ContainerType::Text); + let s = container_id.to_string(); + assert_eq!(s, "cid:root-kkk:Text"); + let actual = ContainerID::try_from(s.as_str()).unwrap(); + assert_eq!(actual, container_id); + } + + #[test] + fn container_idx_preserves_type_bits_and_index_bits() { + let idx = ContainerIdx::from_index_and_type(42, ContainerType::Text); + assert_eq!(idx.get_type(), ContainerType::Text); + assert_eq!(idx.to_index(), 42); + assert!(!idx.is_unknown()); + assert_eq!(idx.to_string(), "ContainerIdx(Text 42)"); + assert_eq!(format!("{idx:?}"), "ContainerIdx(Text 42)"); + + let max_index = + ContainerIdx::from_index_and_type(ContainerIdx::INDEX_MASK, ContainerType::MovableList); + assert_eq!(max_index.get_type(), ContainerType::MovableList); + assert_eq!(max_index.to_index(), ContainerIdx::INDEX_MASK); + + let unknown = ContainerIdx::from_index_and_type(7, ContainerType::Unknown(13)); + assert!(unknown.is_unknown()); + assert_eq!(unknown.get_type(), ContainerType::Unknown(13)); + assert_eq!(unknown.to_index(), 7); + assert_eq!(unknown.to_string(), "ContainerIdx(Unknown(13) 7)"); + } + + #[test] + fn container_id_raw_reapplies_type_for_root_and_normal_ids() { + let root = ContainerIdRaw::from("notes").with_type(ContainerType::Map); + assert_eq!(root, ContainerID::new_root("notes", ContainerType::Map)); + + let normal_id = ID::new(7, 11); + let normal = ContainerIdRaw::from(ContainerID::new_normal(normal_id, ContainerType::List)) + .with_type(ContainerType::Text); + assert_eq!( + normal, + ContainerID::new_normal(normal_id, ContainerType::Text) + ); + + let root_again = ContainerIdRaw::from(&root).with_type(ContainerType::Tree); + assert_eq!( + root_again, + ContainerID::new_root("notes", ContainerType::Tree) + ); + } + + #[test] + fn into_container_id_uses_root_names_ids_and_arena_indexes() { + let arena = SharedArena::new(); + assert_eq!( + "workspace".into_container_id(&arena, ContainerType::Map), + ContainerID::new_root("workspace", ContainerType::Map) + ); + assert_eq!( + String::from("text").into_container_id(&arena, ContainerType::Text), + ContainerID::new_root("text", ContainerType::Text) + ); + + let normal = ContainerID::new_normal(ID::new(1, 2), ContainerType::List); + assert_eq!( + (&normal).into_container_id(&arena, ContainerType::Map), + normal + ); + assert_eq!( + normal + .clone() + .into_container_id(&arena, ContainerType::Text), + normal + ); + + let idx = arena.register_container(&normal); + assert_eq!(idx.into_container_id(&arena, ContainerType::List), normal); + assert_eq!( + (&idx).into_container_id(&arena, ContainerType::List), + normal + ); + } + + #[test] + #[should_panic(expected = "assertion `left == right` failed")] + fn into_container_id_rejects_mismatched_index_type() { + let arena = SharedArena::new(); + let id = ContainerID::new_root("items", ContainerType::List); + let idx = arena.register_container(&id); + + let _ = idx.into_container_id(&arena, ContainerType::Map); + } +} diff --git a/vendor/loro-internal/src/container/list/list_op.rs b/vendor/loro-internal/src/container/list/list_op.rs new file mode 100644 index 00000000..4bee2bf1 --- /dev/null +++ b/vendor/loro-internal/src/container/list/list_op.rs @@ -0,0 +1,803 @@ +use std::ops::Range; + +use append_only_bytes::BytesSlice; +use enum_as_inner::EnumAsInner; +use loro_common::{ContainerType, HasId, HasIdSpan, IdLp, LoroValue, ID}; +use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; + +use crate::{ + container::richtext::richtext_state::unicode_to_utf8_index, + container::richtext::TextStyleInfoFlag, + op::{ListSlice, SliceRange}, + InternalString, +}; + +/// `len` and `pos` is measured in unicode char for text. +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(EnumAsInner, Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum ListOp<'a> { + Insert { + slice: ListSlice<'a>, + pos: usize, + }, + Delete(DeleteSpanWithId), + Move { + from: u32, + to: u32, + elem_id: IdLp, + }, + Set { + elem_id: IdLp, + value: LoroValue, + }, + /// StyleStart and StyleEnd must be paired because the end of a style must take an OpID position. + StyleStart { + start: u32, + end: u32, + key: InternalString, + info: TextStyleInfoFlag, + value: LoroValue, + }, + StyleEnd, +} + +#[derive(EnumAsInner, Debug, Clone)] +pub enum InnerListOp { + // TODO: this is only used for list now? We should rename it to InsertList + Insert { + slice: SliceRange, + pos: usize, + }, + // Note: len may not equal to slice.len() because for text len is unicode len while the slice + // is utf8 bytes. + InsertText { + slice: BytesSlice, + unicode_start: u32, + unicode_len: u32, + pos: u32, + }, + Delete(DeleteSpanWithId), + Move { + from: u32, + /// Element id + elem_id: IdLp, + to: u32, + }, + Set { + elem_id: IdLp, + value: LoroValue, + }, + /// StyleStart and StyleEnd must be paired. + /// The next op of StyleStart must be StyleEnd. + StyleStart { + start: u32, + end: u32, + key: InternalString, + value: LoroValue, + info: TextStyleInfoFlag, + }, + StyleEnd, +} + +impl ListOp<'_> { + pub fn new_del(id_start: ID, pos: usize, len: usize) -> Self { + assert!(len != 0); + Self::Delete(DeleteSpanWithId::new(id_start, pos as isize, len as isize)) + } +} + +impl InnerListOp { + pub fn new_del(id: ID, pos: usize, len: isize) -> Self { + assert!(len != 0); + Self::Delete(DeleteSpanWithId { + id_start: id, + span: DeleteSpan { + pos: pos as isize, + signed_len: len, + }, + }) + } + + pub fn new_insert(slice: Range, pos: usize) -> Self { + Self::Insert { + slice: SliceRange(slice), + pos, + } + } + + pub(crate) fn estimate_storage_size(&self, container_type: ContainerType) -> usize { + match self { + InnerListOp::Insert { slice, .. } => match container_type { + ContainerType::MovableList | ContainerType::List => 4 * slice.atom_len(), + ContainerType::Text => slice.atom_len(), + _ => unreachable!(), + }, + InnerListOp::InsertText { slice, .. } => slice.len(), + InnerListOp::Delete(..) => 8, + InnerListOp::Move { .. } => 8, + InnerListOp::Set { .. } => 7, + InnerListOp::StyleStart { .. } => 10, + InnerListOp::StyleEnd => 1, + } + } +} + +impl HasLength for DeleteSpan { + fn content_len(&self) -> usize { + self.signed_len.unsigned_abs() + } +} + +/// Delete span that the initial id is `id_start`. +/// +/// This span may be reversed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeleteSpanWithId { + /// This is the target id with the smallest counter no matter whether the span is reversed. + /// So it's always the id of the leftmost element of the target span. + pub id_start: ID, + /// The deleted position span + pub span: DeleteSpan, +} + +impl DeleteSpanWithId { + pub fn new(id_start: ID, pos: isize, len: isize) -> Self { + debug_assert!(len != 0); + Self { + id_start, + span: DeleteSpan { + pos, + signed_len: len, + }, + } + } + + #[inline] + pub fn start(&self) -> isize { + self.span.start() + } + + #[inline] + pub fn end(&self) -> isize { + self.span.end() + } + + #[inline] + pub fn last(&self) -> isize { + self.span.last() + } + + #[inline] + pub fn is_reversed(&self) -> bool { + self.span.is_reversed() + } +} + +impl HasLength for DeleteSpanWithId { + fn content_len(&self) -> usize { + self.span.content_len() + } +} + +impl HasId for DeleteSpanWithId { + fn id_start(&self) -> ID { + self.id_start + } +} + +impl Mergable for DeleteSpanWithId { + /// If two spans are mergeable, their ids should be continuous. + /// LHS's end id should be equal to RHS's start id. + /// But their spans may be in a reversed order. + fn is_mergable(&self, rhs: &Self, _conf: &()) -> bool + where + Self: Sized, + { + let this = self.span; + let other = rhs.span; + // merge continuous deletions: + // note that the previous deletions will affect the position of the later deletions + match (self.span.bidirectional(), rhs.span.bidirectional()) { + (true, true) => { + (this.pos == other.pos && self.id_start.inc(1) == rhs.id_start) + || (this.pos == other.pos + 1 && self.id_start == rhs.id_start.inc(1)) + } + (true, false) => { + if this.pos == other.prev_pos() { + if other.signed_len > 0 { + self.id_start.inc(1) == rhs.id_start + } else { + self.id_start == rhs.id_end() + } + } else { + false + } + } + (false, true) => { + if this.next_pos() == other.pos { + if this.signed_len > 0 { + self.id_end() == rhs.id_start + } else { + self.id_start == rhs.id_start.inc(1) + } + } else { + false + } + } + (false, false) => { + if this.next_pos() == other.pos && this.direction() == other.direction() { + if self.span.signed_len > 0 { + self.id_end() == rhs.id_start + } else { + self.id_start == rhs.id_end() + } + } else { + false + } + } + } + } + + fn merge(&mut self, rhs: &Self, _conf: &()) + where + Self: Sized, + { + self.id_start.counter = rhs.id_start.counter.min(self.id_start.counter); + self.span.merge(&rhs.span, &()) + } +} + +impl Sliceable for DeleteSpanWithId { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id_start: if self.span.signed_len > 0 { + self.id_start.inc(from as i32) + } else { + // If the span is reversed, the id_start should be affected by `to` + // + // Example: + // + // a b c + // - - - <-- deletions happen backward + // 0 1 2 <-- counter of the IDs + // ↑ + // id_start + // + // If from=1, to=2 + // a b c + // - - - <-- deletions happen backward + // 0 1 2 <-- counter of the IDs + // ↑ + // id_start + self.id_start.inc((self.atom_len() - to) as i32) + }, + span: self.span.slice(from, to), + } + } +} + +/// `len` can be negative so that we can merge text deletions efficiently. +/// It looks like [crate::span::CounterSpan], but how should they merge ([Mergable] impl) and slice ([Sliceable] impl) are very different +/// +/// len cannot be zero; +/// +/// pos: 5, len: -3 eq a range of (2, 5] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +pub struct DeleteSpan { + pub pos: isize, + pub signed_len: isize, +} + +impl DeleteSpan { + pub fn new(pos: isize, len: isize) -> Self { + debug_assert!(len != 0); + Self { + pos, + signed_len: len, + } + } + + #[inline(always)] + pub fn start(&self) -> isize { + if self.signed_len > 0 { + self.pos + } else { + self.pos + 1 + self.signed_len + } + } + + #[inline(always)] + pub fn last(&self) -> isize { + if self.signed_len > 0 { + self.pos + self.signed_len - 1 + } else { + self.pos + } + } + + #[inline(always)] + pub fn end(&self) -> isize { + if self.signed_len > 0 { + self.pos + self.signed_len + } else { + self.pos + 1 + } + } + + #[inline(always)] + pub fn to_range(self) -> Range { + self.start()..self.end() + } + + #[inline(always)] + pub fn to_urange(self) -> Range { + self.start() as usize..self.end() as usize + } + + #[inline(always)] + pub fn bidirectional(&self) -> bool { + self.signed_len.abs() == 1 + } + + #[inline(always)] + pub fn is_reversed(&self) -> bool { + self.signed_len < 0 + } + + #[inline(always)] + pub fn direction(&self) -> isize { + if self.signed_len > 0 { + 1 + } else { + -1 + } + } + + #[inline(always)] + pub fn next_pos(&self) -> isize { + if self.signed_len > 0 { + self.start() + } else { + self.start() - 1 + } + } + + #[inline(always)] + pub fn prev_pos(&self) -> isize { + if self.signed_len > 0 { + self.pos + } else { + self.end() + } + } + + pub fn len(&self) -> usize { + self.signed_len.unsigned_abs() + } +} + +impl Mergable for DeleteSpan { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + // merge continuous deletions: + // note that the previous deletions will affect the position of the later deletions + match (self.bidirectional(), other.bidirectional()) { + (true, true) => self.pos == other.pos || self.pos == other.pos + 1, + (true, false) => self.pos == other.prev_pos(), + (false, true) => self.next_pos() == other.pos, + (false, false) => self.next_pos() == other.pos && self.direction() == other.direction(), + } + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + match (self.bidirectional(), other.bidirectional()) { + (true, true) => { + if self.pos == other.pos { + self.signed_len = 2; + } else if self.pos == other.pos + 1 { + self.signed_len = -2; + } else { + unreachable!() + } + } + (true, false) => { + assert!(self.pos == other.prev_pos()); + self.signed_len = other.signed_len + other.direction(); + } + (false, true) => { + assert!(self.next_pos() == other.pos); + self.signed_len += self.direction(); + } + (false, false) => { + assert!(self.next_pos() == other.pos && self.direction() == other.direction()); + self.signed_len += other.signed_len; + } + } + } +} + +impl Sliceable for DeleteSpan { + fn slice(&self, from: usize, to: usize) -> Self { + if self.signed_len > 0 { + Self::new(self.pos, to as isize - from as isize) + } else { + Self::new(self.pos - from as isize, from as isize - to as isize) + } + } +} + +impl Mergable for ListOp<'_> { + fn is_mergable(&self, _other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match self { + ListOp::Insert { pos, slice } => match _other { + ListOp::Insert { + pos: other_pos, + slice: other_slice, + } => pos + slice.content_len() == *other_pos && slice.is_mergable(other_slice, &()), + _ => false, + }, + &ListOp::Delete(span) => match _other { + ListOp::Delete(other_span) => span.is_mergable(other_span, &()), + _ => false, + }, + ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. } => false, + } + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + match self { + ListOp::Insert { slice, .. } => match _other { + ListOp::Insert { + slice: other_slice, .. + } => { + slice.merge(other_slice, &()); + } + _ => unreachable!(), + }, + ListOp::Delete(span) => match _other { + ListOp::Delete(other_span) => span.merge(other_span, &()), + _ => unreachable!(), + }, + ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. } => { + unreachable!() + } + } + } +} + +impl HasLength for ListOp<'_> { + fn content_len(&self) -> usize { + match self { + ListOp::Insert { slice, .. } => slice.content_len(), + ListOp::Delete(span) => span.atom_len(), + ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. } => 1, + } + } +} + +impl Sliceable for ListOp<'_> { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + ListOp::Insert { slice, pos } => ListOp::Insert { + slice: slice.slice(from, to), + pos: *pos + from, + }, + ListOp::Delete(span) => ListOp::Delete(span.slice(from, to)), + a @ (ListOp::StyleStart { .. } + | ListOp::StyleEnd { .. } + | ListOp::Move { .. } + | ListOp::Set { .. }) => a.clone(), + } + } +} + +impl Mergable for InnerListOp { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match (self, other) { + ( + InnerListOp::Insert { pos, slice, .. }, + InnerListOp::Insert { + pos: other_pos, + slice: other_slice, + .. + }, + ) => pos + slice.content_len() == *other_pos && slice.is_mergable(other_slice, &()), + (InnerListOp::Delete(span), InnerListOp::Delete(other_span)) => { + span.is_mergable(other_span, &()) + } + ( + InnerListOp::InsertText { + unicode_start, + slice, + pos, + unicode_len: len, + }, + InnerListOp::InsertText { + slice: other_slice, + pos: other_pos, + unicode_start: other_unicode_start, + unicode_len: _, + }, + ) => { + pos + len == *other_pos + && slice.can_merge(other_slice) + && unicode_start + len == *other_unicode_start + } + _ => false, + } + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + match (self, other) { + ( + InnerListOp::Insert { slice, .. }, + InnerListOp::Insert { + slice: other_slice, .. + }, + ) => { + slice.merge(other_slice, &()); + } + (InnerListOp::Delete(span), InnerListOp::Delete(other_span)) => { + span.merge(other_span, &()) + } + ( + InnerListOp::InsertText { + slice, + unicode_len: len, + .. + }, + InnerListOp::InsertText { + slice: other_slice, + unicode_len: other_len, + .. + }, + ) => { + slice.merge(other_slice, &()); + *len += *other_len; + } + _ => unreachable!(), + } + } +} + +impl HasLength for InnerListOp { + fn content_len(&self) -> usize { + match self { + InnerListOp::Insert { slice, .. } => slice.content_len(), + InnerListOp::InsertText { + unicode_len: len, .. + } => *len as usize, + InnerListOp::Delete(span) => span.atom_len(), + InnerListOp::StyleStart { .. } + | InnerListOp::StyleEnd + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => 1, + } + } +} + +impl Sliceable for InnerListOp { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + InnerListOp::Insert { slice, pos } => InnerListOp::Insert { + slice: slice.slice(from, to), + pos: *pos + from, + }, + InnerListOp::InsertText { + slice, + unicode_start, + unicode_len, + pos, + } => InnerListOp::InsertText { + slice: { + // SAFETY: we know it's a valid utf8 string + let s = unsafe { std::str::from_utf8_unchecked(slice) }; + let total_unicode_len = *unicode_len as usize; + let total_bytes_len = slice.len(); + let (from_byte, to_byte) = if total_bytes_len == total_unicode_len { + // ASCII fast path: unicode index equals byte offset. + (from, to) + } else { + let from_byte = if from == 0 { + 0 + } else { + unicode_to_utf8_index(s, from).expect("unicode index should be valid") + }; + let to_byte = if to == total_unicode_len { + total_bytes_len + } else { + unicode_to_utf8_index(s, to).expect("unicode index should be valid") + }; + (from_byte, to_byte) + }; + + slice.slice(from_byte, to_byte) + }, + unicode_start: *unicode_start + from as u32, + unicode_len: (to - from) as u32, + pos: *pos + from as u32, + }, + InnerListOp::Delete(span) => InnerListOp::Delete(span.slice(from, to)), + InnerListOp::StyleStart { .. } + | InnerListOp::StyleEnd + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => { + assert!(from == 0 && to == 1); + self.clone() + } + } + } +} + +#[cfg(test)] +mod test { + use append_only_bytes::BytesSlice; + use loro_common::ID; + use rle::{HasLength as _, Mergable, Sliceable}; + use std::time::Instant; + + use crate::{container::list::list_op::DeleteSpanWithId, op::ListSlice}; + + use super::{DeleteSpan, InnerListOp, ListOp}; + + #[test] + fn fix_fields_order() { + let list_op = vec![ + ListOp::Insert { + pos: 0, + slice: ListSlice::from_borrowed_str(""), + }, + ListOp::Delete(DeleteSpanWithId::new(ID::new(0, 0), 0, 3)), + ]; + let actual = postcard::to_allocvec(&list_op).unwrap(); + let list_op_buf = vec![2, 0, 1, 0, 0, 0, 1, 0, 0, 0, 6]; + assert_eq!(&actual, &list_op_buf); + assert_eq!( + postcard::from_bytes::>(&list_op_buf).unwrap(), + list_op + ); + + let delete_span = DeleteSpan { + pos: 0, + signed_len: 3, + }; + let delete_span_buf = vec![0, 6]; + assert_eq!( + postcard::from_bytes::(&delete_span_buf).unwrap(), + delete_span + ); + } + + #[test] + fn test_del_span_merge_slice() { + let a = DeleteSpan::new(0, 100); + let mut b = a.slice(0, 1); + let c = a.slice(1, 100); + b.merge(&c, &()); + assert_eq!(b, a); + + // reverse + let a = DeleteSpan::new(99, -100); + let mut b = a.slice(0, 1); + let c = a.slice(1, 100); + b.merge(&c, &()); + assert_eq!(b, a); + + // merge bidirectional + let mut a = DeleteSpan::new(1, -1); + let b = DeleteSpan::new(1, -1); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, DeleteSpan::new(1, 2)); + + let mut a = DeleteSpan::new(1, -1); + let b = DeleteSpan::new(0, -1); + assert_eq!(b.to_range(), 0..1); + assert!(a.is_mergable(&b, &())); + a.merge(&b, &()); + assert_eq!(a, DeleteSpan::new(1, -2)); + + // not merging + let a = DeleteSpan::new(4, 1); + let b = DeleteSpan::new(5, 2); + assert!(!a.is_mergable(&b, &())); + + // next/prev span + let a = DeleteSpan::new(6, -2); + assert_eq!(a.next_pos(), 4); + assert_eq!(a.prev_pos(), 7); + let a = DeleteSpan::new(6, 2); + assert_eq!(a.next_pos(), 6); + assert_eq!(a.prev_pos(), 6); + assert!(a.slice(0, 1).is_mergable(&a.slice(1, 2), &())); + + // neg merge + let mut a = DeleteSpan::new(1, 1); + let b = DeleteSpan::new(0, 1); + a.merge(&b, &()); + assert_eq!(a, DeleteSpan::new(1, -2)); + assert_eq!(a.slice(0, 1), DeleteSpan::new(1, -1)); + assert_eq!(a.slice(1, 2), DeleteSpan::new(0, -1)); + assert_eq!(a.slice(0, 1).to_range(), 1..2); + assert_eq!(a.slice(1, 2).to_range(), 0..1); + } + + #[test] + fn mergeable() { + let a = DeleteSpan::new(14852, 1); + let mut a_with_id = DeleteSpanWithId { + id_start: ID::new(0, 9), + span: a, + }; + let b = DeleteSpan::new(14851, 1); + let b_with_id = DeleteSpanWithId { + id_start: ID::new(0, 8), + span: b, + }; + assert!(a_with_id.is_mergable(&b_with_id, &())); + a_with_id.merge(&b_with_id, &()); + assert!(a_with_id.span.signed_len == -2); + } + + #[test] + #[ignore] + fn perf_insert_text_slice_split_suffix() { + // Run with: + // cargo test -p loro-internal perf_insert_text_slice_split_suffix -- --ignored --nocapture + const TEXT_LEN: usize = 2 * 1024 * 1024; + const CHUNK_UNICODE_LEN: usize = 4096; + + let text = "a".repeat(TEXT_LEN); + let slice = BytesSlice::from_bytes(text.as_bytes()); + let mut op = InnerListOp::InsertText { + slice, + unicode_start: 0, + unicode_len: TEXT_LEN as u32, + pos: 0, + }; + + let start = Instant::now(); + let mut total = 0usize; + while op.content_len() > CHUNK_UNICODE_LEN { + let end = CHUNK_UNICODE_LEN.min(op.content_len()); + let _prefix = op.slice(0, end); + op = op.slice(end, op.content_len()); + total += end; + } + total += op.content_len(); + let elapsed = start.elapsed(); + assert_eq!(total, TEXT_LEN); + println!( + "perf_insert_text_slice_split_suffix: text_len={}, chunk_unicode_len={}, elapsed={:?}", + TEXT_LEN, CHUNK_UNICODE_LEN, elapsed + ); + } +} diff --git a/vendor/loro-internal/src/container/list/mod.rs b/vendor/loro-internal/src/container/list/mod.rs new file mode 100644 index 00000000..1da3d626 --- /dev/null +++ b/vendor/loro-internal/src/container/list/mod.rs @@ -0,0 +1 @@ +pub(crate) mod list_op; diff --git a/vendor/loro-internal/src/container/map/map_content.rs b/vendor/loro-internal/src/container/map/map_content.rs new file mode 100644 index 00000000..993718ad --- /dev/null +++ b/vendor/loro-internal/src/container/map/map_content.rs @@ -0,0 +1,42 @@ +use rle::{HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; + +use crate::{InternalString, LoroValue}; + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct MapSet { + pub(crate) key: InternalString, + // the key is deleted if value is None + pub(crate) value: Option, +} + +impl Mergable for MapSet {} +impl Sliceable for MapSet { + fn slice(&self, from: usize, to: usize) -> Self { + assert!(from == 0 && to == 1); + self.clone() + } +} +impl HasLength for MapSet { + fn content_len(&self) -> usize { + 1 + } +} + +#[cfg(test)] +mod test { + use super::MapSet; + #[test] + fn fix_fields_order() { + let map_set = vec![MapSet { + key: "key".to_string().into(), + value: Some("value".to_string().into()), + }]; + let map_set_buf = vec![1, 3, 107, 101, 121, 1, 4, 5, 118, 97, 108, 117, 101]; + assert_eq!( + postcard::from_bytes::>(&map_set_buf).unwrap(), + map_set + ); + } +} diff --git a/vendor/loro-internal/src/container/map/mod.rs b/vendor/loro-internal/src/container/map/mod.rs new file mode 100644 index 00000000..1a667865 --- /dev/null +++ b/vendor/loro-internal/src/container/map/mod.rs @@ -0,0 +1,3 @@ +mod map_content; + +pub(crate) use map_content::MapSet; diff --git a/vendor/loro-internal/src/container/richtext.rs b/vendor/loro-internal/src/container/richtext.rs new file mode 100644 index 00000000..ffe4aa33 --- /dev/null +++ b/vendor/loro-internal/src/container/richtext.rs @@ -0,0 +1,300 @@ +//! # Index +//! +//! There are several types of indexes: +//! +//! - Unicode index: the index of a unicode code point in the text. +//! - Entity index: unicode index + style anchor index. Each unicode code point or style anchor is an entity. +//! - Utf16 index +//! +//! In [crate::op::Op], we always use entity index to persist richtext ops. +//! +//! The users of this type can only operate on unicode index or utf16 index, but calculated entity index will be provided. + +pub(crate) mod config; +mod fugue_span; +pub(crate) mod richtext_state; +pub(crate) mod str_slice; +mod style_range_map; +#[cfg(all(test, miri))] +mod tinyvec; +mod tracker; + +use crate::{change::Lamport, delta::StyleMeta, utils::string_slice::StringSlice, InternalString}; +use fugue_span::*; +use loro_common::{Counter, IdFull, IdLp, LoroValue, PeerID, ID}; +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +pub(crate) use fugue_span::{RichtextChunk, RichtextChunkValue}; +pub(crate) use richtext_state::RichtextState; +pub(crate) use style_range_map::Styles; +pub(crate) use tracker::{CrdtRopeDelta, Tracker as RichtextTracker}; + +/// This is the data structure that represents a span of rich text. +/// It's used to communicate with the frontend. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct RichtextSpan { + pub text: StringSlice, + pub attributes: StyleMeta, +} + +/// This is used to communicate with the frontend. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct Style { + pub key: InternalString, + pub data: LoroValue, +} + +// TODO: change visibility back to crate after #116 is done +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct StyleOp { + pub(crate) lamport: Lamport, + pub(crate) peer: PeerID, + pub(crate) cnt: Counter, + pub(crate) key: InternalString, + pub(crate) value: LoroValue, + pub(crate) info: TextStyleInfoFlag, +} + +#[derive(Debug, Hash, Eq, PartialEq, Clone, Serialize, Deserialize)] +pub(crate) enum StyleKey { + Key(InternalString), +} + +impl StyleKey { + pub fn key(&self) -> &InternalString { + match self { + Self::Key(key) => key, + } + } +} + +impl StyleOp { + pub fn to_style(&self) -> Style { + Style { + key: self.key.clone(), + data: self.value.clone(), + } + } + + pub fn to_value(&self) -> LoroValue { + self.value.clone() + } + + pub(crate) fn get_style_key(&self) -> StyleKey { + StyleKey::Key(self.key.clone()) + } + + #[cfg(test)] + pub fn new_for_test(n: isize, key: &str, value: LoroValue, info: TextStyleInfoFlag) -> Self { + Self { + lamport: n as Lamport, + peer: n as PeerID, + cnt: n as Counter, + key: key.to_string().into(), + value, + info, + } + } + + #[inline(always)] + pub fn id(&self) -> ID { + ID::new(self.peer, self.cnt) + } + + pub fn idlp(&self) -> IdLp { + IdLp::new(self.peer, self.lamport) + } + + pub fn id_full(&self) -> IdFull { + IdFull::new(self.peer, self.cnt, self.lamport) + } +} + +impl PartialOrd for StyleOp { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for StyleOp { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport + .cmp(&other.lamport) + .then(self.peer.cmp(&other.peer)) + } +} + +/// TODO: We can remove this type already +/// +/// A compact representation of a rich text style config. +/// +/// Note: we assume style with the same key has the same `Mergeable` and `isContainer` value. +/// +/// - 0 (1st bit) +/// - Expand Before (2nd bit): when inserting new text before this style, whether the new text should inherit this style. +/// - Expand After (3rd bit): when inserting new text after this style, whether the new text should inherit this style. +/// - 0 (4th bit): +/// - 0 (5th bit): +/// - 0 (6th bit) +/// - 0 (7th bit) +/// - 0 (8th bit): +#[derive(Default, Clone, Copy, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)] +pub struct TextStyleInfoFlag { + data: u8, +} + +impl Debug for TextStyleInfoFlag { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextStyleInfo") + // write data in binary format + .field("data", &format!("{:#010b}", self.data)) + .field("expand_before", &self.expand_before()) + .field("expand_after", &self.expand_after()) + .finish() + } +} + +const EXPAND_BEFORE_MASK: u8 = 0b0000_0010; +const EXPAND_AFTER_MASK: u8 = 0b0000_0100; +const ALIVE_MASK: u8 = 0b1000_0000; + +/// Whether to expand the style when inserting new text around it. +/// +/// - Before: when inserting new text before this style, the new text should inherit this style. +/// - After: when inserting new text after this style, the new text should inherit this style. +/// - Both: when inserting new text before or after this style, the new text should inherit this style. +/// - None: when inserting new text before or after this style, the new text should **not** inherit this style. +#[derive(Clone, Copy, Eq, PartialEq, Debug, Hash)] +pub enum ExpandType { + Before, + After, + Both, + None, +} + +#[derive( + Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash, serde::Serialize, serde::Deserialize, +)] +pub enum AnchorType { + Start, + End, +} + +impl ExpandType { + #[inline(always)] + pub const fn expand_before(&self) -> bool { + matches!(self, ExpandType::Before | ExpandType::Both) + } + + #[inline(always)] + pub const fn expand_after(&self) -> bool { + matches!(self, ExpandType::After | ExpandType::Both) + } + + /// 'before'|'after'|'both'|'none' + pub fn try_from_str(s: &str) -> Option { + match s { + "before" => Some(ExpandType::Before), + "after" => Some(ExpandType::After), + "both" => Some(ExpandType::Both), + "none" => Some(ExpandType::None), + _ => None, + } + } + + /// Toggle expand type between for deletion and for creation + /// + /// For a style that expand after, when we delete the style, we need to have another style that expands after to nullify it, + /// so that the expand behavior is not changed. + /// + /// Before -> Before + /// After -> After + /// Both -> None + /// None -> Both + /// + /// Because the creation of text styles and the deletion of the text styles have reversed expand type. + /// This method is useful to convert between the two + pub const fn reverse(self) -> Self { + match self { + ExpandType::Before => ExpandType::Before, + ExpandType::After => ExpandType::After, + ExpandType::Both => ExpandType::None, + ExpandType::None => ExpandType::Both, + } + } +} + +impl TextStyleInfoFlag { + /// When inserting new text around this style, prefer inserting after it. + #[inline(always)] + pub const fn expand_before(self) -> bool { + self.data & EXPAND_BEFORE_MASK != 0 + } + + /// When inserting new text around this style, prefer inserting before it. + #[inline(always)] + pub const fn expand_after(self) -> bool { + self.data & EXPAND_AFTER_MASK != 0 + } + + pub const fn expand_type(self) -> ExpandType { + match (self.expand_before(), self.expand_after()) { + (true, true) => ExpandType::Both, + (true, false) => ExpandType::Before, + (false, true) => ExpandType::After, + (false, false) => ExpandType::None, + } + } + + /// This method tells that when we can insert text before/after this style anchor, whether we insert the new text before the anchor. + #[inline] + pub fn prefer_insert_before(self, anchor_type: AnchorType) -> bool { + match anchor_type { + AnchorType::Start => { + // If we need to expand the style, the new text should be inserted **after** the start anchor + !self.expand_before() + } + AnchorType::End => { + // If we need to expand the style, the new text should be inserted **before** the end anchor + self.expand_after() + } + } + } + + pub const fn new(expand_type: ExpandType) -> Self { + let mut data = ALIVE_MASK; + if expand_type.expand_before() { + data |= EXPAND_BEFORE_MASK; + } + if expand_type.expand_after() { + data |= EXPAND_AFTER_MASK; + } + + TextStyleInfoFlag { data } + } + + #[inline(always)] + pub const fn to_delete(self) -> Self { + TextStyleInfoFlag::new(self.expand_type().reverse()) + } + + pub const BOLD: TextStyleInfoFlag = TextStyleInfoFlag::new(ExpandType::After); + pub const LINK: TextStyleInfoFlag = TextStyleInfoFlag::new(ExpandType::None); + pub const COMMENT: TextStyleInfoFlag = TextStyleInfoFlag::new(ExpandType::None); + + pub const fn to_byte(&self) -> u8 { + self.data + } + + pub const fn from_byte(data: u8) -> Self { + Self { data } + } +} + +#[cfg(test)] +mod test { + + #[test] + fn test() {} +} diff --git a/vendor/loro-internal/src/container/richtext/config.rs b/vendor/loro-internal/src/container/richtext/config.rs new file mode 100644 index 00000000..8c89264f --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/config.rs @@ -0,0 +1,133 @@ +use loro_common::InternalString; +use rustc_hash::FxHashMap; + +use super::{ExpandType, TextStyleInfoFlag}; + +#[derive(Debug, Default, Clone)] +pub struct StyleConfigMap { + pub(crate) map: FxHashMap, + pub(crate) default_style: Option, +} + +impl StyleConfigMap { + pub fn new() -> Self { + Self { + map: FxHashMap::default(), + default_style: None, + } + } + + pub fn insert(&mut self, key: InternalString, value: StyleConfig) { + if key.contains(':') { + panic!("style key should not contain ':'"); + } + + self.map.insert(key, value); + } + + pub fn get(&self, key: &InternalString) -> Option { + self.map.get(key).copied().or(self.default_style) + } + + pub fn get_style_flag(&self, key: &InternalString) -> Option { + self._get_style_flag(key, false) + } + + pub fn get_style_flag_for_unmark(&self, key: &InternalString) -> Option { + self._get_style_flag(key, true) + } + + fn _get_style_flag(&self, key: &InternalString, is_del: bool) -> Option { + let f = |x: StyleConfig| { + TextStyleInfoFlag::new(if is_del { x.expand.reverse() } else { x.expand }) + }; + if let Some(index) = key.find(':') { + let key = key[..index].into(); + self.map.get(&key).copied().or(self.default_style).map(f) + } else { + self.map.get(key).copied().or(self.default_style).map(f) + } + } + + pub fn default_rich_text_config() -> Self { + let mut map = Self { + map: FxHashMap::default(), + default_style: None, + }; + + map.map.insert( + "bold".into(), + StyleConfig { + expand: ExpandType::After, + }, + ); + + map.map.insert( + "italic".into(), + StyleConfig { + expand: ExpandType::After, + }, + ); + + map.map.insert( + "underline".into(), + StyleConfig { + expand: ExpandType::After, + }, + ); + + map.map.insert( + "link".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map.map.insert( + "highlight".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map.map.insert( + "comment".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map.map.insert( + "code".into(), + StyleConfig { + expand: ExpandType::None, + }, + ); + + map + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StyleConfig { + pub expand: ExpandType, +} + +impl StyleConfig { + pub fn new() -> Self { + Self { + expand: ExpandType::None, + } + } + + pub fn expand(mut self, expand: ExpandType) -> Self { + self.expand = expand; + self + } +} + +impl Default for StyleConfig { + fn default() -> Self { + Self::new() + } +} diff --git a/vendor/loro-internal/src/container/richtext/fugue_span.rs b/vendor/loro-internal/src/container/richtext/fugue_span.rs new file mode 100644 index 00000000..69680788 --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/fugue_span.rs @@ -0,0 +1,386 @@ +use std::ops::Range; + +use generic_btree::rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}; +use loro_common::{CompactId, Counter, HasId, IdFull, IdSpan, Lamport, ID}; +use serde::{Deserialize, Serialize}; + +use super::{tracker::UNKNOWN_PEER_ID, AnchorType}; + +#[derive(Clone, PartialEq, Eq, Copy, Serialize, Deserialize)] +pub(crate) struct RichtextChunk { + start: u32, + end: u32, +} + +impl std::fmt::Debug for RichtextChunk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RichtextChunk") + .field("value", &self.value()) + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(crate) enum RichtextChunkKind { + Text, + StyleAnchor, + Unknown, + /// Move anchor is a special anchor that is used to mark the FugueSpan created + /// by a move operation. + MoveAnchor, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum RichtextChunkValue { + Text(Range), + StyleAnchor { + id: u32, + anchor_type: AnchorType, + }, + Unknown(u32), + /// Move anchor is a special anchor that is used to mark the FugueSpan created + /// by a move operation. + MoveAnchor, +} + +impl RichtextChunk { + pub(crate) const UNKNOWN: u32 = u32::MAX; + pub(crate) const START_STYLE_ANCHOR: u32 = u32::MAX - 1; + pub(crate) const END_STYLE_ANCHOR: u32 = u32::MAX - 2; + pub(crate) const MOVE_ANCHOR: u32 = u32::MAX - 3; + + #[inline] + pub fn new_text(range: Range) -> Self { + Self { + start: range.start, + end: range.end, + } + } + + #[inline] + pub fn new_style_anchor(idx: u32, anchor_type: AnchorType) -> Self { + match anchor_type { + AnchorType::Start => Self { + start: Self::START_STYLE_ANCHOR, + end: idx, + }, + AnchorType::End => Self { + start: Self::END_STYLE_ANCHOR, + end: idx, + }, + } + } + + #[inline] + pub fn new_unknown(len: u32) -> Self { + Self { + start: Self::UNKNOWN, + end: len, + } + } + + #[inline] + pub fn new_move() -> Self { + Self { + start: Self::MOVE_ANCHOR, + end: Self::MOVE_ANCHOR, + } + } + + #[inline] + pub(crate) fn kind(&self) -> RichtextChunkKind { + match self.start { + Self::START_STYLE_ANCHOR => RichtextChunkKind::StyleAnchor, + Self::END_STYLE_ANCHOR => RichtextChunkKind::StyleAnchor, + Self::UNKNOWN => RichtextChunkKind::Unknown, + Self::MOVE_ANCHOR => RichtextChunkKind::MoveAnchor, + _ => RichtextChunkKind::Text, + } + } + + #[inline(always)] + pub fn len(&self) -> usize { + match self.start { + Self::UNKNOWN => self.end as usize, + Self::START_STYLE_ANCHOR | Self::END_STYLE_ANCHOR | Self::MOVE_ANCHOR => 1, + _ => (self.end - self.start) as usize, + } + } + + #[inline] + pub(crate) fn value(&self) -> RichtextChunkValue { + match self.start { + Self::UNKNOWN => RichtextChunkValue::Unknown(self.end), + Self::START_STYLE_ANCHOR => RichtextChunkValue::StyleAnchor { + id: self.end, + anchor_type: AnchorType::Start, + }, + Self::END_STYLE_ANCHOR => RichtextChunkValue::StyleAnchor { + id: self.end, + anchor_type: AnchorType::End, + }, + Self::MOVE_ANCHOR => RichtextChunkValue::MoveAnchor, + _ => RichtextChunkValue::Text(self.start..self.end), + } + } +} + +impl Mergeable for RichtextChunk { + fn can_merge(&self, rhs: &Self) -> bool { + match (self.kind(), rhs.kind()) { + (RichtextChunkKind::Text, RichtextChunkKind::Text) => self.end == rhs.start, + (RichtextChunkKind::Unknown, RichtextChunkKind::Unknown) => true, + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self.kind(), rhs.kind()) { + (RichtextChunkKind::Text, RichtextChunkKind::Text) => self.end = rhs.end, + (RichtextChunkKind::Unknown, RichtextChunkKind::Unknown) => self.end += rhs.end, + _ => unreachable!(), + } + } + + fn merge_left(&mut self, left: &Self) { + match (self.kind(), left.kind()) { + (RichtextChunkKind::Text, RichtextChunkKind::Text) => self.start = left.start, + (RichtextChunkKind::Unknown, RichtextChunkKind::Unknown) => self.end += left.end, + _ => unreachable!(), + } + } +} + +impl HasLength for RichtextChunk { + #[inline(always)] + fn rle_len(&self) -> usize { + self.len() + } +} + +impl Sliceable for RichtextChunk { + fn _slice(&self, range: Range) -> Self { + match self.kind() { + RichtextChunkKind::Text => { + assert!( + range.len() <= self.len(), + "range: {:?}, self: {:?}", + range, + self + ); + Self { + start: self.start + range.start as u32, + end: self.start + range.end as u32, + } + } + RichtextChunkKind::StyleAnchor | RichtextChunkKind::MoveAnchor => { + assert_eq!(range.len(), 1); + *self + } + RichtextChunkKind::Unknown => { + assert!(range.len() <= self.len()); + Self { + start: Self::UNKNOWN, + end: range.len() as u32, + } + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(super) struct FugueSpan { + pub id: IdFull, + /// Sometimes, the id of the span is just a placeholder. + /// This field is used to track the real id of the span. + /// We know the real id when deletion happens because we + /// have the start_id info for each deletion. + pub real_id: Option, + /// The status at the current version + pub status: Status, + /// The status at the `new` version. + /// It's used when calculating diff. + pub diff_status: Option, + pub origin_left: Option, + pub origin_right: Option, + pub content: RichtextChunk, +} + +pub(super) enum DiffStatus { + NotChanged, + Created, + Deleted, +} + +impl FugueSpan { + #[inline(always)] + pub fn id_span(&self) -> IdSpan { + IdSpan::new( + self.id.peer, + self.id.counter, + self.id.counter + self.content.len() as Counter, + ) + } + + #[inline] + pub fn diff(&self) -> DiffStatus { + if self.diff_status.is_none() { + return DiffStatus::NotChanged; + } + + match ( + self.status.is_activated(), + self.diff_status.unwrap().is_activated(), + ) { + (true, false) => DiffStatus::Deleted, + (false, true) => DiffStatus::Created, + _ => DiffStatus::NotChanged, + } + } + + pub fn is_activated_in_diff(&self) -> bool { + match self.diff_status { + Some(d) => d.is_activated(), + None => self.status.is_activated(), + } + } + + pub(crate) fn real_id(&self) -> ID { + if let Some(id) = self.real_id { + id.to_id() + } else { + self.id.id() + } + } +} + +impl Sliceable for FugueSpan { + fn _slice(&self, range: Range) -> Self { + Self { + id: self.id.inc(range.start as Counter), + real_id: self.real_id.map(|x| x.inc(range.start as Counter)), + status: self.status, + diff_status: self.diff_status, + origin_left: if range.start == 0 { + self.origin_left + } else { + Some( + self.id + .inc((range.start - 1) as Counter) + .id() + .try_into() + .unwrap(), + ) + }, + origin_right: self.origin_right, + content: self.content._slice(range), + } + } +} + +impl HasLength for FugueSpan { + #[inline(always)] + fn rle_len(&self) -> usize { + self.content.len() + } +} + +impl Mergeable for FugueSpan { + fn can_merge(&self, rhs: &Self) -> bool { + self.id.peer == rhs.id.peer + && self.status == rhs.status + && self.diff_status == rhs.diff_status + && self.id.counter + self.content.len() as Counter == rhs.id.counter + && self.id.lamport + self.content.len() as Lamport == rhs.id.lamport + && rhs.origin_left.is_some() + && rhs.origin_left.unwrap().peer == self.id.peer + && rhs.origin_left.unwrap().counter.get() + == self.id.counter + self.content.len() as Counter - 1 + && self.origin_right == rhs.origin_right + && self.content.can_merge(&rhs.content) + && ((self.real_id.is_none() && rhs.real_id.is_none()) + || (self.real_id.is_some() + && rhs.real_id.is_some() + && self.real_id.unwrap().inc(self.content.len() as i32) + == rhs.real_id.unwrap())) + } + + fn merge_right(&mut self, rhs: &Self) { + self.content.merge_right(&rhs.content); + } + + fn merge_left(&mut self, left: &Self) { + self.id = left.id; + self.real_id = left.real_id; + self.origin_left = left.origin_left; + self.content.merge_left(&left.content); + } +} + +impl TryInsert for FugueSpan { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } +} + +impl CanRemove for FugueSpan { + fn can_remove(&self) -> bool { + self.content.len() == 0 + } +} + +impl FugueSpan { + #[allow(unused)] + pub fn new(id: IdFull, content: RichtextChunk) -> Self { + Self { + id, + real_id: if id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(id.id().try_into().unwrap()) + }, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + content, + } + } + + #[inline(always)] + pub fn is_activated(&self) -> bool { + self.status.is_activated() + } + + #[inline] + pub fn activated_len(&self) -> usize { + if self.is_activated() { + self.content.len() + } else { + 0 + } + } +} + +impl HasId for FugueSpan { + fn id_start(&self) -> ID { + self.id.id() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Hash, Copy)] +pub(super) struct Status { + /// is this span from a future operation + pub future: bool, + pub delete_times: i16, +} + +impl Status { + #[inline(always)] + pub fn is_activated(&self) -> bool { + self.delete_times == 0 && !self.future + } +} diff --git a/vendor/loro-internal/src/container/richtext/richtext_state.rs b/vendor/loro-internal/src/container/richtext/richtext_state.rs new file mode 100644 index 00000000..47603edd --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/richtext_state.rs @@ -0,0 +1,3491 @@ +use append_only_bytes::BytesSlice; +use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, Cursor, LeafIndex, +}; +use loro_common::{Counter, IdFull, IdSpan, LoroError, LoroResult, LoroValue, ID}; +use query::{ByteQuery, ByteQueryT}; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::{ser::SerializeStruct, Serialize}; +use std::{ + fmt::{Display, Formatter}, + ops::{Bound, RangeBounds}, +}; +use std::{ + ops::{Add, AddAssign, Range, Sub}, + str::Utf8Error, + sync::Arc, +}; +use tracing::instrument; + +use crate::{ + container::richtext::style_range_map::EMPTY_STYLES, + delta::{DeltaValue, StyleMeta}, + utils::query_by_len::{EntityIndexQueryWithEventIndex, IndexQueryWithEntityIndex, QueryByLen}, +}; + +use self::query::{ + EntityQuery, EntityQueryT, EventIndexQuery, EventIndexQueryT, UnicodeQuery, UnicodeQueryT, + Utf16Query, Utf16QueryT, +}; + +use super::{ + style_range_map::{IterAnchorItem, StyleRangeMap, Styles}, + AnchorType, RichtextSpan, StyleKey, StyleOp, +}; + +pub(crate) use crate::cursor::PosType; + +/// §act-eleven A11.10 (flowstate vendor patch): folds a stream of +/// `(text, styles)` spans into the delta-shaped `LoroValue` list that +/// `get_richtext_value` has always produced — adjacent spans whose VISIBLE +/// (null-filtered) attribute values are equal merge into one insert entry. +/// Both the built-state and loader-fast-path value functions are expressed on +/// this single builder, so the merge law lives in exactly one place. +#[derive(Default)] +struct RichtextValueBuilder { + ans: Vec, + last_attributes: Option, +} + +impl RichtextValueBuilder { + fn push(&mut self, text: &str, styles: &StyleMeta) { + let attributes: LoroValue = styles.to_value(); + if let Some(last) = self.last_attributes.as_ref() { + if &attributes == last { + let hash_map = self.ans.last_mut().unwrap().as_map_mut().unwrap(); + let s = hash_map + .make_mut() + .get_mut("insert") + .unwrap() + .as_string_mut() + .unwrap(); + s.make_mut().push_str(text); + return; + } + } + + let mut value = FxHashMap::default(); + value.insert("insert".into(), LoroValue::String(text.into())); + if !attributes.as_map().unwrap().is_empty() { + value.insert("attributes".into(), attributes.clone()); + } + self.ans.push(LoroValue::Map(value.into())); + self.last_attributes = Some(attributes); + } + + fn finish(self) -> LoroValue { + LoroValue::List(self.ans.into()) + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct RichtextState { + tree: BTree, + style_ranges: Option>, + cached_cursor: Option, +} + +impl Display for RichtextState { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + for span in self.tree.iter() { + match span { + RichtextStateChunk::Style { .. } => {} + RichtextStateChunk::Text(s) => { + f.write_str(s.as_str())?; + } + } + } + + Ok(()) + } +} + +use cache::CachedCursor; + +mod cache { + use super::*; + + #[derive(Clone, Debug)] + pub(super) struct CachedCursor { + leaf: LeafIndex, + index: FxHashMap, + } + + impl RichtextState { + pub(super) fn clear_cache(&mut self) { + self.cached_cursor = None; + } + + pub(super) fn record_cache( + &mut self, + leaf: LeafIndex, + pos: usize, + pos_type: PosType, + entity_offset: usize, + entity_index: Option, + ) { + let offset = if entity_offset != 0 { + let elem = self.tree.get_elem(leaf).unwrap(); + entity_offset_to_pos_type_offset(pos_type, elem, entity_offset) + } else { + 0 + }; + + let pos = pos - offset; + match &mut self.cached_cursor { + Some(c) => { + if c.leaf == leaf { + c.index.insert(pos_type, pos); + } else { + c.leaf = leaf; + c.index.clear(); + c.index.insert(pos_type, pos); + } + } + None => { + self.cached_cursor = Some(CachedCursor { + leaf, + index: FxHashMap::default(), + }); + self.cached_cursor + .as_mut() + .unwrap() + .index + .insert(pos_type, pos); + } + } + if let Some(entity_index) = entity_index { + self.cached_cursor + .as_mut() + .unwrap() + .index + .insert(PosType::Entity, entity_index - entity_offset); + } + } + + pub(super) fn try_get_cache_or_clean( + &mut self, + index: usize, + pos_type: PosType, + ) -> Option { + let mut cursor = self.cached_cursor.take(); + let ans = 'block: { + match &mut cursor { + Some(c) => { + let cache_index = c.index.get(&pos_type); + if let Some(cache_index) = cache_index { + if index < *cache_index { + return None; + } + } + + let cached_index = match c.index.entry(pos_type) { + std::collections::hash_map::Entry::Vacant(vacant_entry) => { + let index = self.get_index_from_cursor( + Cursor { + leaf: c.leaf, + offset: 0, + }, + pos_type, + )?; + vacant_entry.insert(index); + index + } + std::collections::hash_map::Entry::Occupied(occupied_entry) => { + *occupied_entry.get() + } + }; + + if index < cached_index { + return None; + } + + if cached_index == index { + break 'block Some(Cursor { + leaf: c.leaf, + offset: 0, + }); + } + + let elem = self.tree.get_elem(c.leaf)?; + let elem_len = elem.len_with(pos_type); + if cached_index + elem_len == index { + let offset = elem.len_with(PosType::Entity); + break 'block Some(Cursor { + leaf: c.leaf, + offset, + }); + } + + if index > cached_index + elem_len { + return None; + } + + let offset = + pos_type_offset_to_entity_offset(pos_type, elem, index - cached_index)?; + Some(Cursor { + leaf: c.leaf, + offset, + }) + } + None => None, + } + }; + + self.cached_cursor = cursor; + #[cfg(debug_assertions)] + { + if let Some(c) = ans.as_ref() { + let actual = self.get_index_from_cursor(*c, pos_type); + assert_eq!(actual.unwrap(), index); + } + } + ans + } + + pub(super) fn get_cache_entity_index(&mut self) -> Option { + let mut cursor = self.cached_cursor.take()?; + let ans = { + let leaf = cursor.leaf; + match cursor.index.entry(PosType::Entity) { + std::collections::hash_map::Entry::Vacant(vacant_entry) => { + let index = self + .get_index_from_cursor(Cursor { leaf, offset: 0 }, PosType::Entity)?; + vacant_entry.insert(index); + index + } + std::collections::hash_map::Entry::Occupied(occupied_entry) => { + *occupied_entry.get() + } + } + }; + + self.cached_cursor = Some(cursor); + Some(ans) + } + + pub(crate) fn check_cache(&self) { + #[cfg(debug_assertions)] + { + if let Some(c) = &self.cached_cursor { + for (pos_type, index) in &c.index { + let actual = self.get_index_from_cursor( + Cursor { + leaf: c.leaf, + offset: 0, + }, + *pos_type, + ); + + assert_eq!(actual.unwrap(), *index); + } + } + } + } + } +} + +pub(crate) use text_chunk::TextChunk; +mod text_chunk { + use std::ops::Range; + + use append_only_bytes::BytesSlice; + use loro_common::{IdFull, ID}; + + #[derive(Clone, PartialEq)] + pub(crate) struct TextChunk { + bytes: BytesSlice, + unicode_len: i32, + utf16_len: i32, + id: IdFull, + } + + impl std::fmt::Debug for TextChunk { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextChunk") + .field("text", &self.as_str()) + .field("unicode_len", &self.unicode_len) + .field("utf16_len", &self.utf16_len) + .field("id", &self.id) + .finish() + } + } + + impl TextChunk { + pub fn new(bytes: BytesSlice, id: IdFull) -> Self { + let mut utf16_len = 0; + let mut unicode_len = 0; + for c in std::str::from_utf8(&bytes).unwrap().chars() { + utf16_len += c.len_utf16(); + unicode_len += 1; + } + + Self { + unicode_len, + bytes, + utf16_len: utf16_len as i32, + id, + } + } + + #[inline] + pub fn id(&self) -> ID { + self.id.id() + } + + #[inline] + #[allow(unused)] + pub fn id_full(&self) -> IdFull { + self.id + } + + #[inline] + pub fn bytes(&self) -> &BytesSlice { + &self.bytes + } + + #[inline] + pub fn as_str(&self) -> &str { + // SAFETY: We know that the text is valid UTF-8 + unsafe { std::str::from_utf8_unchecked(&self.bytes) } + } + + #[inline] + pub fn len(&self) -> i32 { + self.unicode_len + } + + #[inline] + pub fn utf8_len(&self) -> i32 { + self.bytes.len() as i32 + } + + #[inline] + pub fn unicode_len(&self) -> i32 { + self.unicode_len + } + + #[inline] + pub fn utf16_len(&self) -> i32 { + self.utf16_len + } + + #[inline] + pub fn event_len(&self) -> i32 { + if cfg!(feature = "wasm") { + self.utf16_len + } else { + self.unicode_len + } + } + + /// Convert a unicode index on this text to an event index + pub fn convert_unicode_offset_to_event_offset(&self, offset: usize) -> usize { + if cfg!(feature = "wasm") { + let mut event_offset = 0; + for (i, c) in self.as_str().chars().enumerate() { + if i == offset { + return event_offset; + } + event_offset += c.len_utf16(); + } + event_offset + } else { + offset + } + } + + pub fn new_empty() -> Self { + Self { + unicode_len: 0, + bytes: BytesSlice::empty(), + utf16_len: 0, + // This is a dummy value. + // It's fine because the length is 0. We never actually use this value. + id: IdFull::NONE_ID, + } + } + + pub(crate) fn delete_by_entity_index( + &mut self, + unicode_offset: usize, + unicode_len: usize, + ) -> (Option, usize) { + let s = self.as_str(); + let mut start_byte = 0; + let mut end_byte = self.bytes().len(); + let start_unicode_index = unicode_offset; + let end_unicode_index = unicode_offset + unicode_len; + let mut start_utf16_index = 0; + let mut current_utf16_index = 0; + let mut current_utf8_index = 0; + for (current_unicode_index, c) in s.chars().enumerate() { + if current_unicode_index == start_unicode_index { + start_utf16_index = current_utf16_index; + start_byte = current_utf8_index; + } + + if current_unicode_index == end_unicode_index { + end_byte = current_utf8_index; + break; + } + + current_utf16_index += c.len_utf16(); + current_utf8_index += c.len_utf8(); + } + + self.utf16_len -= (current_utf16_index - start_utf16_index) as i32; + + let event_len = if cfg!(feature = "wasm") { + current_utf16_index - start_utf16_index + } else { + unicode_len + }; + + self.unicode_len -= unicode_len as i32; + let next = match (start_byte == 0, end_byte == self.bytes.len()) { + (true, true) => { + self.bytes = BytesSlice::empty(); + None + } + (true, false) => { + self.bytes.slice_(end_byte..); + self.id = self.id.inc(end_unicode_index as i32); + None + } + (false, true) => { + self.bytes.slice_(..start_byte); + None + } + (false, false) => { + let next = self.bytes.slice_clone(end_byte..); + let next = Self::new(next, self.id.inc(end_unicode_index as i32)); + self.unicode_len -= next.unicode_len; + self.utf16_len -= next.utf16_len; + self.bytes.slice_(..start_byte); + Some(next) + } + }; + + self.check(); + if let Some(next) = next.as_ref() { + next.check(); + } + (next, event_len) + } + + fn check(&self) { + if !cfg!(any(debug_assertions, test)) { + return; + } + + let bytes_len = self.bytes.len() as i32; + if bytes_len == self.unicode_len { + assert_eq!(self.utf16_len, self.unicode_len); + return; + } + + assert_eq!(self.unicode_len, self.as_str().chars().count() as i32); + assert_eq!( + self.utf16_len, + self.as_str().chars().map(|c| c.len_utf16()).sum::() as i32 + ); + } + + pub(crate) fn entity_range_to_event_range(&self, range: Range) -> Range { + if cfg!(feature = "wasm") { + assert!(range.start <= range.end); + if range.start == 0 && range.end == self.unicode_len as usize { + return 0..self.utf16_len as usize; + } + + let mut start = 0; + let mut end = 0; + let mut utf16_index = 0; + for (unicode_index, c) in self.as_str().chars().enumerate() { + if unicode_index == range.start { + start = utf16_index; + } + if unicode_index == range.end { + end = utf16_index; + break; + } + + utf16_index += c.len_utf16(); + } + + if end == 0 { + end = utf16_index; + } + + start..end + } else { + range + } + } + } + + impl generic_btree::rle::HasLength for TextChunk { + fn rle_len(&self) -> usize { + self.unicode_len as usize + } + } + + impl generic_btree::rle::Sliceable for TextChunk { + fn _slice(&self, range: Range) -> Self { + assert!(range.start < range.end); + let total_unicode_len = self.unicode_len as usize; + if range.start == 0 && range.end == total_unicode_len { + return self.clone(); + } + + // Fast path for ASCII text: unicode index == byte index, and utf16 index == unicode index. + if self.bytes.len() as i32 == self.unicode_len { + let ans = Self { + unicode_len: range.len() as i32, + bytes: self.bytes.slice_clone(range.start..range.end), + utf16_len: range.len() as i32, + id: self.id.inc(range.start as i32), + }; + ans.check(); + return ans; + } + + // Fast path for slicing a suffix/prefix to avoid scanning the whole chunk. + if range.end == total_unicode_len { + let mut utf16_offset = 0; + let mut start_byte = self.bytes.len(); + for (unicode_index, (byte_index, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == range.start { + start_byte = byte_index; + break; + } + utf16_offset += c.len_utf16(); + } + + let ans = Self { + unicode_len: (total_unicode_len - range.start) as i32, + bytes: self.bytes.slice_clone(start_byte..), + utf16_len: self.utf16_len - utf16_offset as i32, + id: self.id.inc(range.start as i32), + }; + ans.check(); + return ans; + } + + if range.start == 0 { + let mut utf16_len = 0; + let mut end_byte = self.bytes.len(); + for (unicode_index, (byte_index, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == range.end { + end_byte = byte_index; + break; + } + utf16_len += c.len_utf16(); + } + + let ans = Self { + unicode_len: range.end as i32, + bytes: self.bytes.slice_clone(..end_byte), + utf16_len: utf16_len as i32, + id: self.id, + }; + ans.check(); + return ans; + } + + // General middle slice. + let mut utf16_len = 0; + let mut start = 0; + let mut end = 0; + let mut started = false; + let mut last_unicode_index = 0; + for (unicode_index, (i, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == range.start { + start = i; + started = true; + } + + if unicode_index == range.end { + end = i; + break; + } + if started { + utf16_len += c.len_utf16(); + } + + last_unicode_index = unicode_index; + } + + assert!(started); + if end == 0 { + assert_eq!(last_unicode_index + 1, range.end); + end = self.bytes.len(); + } + + let ans = Self { + unicode_len: range.len() as i32, + bytes: self.bytes.slice_clone(start..end), + utf16_len: utf16_len as i32, + id: self.id.inc(range.start as i32), + }; + ans.check(); + ans + } + + fn split(&mut self, pos: usize) -> Self { + let mut utf16_len = 0; + let mut byte_offset = 0; + for (unicode_index, (i, c)) in self.as_str().char_indices().enumerate() { + if unicode_index == pos { + byte_offset = i; + break; + } + + utf16_len += c.len_utf16(); + } + let right = Self { + unicode_len: self.unicode_len - pos as i32, + bytes: self.bytes.slice_clone(byte_offset..), + utf16_len: self.utf16_len - utf16_len as i32, + id: self.id.inc(pos as i32), + }; + + self.unicode_len = pos as i32; + self.utf16_len = utf16_len as i32; + self.bytes.slice_(..byte_offset); + right.check(); + self.check(); + right + } + } + + impl generic_btree::rle::Mergeable for TextChunk { + fn can_merge(&self, rhs: &Self) -> bool { + self.bytes.can_merge(&rhs.bytes) && self.id.inc(self.unicode_len) == rhs.id + } + + fn merge_right(&mut self, rhs: &Self) { + self.bytes.try_merge(&rhs.bytes).unwrap(); + self.utf16_len += rhs.utf16_len; + self.unicode_len += rhs.unicode_len; + self.check(); + } + + fn merge_left(&mut self, left: &Self) { + let mut new = left.bytes.clone(); + new.try_merge(&self.bytes).unwrap(); + self.bytes = new; + self.utf16_len += left.utf16_len; + self.unicode_len += left.unicode_len; + self.id = left.id; + self.check(); + } + } +} + +// TODO: change visibility back to crate after #116 is done +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum RichtextStateChunk { + Text(TextChunk), + Style { + style: Arc, + anchor_type: AnchorType, + }, +} + +impl Default for RichtextStateChunk { + fn default() -> Self { + Self::new_empty() + } +} + +impl RichtextStateChunk { + pub fn new_text(s: BytesSlice, id: IdFull) -> Self { + Self::Text(TextChunk::new(s, id)) + } + + pub fn new_empty() -> Self { + Self::Text(TextChunk::new_empty()) + } + + pub fn new_style(style: Arc, anchor_type: AnchorType) -> Self { + Self::Style { style, anchor_type } + } + + pub(crate) fn get_id_span(&self) -> IdSpan { + match self { + RichtextStateChunk::Text(t) => { + let id = t.id(); + IdSpan::new(id.peer, id.counter, id.counter + t.unicode_len() as Counter) + } + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + AnchorType::Start => style.id().into(), + AnchorType::End => { + let id = style.id(); + id.to_span(1) + } + }, + } + } + + pub(crate) fn counter(&self) -> Counter { + match self { + RichtextStateChunk::Text(t) => t.id().counter, + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + AnchorType::Start => style.id().counter, + AnchorType::End => { + let id = style.id(); + id.counter + 1 + } + }, + } + } + + pub fn entity_range_to_event_range(&self, range: Range) -> Range { + match self { + RichtextStateChunk::Text(t) => t.entity_range_to_event_range(range), + RichtextStateChunk::Style { .. } => { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + 0..1 + } + } + } + + pub fn len_with(&self, pos_type: PosType) -> usize { + match self { + RichtextStateChunk::Text(t) => match pos_type { + PosType::Bytes => t.utf8_len() as usize, + PosType::Utf16 => t.utf16_len() as usize, + PosType::Event if cfg!(feature = "wasm") => t.utf16_len() as usize, + PosType::Event => t.unicode_len() as usize, + PosType::Entity => t.unicode_len() as usize, + PosType::Unicode => t.unicode_len() as usize, + }, + RichtextStateChunk::Style { .. } => { + if let PosType::Entity = pos_type { + 1 + } else { + 0 + } + } + } + } +} + +impl loro_delta::delta_trait::DeltaValue for RichtextStateChunk {} + +impl DeltaValue for RichtextStateChunk { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + Err(other) + } + + fn take(&mut self, length: usize) -> Self { + let mut right = self.split(length); + std::mem::swap(self, &mut right); + right + } + + fn length(&self) -> usize { + self.rle_len() + } +} + +impl Serialize for RichtextStateChunk { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + RichtextStateChunk::Text(text) => { + let mut state = serializer.serialize_struct("RichtextStateChunk", 3)?; + state.serialize_field("type", "Text")?; + state.serialize_field("unicode_len", &text.unicode_len())?; + state.serialize_field("text", text.as_str())?; + state.end() + } + RichtextStateChunk::Style { style, anchor_type } => { + let mut state = serializer.serialize_struct("RichtextStateChunk", 3)?; + state.serialize_field("type", "Style")?; + state.serialize_field("style", &style.key)?; + state.serialize_field("anchor_type", anchor_type)?; + state.end() + } + } + } +} + +impl RichtextStateChunk { + pub fn try_new(s: BytesSlice, id: IdFull) -> Result { + std::str::from_utf8(&s)?; + Ok(RichtextStateChunk::Text(TextChunk::new(s, id))) + } + + pub fn from_style(style: Arc, anchor_type: AnchorType) -> Self { + Self::Style { style, anchor_type } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + RichtextStateChunk::Text(text) => Some(text.as_str()), + _ => None, + } + } +} + +impl HasLength for RichtextStateChunk { + fn rle_len(&self) -> usize { + match self { + RichtextStateChunk::Text(s) => s.rle_len(), + RichtextStateChunk::Style { .. } => 1, + } + } +} + +impl Mergeable for RichtextStateChunk { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + (RichtextStateChunk::Text(l), RichtextStateChunk::Text(r)) => l.can_merge(r), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + (RichtextStateChunk::Text(l), RichtextStateChunk::Text(r)) => l.merge_right(r), + _ => unreachable!(), + } + } + + fn merge_left(&mut self, left: &Self) { + match (self, left) { + (RichtextStateChunk::Text(this), RichtextStateChunk::Text(left)) => { + this.merge_left(left) + } + _ => unreachable!(), + } + } +} + +impl Sliceable for RichtextStateChunk { + fn _slice(&self, range: Range) -> Self { + match self { + RichtextStateChunk::Text(s) => RichtextStateChunk::Text(s._slice(range)), + RichtextStateChunk::Style { style, anchor_type } => { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + RichtextStateChunk::Style { + style: style.clone(), + anchor_type: *anchor_type, + } + } + } + } + + fn split(&mut self, pos: usize) -> Self { + match self { + RichtextStateChunk::Text(s) => RichtextStateChunk::Text(s.split(pos)), + RichtextStateChunk::Style { .. } => { + unreachable!() + } + } + } +} + +impl TryInsert for RichtextStateChunk { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } +} + +impl CanRemove for RichtextStateChunk { + fn can_remove(&self) -> bool { + self.rle_len() == 0 + } +} + +//TODO: start/end can be scanned in one loop, but now it takes twice the time +fn unicode_slice(s: &str, start_index: usize, end_index: usize) -> Result<&str, ()> { + let (Some(start), Some(end)) = ( + unicode_to_utf8_index(s, start_index), + unicode_to_utf8_index(s, end_index), + ) else { + return Err(()); + }; + Ok(&s[start..end]) +} + +pub(crate) fn unicode_to_utf8_index(s: &str, unicode_index: usize) -> Option { + let mut current_unicode_index = 0; + for (byte_index, _) in s.char_indices() { + if current_unicode_index == unicode_index { + return Some(byte_index); + } + current_unicode_index += 1; + } + + if current_unicode_index == unicode_index { + return Some(s.len()); + } + + None +} + +pub(crate) fn unicode_to_utf16_index(s: &str, unicode_index: usize) -> Option { + if unicode_index == 0 { + return Some(0); + } + + let mut current_unicode_index = 0; + let mut current_utf16_index = 0; + for c in s.chars() { + let len = c.len_utf16(); + current_unicode_index += 1; + current_utf16_index += len; + if current_unicode_index == unicode_index { + return Some(current_utf16_index); + } + } + + None +} + +pub(crate) fn utf16_to_utf8_index(s: &str, utf16_index: usize) -> Option { + if utf16_index == 0 { + return Some(0); + } + + let mut current_utf16_index = 0; + for (byte_index, c) in s.char_indices() { + let len = c.len_utf16(); + current_utf16_index += len; + if current_utf16_index == utf16_index { + return Some(byte_index + c.len_utf8()); + } + } + + if current_utf16_index == utf16_index { + return Some(s.len()); + } + + None +} + +/// Returns the unicode index of the character at the given utf16 index. +/// +/// If the given utf16 index is not at the correct boundary, returns the unicode index of the +/// character before the given utf16 index. +pub(crate) fn utf16_to_unicode_index(s: &str, utf16_index: usize) -> Result { + if utf16_index == 0 { + return Ok(0); + } + + let mut current_utf16_index = 0; + let mut current_unicode_index = 0; + for (i, c) in s.chars().enumerate() { + current_utf16_index += c.len_utf16(); + if current_utf16_index == utf16_index { + return Ok(i + 1); + } + if current_utf16_index > utf16_index { + loro_common::info!("WARNING: UTF16 MISMATCHED!"); + return Err(i); + } + current_unicode_index = i + 1; + } + + loro_common::info!("WARNING: UTF16 MISMATCHED!"); + Err(current_unicode_index) +} + +pub(crate) fn utf8_to_unicode_index(s: &str, utf8_index: usize) -> Result { + if utf8_index == 0 { + return Ok(0); + } + + let mut current_utf8_index = 0; + let mut current_unicode_index = 0; + for (i, c) in s.chars().enumerate() { + let char_start = current_utf8_index; + current_utf8_index += c.len_utf8(); + + if utf8_index == char_start { + return Ok(i); + } + + if utf8_index < current_utf8_index { + loro_common::info!("WARNING: UTF-8 index is in the middle of a codepoint!"); + return Err(i); + } + current_unicode_index = i + 1; + } + + if current_utf8_index == utf8_index { + Ok(current_unicode_index) + } else { + Err(current_unicode_index) + } +} + +#[derive(Clone, Debug, Copy, PartialEq, Eq, Default)] +pub(crate) struct PosCache { + pub(super) unicode_len: i32, + pub(super) bytes: i32, + pub(super) utf16_len: i32, + pub(crate) entity_len: i32, +} + +impl PosCache { + pub(crate) fn event_len(&self) -> i32 { + if cfg!(feature = "wasm") { + self.utf16_len + } else { + self.unicode_len + } + } + + #[allow(unused)] + fn get_len(&self, pos_type: PosType) -> i32 { + match pos_type { + PosType::Bytes => self.bytes, + PosType::Unicode => self.unicode_len, + PosType::Utf16 => self.utf16_len, + PosType::Entity => self.entity_len, + PosType::Event => self.event_len(), + } + } +} + +impl AddAssign for PosCache { + fn add_assign(&mut self, rhs: Self) { + self.unicode_len += rhs.unicode_len; + self.bytes += rhs.bytes; + self.utf16_len += rhs.utf16_len; + self.entity_len += rhs.entity_len; + } +} + +impl Add for PosCache { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self { + bytes: self.bytes + rhs.bytes, + unicode_len: self.unicode_len + rhs.unicode_len, + utf16_len: self.utf16_len + rhs.utf16_len, + entity_len: self.entity_len + rhs.entity_len, + } + } +} + +impl Sub for PosCache { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self { + bytes: self.bytes - rhs.bytes, + unicode_len: self.unicode_len - rhs.unicode_len, + utf16_len: self.utf16_len - rhs.utf16_len, + entity_len: self.entity_len - rhs.entity_len, + } + } +} + +impl CanRemove for PosCache { + fn can_remove(&self) -> bool { + self.bytes == 0 + } +} + +pub(crate) struct RichtextTreeTrait; + +#[derive(Debug)] +pub(crate) struct EntityRangeInfo { + pub id_start: ID, + pub entity_start: usize, + pub entity_end: usize, + pub event_len: usize, +} + +impl EntityRangeInfo { + pub fn entity_len(&self) -> usize { + self.entity_end - self.entity_start + } +} + +impl BTreeTrait for RichtextTreeTrait { + type Elem = RichtextStateChunk; + + type Cache = PosCache; + + type CacheDiff = PosCache; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let mut new_cache = PosCache::default(); + for child in caches { + new_cache += child.cache; + } + + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + #[inline] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + match elem { + RichtextStateChunk::Text(s) => PosCache { + bytes: s.bytes().len() as i32, + unicode_len: s.unicode_len(), + utf16_len: s.utf16_len(), + entity_len: s.unicode_len(), + }, + RichtextStateChunk::Style { .. } => PosCache { + bytes: 0, + unicode_len: 0, + utf16_len: 0, + entity_len: 1, + }, + } + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + #[inline(always)] + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + PosCache { + bytes: cache_lhs.bytes - cache_rhs.bytes, + unicode_len: cache_lhs.unicode_len - cache_rhs.unicode_len, + utf16_len: cache_lhs.utf16_len - cache_rhs.utf16_len, + entity_len: cache_lhs.entity_len - cache_rhs.entity_len, + } + } +} + +// This query implementation will prefer right element when both left element and right element are valid. +mod query { + use crate::utils::query_by_len::{IndexQuery, QueryByLen}; + + use super::*; + + #[cfg(not(feature = "wasm"))] + pub(super) type EventIndexQuery = UnicodeQuery; + #[cfg(feature = "wasm")] + pub(super) type EventIndexQuery = Utf16Query; + + #[cfg(not(feature = "wasm"))] + pub(super) type EventIndexQueryT = UnicodeQueryT; + #[cfg(feature = "wasm")] + pub(super) type EventIndexQueryT = Utf16QueryT; + + pub(super) struct UnicodeQueryT; + pub(super) type UnicodeQuery = IndexQuery; + + impl QueryByLen for UnicodeQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.unicode_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.rle_len(), + RichtextStateChunk::Style { .. } => 0, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if s.rle_len() >= left { + return (left, true); + } + + (left, false) + } + RichtextStateChunk::Style { .. } => (1, false), + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } + + pub(crate) struct Utf16QueryT; + pub(crate) type Utf16Query = IndexQuery; + + impl QueryByLen for Utf16QueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.utf16_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.utf16_len() as usize, + RichtextStateChunk::Style { .. } => 0, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if left == 0 { + return (0, true); + } + + // Allow left to not at the correct utf16 boundary. If so fallback to the last position. + // TODO: if we remove the use of query(pos-1), we won't need this fallback behavior + // WARNING: Unable to report error!!! + let offset = utf16_to_unicode_index(s.as_str(), left).unwrap_or_else(|e| e); + (offset, true) + } + RichtextStateChunk::Style { .. } => (1, false), + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } + + pub(super) struct EntityQueryT; + pub(super) type EntityQuery = IndexQuery; + + impl QueryByLen for EntityQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.rle_len(), + RichtextStateChunk::Style { .. } => 1, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if s.rle_len() >= left { + return (left, true); + } + + (left, false) + } + RichtextStateChunk::Style { .. } => { + if left == 0 { + return (0, true); + } + + (left, false) + } + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } + + pub(super) struct ByteQueryT; + pub(super) type ByteQuery = IndexQuery; + impl QueryByLen for ByteQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.bytes as usize + } + fn get_elem_len(elem: &::Elem) -> usize { + match elem { + RichtextStateChunk::Text(s) => s.utf8_len() as usize, + RichtextStateChunk::Style { .. } => 0, + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + match elem { + RichtextStateChunk::Text(s) => { + if left == 0 { + return (0, true); + } + + // Allow left to not at the correct utf16 boundary. If so fallback to the last position. + // TODO: if we remove the use of query(pos-1), we won't need this fallback behavior + // WARNING: Unable to report error!!! + let offset = utf8_to_unicode_index(s.as_str(), left).unwrap_or_else(|e| e); + (offset, true) + } + RichtextStateChunk::Style { .. } => (1, false), + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.entity_len as usize + } + } +} + +impl RichtextState { + pub(crate) fn from_chunks>>(i: I) -> Self { + Self { + tree: i.collect(), + style_ranges: Default::default(), + cached_cursor: None, + } + } + + pub(crate) fn get_entity_index_for_text_insert( + &mut self, + pos: usize, + pos_type: PosType, + ) -> Result<(usize, Option), LoroError> { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return Ok((0, None)); + } + + if let Some(c) = self.try_get_cache_or_clean(pos, pos_type) { + let entity_index = self.get_cache_entity_index().unwrap(); + Ok((entity_index + c.offset, Some(c))) + } else { + let (c, entity_index) = match pos_type { + PosType::Bytes => self.find_best_insert_pos::(pos), + PosType::Unicode => self.find_best_insert_pos::(pos), + PosType::Utf16 => self.find_best_insert_pos::(pos), + PosType::Entity => self.find_best_insert_pos::(pos), + PosType::Event => self.find_best_insert_pos::(pos), + }; + + if let Some(c) = c { + debug_assert_eq!( + self.get_index_from_cursor(c, PosType::Entity).unwrap(), + entity_index + ); + self.record_cache( + c.leaf, + entity_index, + PosType::Entity, + c.offset, + Some(entity_index), + ); + } + Ok((entity_index, c)) + } + }; + self.check_cache(); + result + } + + pub(crate) fn has_styles(&self) -> bool { + self.style_ranges + .as_ref() + .map(|x| x.has_style()) + .unwrap_or(false) + } + + pub(crate) fn range_has_style_key(&mut self, range: Range, key: &StyleKey) -> bool { + self.check_cache(); + let result = match self.style_ranges.as_ref() { + Some(s) => s.range_contains_key(range, key), + None => false, + }; + self.check_cache(); + result + } + + /// Return the entity range and text styles at the given range. + /// If in the target range the leaves are not in the same span, the returned styles would be None + pub(crate) fn get_entity_range_and_text_styles_at_range( + &mut self, + range: Range, + pos_type: PosType, + ) -> (Range, Option<&Styles>) { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return (0..0, None); + } + + let (start, _) = self + .get_entity_index_for_text_insert(range.start, pos_type) + .unwrap(); + let (end, _) = self + .get_entity_index_for_text_insert(range.end, pos_type) + .unwrap(); + if self.has_styles() { + ( + start..end, + self.style_ranges + .as_ref() + .unwrap() + .get_styles_of_range(start..end), + ) + } else { + (start..end, None) + } + }; + self.check_cache(); + result + } + + /// Get the insert text styles at the given entity index if we insert text at that position + /// + // TODO: PERF we can avoid this calculation by getting it when inserting new text + // but that requires a lot of changes + pub(crate) fn get_styles_at_entity_index_for_insert( + &mut self, + entity_index: usize, + ) -> StyleMeta { + self.check_cache(); + let result = { + if !self.has_styles() { + return Default::default(); + } + + self.style_ranges + .as_mut() + .unwrap() + .get_styles_for_insert(entity_index) + }; + self.check_cache(); + result + } + + /// This is used to accept changes from DiffCalculator + pub(crate) fn insert_at_entity_index( + &mut self, + entity_index: usize, + text: BytesSlice, + id: IdFull, + ) -> Cursor { + self.check_cache(); + let result = { + let elem = RichtextStateChunk::try_new(text, id).unwrap(); + self.style_ranges + .as_mut() + .map(|x| x.insert(entity_index, elem.rle_len())); + let q = &entity_index; + if let Some(c) = self.try_get_cache_or_clean(entity_index, PosType::Entity) { + let p = self.tree.prefer_left(c).unwrap_or(c); + self.tree.insert_by_path(p, elem).0 + } else { + match self.tree.query::(q) { + Some(result) => { + let p = self + .tree + .prefer_left(result.cursor) + .unwrap_or(result.cursor); + self.tree.insert_by_path(p, elem).0 + } + None => self.tree.push(elem), + } + } + }; + self.record_cache( + result.leaf, + entity_index, + PosType::Entity, + result.offset, + None, + ); + self.check_cache(); + result + } + + /// This is used to accept changes from DiffCalculator. + /// + /// Return (event_index, styles) + pub(crate) fn insert_elem_at_entity_index( + &mut self, + entity_index: usize, + elem: RichtextStateChunk, + ) -> (usize, &Styles) { + self.clear_cache(); + let result = { + debug_assert!( + entity_index <= self.len_entity(), + "entity_index={} len={} self={:#?}", + entity_index, + self.len_entity(), + &self + ); + + let (c, f) = self + .tree + .query_with_finder_return::(&entity_index); + let cursor = c.map(|x| x.cursor); + let event_index = f.event_index; + self.clear_cache(); + + match cursor { + Some(cursor) => { + let styles = self + .style_ranges + .as_mut() + .map(|x| x.insert(entity_index, elem.rle_len())) + .unwrap_or(&EMPTY_STYLES); + self.tree.insert_by_path(cursor, elem); + (event_index, styles) + } + None => { + let styles = self + .style_ranges + .as_mut() + .map(|x| x.insert(entity_index, elem.rle_len())) + .unwrap_or(&EMPTY_STYLES); + self.tree.push(elem); + (0, styles) + } + } + }; + result + } + + /// Convert cursor position to event index: + /// + /// - If feature="wasm", index is utf16 index, + /// - If feature!="wasm", index is unicode index, + pub(crate) fn cursor_to_event_index(&self, cursor: Cursor) -> usize { + self.check_cache(); + let result = self.get_index_from_cursor(cursor, PosType::Event).unwrap(); + self.check_cache(); + result + } + + pub(crate) fn cursor_to_unicode_index(&self, cursor: Cursor) -> usize { + self.check_cache(); + let result = self + .get_index_from_cursor(cursor, PosType::Unicode) + .unwrap(); + self.check_cache(); + result + } + + /// This method only updates `style_ranges`. + /// When this method is called, the style start anchor and the style end anchor should already have been inserted. + pub(crate) fn annotate_style_range(&mut self, range: Range, style: Arc) { + self.check_cache(); + self.clear_cache(); + self.ensure_style_ranges_mut().annotate(range, style, None); + self.check_cache(); + } + + /// This method only updates `style_ranges`. + /// When this method is called, the style start anchor and the style end anchor should already have been inserted. + /// + /// This method will return the event of this annotation in event length + pub(crate) fn annotate_style_range_with_event( + &mut self, + range: Range, + style: Arc, + ) -> impl Iterator + '_ { + self.check_cache(); + self.clear_cache(); + let mut ranges_in_entity_index: Vec<(StyleMeta, Range)> = Vec::new(); + let mut start = range.start; + let end = range.end; + self.ensure_style_ranges_mut().annotate( + range, + style, + Some(&mut |s, len| { + let range = start..start + len; + start += len; + ranges_in_entity_index.push((s.into(), range)); + }), + ); + + assert_eq!(ranges_in_entity_index.last().unwrap().1.end, end); + let mut converter = ContinuousIndexConverter::new(self); + let result = ranges_in_entity_index + .into_iter() + .filter_map(move |(meta, range)| { + let start = converter.convert_entity_index_to_event_index(range.start); + let end = converter.convert_entity_index_to_event_index(range.end); + if end == start { + return None; + } + + Some((meta, end - start)) + }); + self.check_cache(); + result + } + + /// init style ranges if not initialized + fn ensure_style_ranges_mut(&mut self) -> &mut StyleRangeMap { + self.clear_cache(); + let result = { + if self.style_ranges.is_none() { + self.style_ranges = Some(Box::default()); + } + + self.style_ranges.as_mut().unwrap() + }; + result + } + + pub(crate) fn get_char_by_event_index(&self, pos: usize) -> Result { + self.check_cache(); + let result = { + let cursor = self.tree.query::(&pos).unwrap().cursor; + let Some(str) = &self.tree.get_elem(cursor.leaf) else { + return Err(()); + }; + if cfg!(not(feature = "wasm")) { + let mut char_iter = str.as_str().unwrap().chars(); + match &mut char_iter.nth(cursor.offset) { + Some(c) => Ok(*c), + None => Err(()), + } + } else { + let s = str.as_str().unwrap(); + let utf16offset = unicode_to_utf16_index(s, cursor.offset).unwrap(); + // Convert utf16 offset to actual character by finding the character at that position + let char_iter = s.chars(); + let mut current_utf16_offset = 0; + + for c in char_iter { + if current_utf16_offset == utf16offset { + return Ok(c); + } + current_utf16_offset += c.len_utf16(); + if current_utf16_offset > utf16offset { + return Err(()); + } + } + Err(()) + } + }; + self.check_cache(); + result + } + + /// Find the best insert position based on the rich-text CRDT algorithm. + /// + /// The result is only different from `query` when there are style anchors around the insert pos. + /// Returns the right neighbor of the insert pos and the entity index. + /// + /// 1. Insertions occur before style anchors that contain the beginning of new marks. + /// 2. Insertions occur before style anchors that contain the end of bold-like marks + /// 3. Insertions occur after style anchors that contain the end of link-like marks + /// + /// Rule 1 should be satisfied before rules 2 and 3 to avoid creating a new style out of nowhere + /// + /// The current method will scan forward to find the last position that satisfies 1 and 2. + /// Then it scans backward to find the first position that satisfies 3. + fn find_best_insert_pos>( + &self, + pos: usize, + ) -> (Option, usize) { + type Query = IndexQueryWithEntityIndex; + if self.tree.is_empty() { + return (None, 0); + } + + // There are a range of elements may share the same unicode index + // because style anchors' lengths are zero in unicode index. + + // Find the start and the end of the range, and entity index of left cursor + let (left, right, mut entity_index) = if pos == 0 { + let left = self.tree.start_cursor().unwrap(); + let mut right = left; + let mut elem = self.tree.get_elem(right.leaf).unwrap(); + let entity_index = 0; + if matches!(elem, RichtextStateChunk::Text { .. }) { + return (Some(right), 0); + } else { + while Q::get_elem_len(elem) == 0 { + match self.tree.next_elem(right) { + Some(r) => { + right = r; + elem = self.tree.get_elem(right.leaf).unwrap(); + } + None => { + right.offset = elem.rle_len(); + break; + } + } + } + + (left, right, entity_index) + } + } else { + // The query prefers right when there are empty elements (style anchors) + // So the nodes between (pos-1) and pos are all style anchors. + let (q, f) = self.tree.query_with_finder_return::>(&(pos - 1)); + let q = q.unwrap(); + let mut entity_index = f.entity_index(); + + let elem = self.tree.get_elem(q.leaf()).unwrap(); + let mut right = q.cursor; + right.offset += 1; + entity_index += 1; + if elem.rle_len() > right.offset { + // The cursor is in the middle of a style anchor + return (Some(right), entity_index); + } + + match self.tree.next_elem(q.cursor) { + // If next is None, we know the range is empty, return directly + None => return (self.tree.end_cursor(), entity_index), + Some(x) => { + assert_eq!(right.offset, elem.rle_len()); + right = x; + let mut elem = self.tree.get_elem(right.leaf).unwrap(); + if matches!(elem, RichtextStateChunk::Text { .. }) { + return (Some(right), entity_index); + } + + let left = x; + while matches!(elem, RichtextStateChunk::Style { .. }) { + match self.tree.next_elem(right) { + Some(r) => { + right = r; + elem = self.tree.get_elem(right.leaf).unwrap(); + } + None => { + // this is last element + right.offset = 1; + break; + } + } + } + + (left, right, entity_index) + } + } + }; + + let mut iter = left; + + // need to scan from left to right + let mut visited = Vec::new(); + while iter != right { + let Some(elem) = self.tree.get_elem(iter.leaf) else { + break; + }; + + let (style, anchor_type) = match elem { + RichtextStateChunk::Text { .. } => unreachable!(), + RichtextStateChunk::Style { style, anchor_type } => (style, *anchor_type), + }; + + visited.push((style, anchor_type, iter, entity_index)); + if anchor_type == AnchorType::Start && (style.value.is_null() || style.value.is_false()) + { + // case 1. should be before this anchor + break; + } + + if style.info.prefer_insert_before(anchor_type) { + // case 2. + break; + } + + iter = match self.tree.next_elem(iter) { + Some(x) => x, + None => self.tree.end_cursor().unwrap(), + }; + entity_index += 1; + } + + while let Some((style, anchor_type, top_elem, top_entity_index)) = visited.pop() { + if !style.info.prefer_insert_before(anchor_type) { + // case 3. + break; + } + + iter = top_elem; + entity_index = top_entity_index; + } + + (Some(iter), entity_index) + } + + fn get_entity_index_from_path(&self, right: generic_btree::Cursor) -> usize { + self.get_index_from_cursor(right, PosType::Entity).unwrap() + } + + pub fn get_index_from_cursor( + &self, + cursor: generic_btree::Cursor, + pos_type: PosType, + ) -> Option { + let mut index = 0; + self.tree + .visit_previous_caches(cursor, |cache| match cache { + generic_btree::PreviousCache::NodeCache(c) => { + index += c.get_len(pos_type) as usize; + } + generic_btree::PreviousCache::PrevSiblingElem(c) => { + index += c.len_with(pos_type); + } + generic_btree::PreviousCache::ThisElemAndOffset { elem, offset } => { + if offset != 0 { + index += entity_offset_to_pos_type_offset(pos_type, elem, offset) + } + } + }); + + Some(index) + } + + pub(crate) fn get_text_entity_ranges( + &self, + pos: usize, + len: usize, + pos_type: PosType, + ) -> LoroResult> { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return Ok(Vec::new()); + } + + if len == 0 { + return Ok(Vec::new()); + } + + if pos + len > self.len(pos_type) { + return Ok(Vec::new()); + } + + let mut ans: Vec = Vec::new(); + let (start, end) = match pos_type { + PosType::Bytes => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree.query::(&(pos + len)).unwrap().cursor, + ), + PosType::Unicode => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree + .query::(&(pos + len)) + .unwrap() + .cursor, + ), + PosType::Utf16 => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree.query::(&(pos + len)).unwrap().cursor, + ), + PosType::Entity => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree.query::(&(pos + len)).unwrap().cursor, + ), + PosType::Event => ( + self.tree.query::(&pos).unwrap().cursor, + self.tree + .query::(&(pos + len)) + .unwrap() + .cursor, + ), + }; + + // TODO: assert end cursor is valid + let mut entity_index = self.get_entity_index_from_path(start); + for span in self.tree.iter_range(start..end) { + let start = span.start.unwrap_or(0); + let end = span.end.unwrap_or(span.elem.rle_len()); + if end == 0 { + break; + } + + let len = end - start; + match span.elem { + RichtextStateChunk::Text(s) => { + let event_len = s.entity_range_to_event_range(start..end).len(); + let id = s.id().inc(start as i32); + match ans.last_mut() { + Some(last) + if last.entity_end == entity_index + && last.id_start.inc(last.event_len as i32) == id => + { + last.entity_end += len; + last.event_len += event_len; + } + _ => { + ans.push(EntityRangeInfo { + id_start: id, + entity_start: entity_index, + entity_end: entity_index + len, + event_len, + }); + } + } + entity_index += len; + } + RichtextStateChunk::Style { .. } => { + entity_index += 1; + } + } + } + + Ok(ans) + }; + self.check_cache(); + result + } + + pub(crate) fn get_text_slice_by_event_index( + &self, + pos: usize, + len: usize, + ) -> LoroResult { + self.check_cache(); + let result = { + if self.tree.is_empty() { + return Ok(String::new()); + } + + if len == 0 { + return Ok(String::new()); + } + + if pos + len > self.len_event() { + return Err(LoroError::OutOfBound { + pos: pos + len, + len: self.len_event(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let mut ans = String::new(); + let (start, end) = ( + self.tree.query::(&pos).unwrap().cursor, + self.tree + .query::(&(pos + len)) + .unwrap() + .cursor, + ); + + for span in self.tree.iter_range(start..end) { + let start = span.start.unwrap_or(0); + let end = span.end.unwrap_or(span.elem.rle_len()); + if end == 0 { + break; + } + + if let RichtextStateChunk::Text(s) = span.elem { + match unicode_slice(s.as_str(), start, end) { + Ok(x) => ans.push_str(x), + Err(()) => { + return Err(LoroError::UTF16InUnicodeCodePoint { pos: pos + len }) + } + } + } + } + + Ok(ans) + }; + self.check_cache(); + result + } + + pub(crate) fn slice_delta( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult> { + if end_index < start_index { + return Err(LoroError::EndIndexLessThanStartIndex { + start: start_index, + end: end_index, + }); + } + + if end_index == start_index { + return Ok(Vec::new()); + } + + self.check_cache(); + let result = { + // 1. Convert start/end pos to cursors + let (start_query, end_query) = match pos_type { + PosType::Event => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Bytes => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Unicode => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Utf16 => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + PosType::Entity => ( + self.tree.query::(&start_index), + self.tree.query::(&end_index), + ), + }; + + let start_cursor = start_query + .ok_or(LoroError::OutOfBound { + pos: start_index, + len: self.len(pos_type), + info: "".into(), + })? + .cursor; + let end_cursor = end_query + .ok_or(LoroError::OutOfBound { + pos: end_index, + len: self.len(pos_type), + info: "".into(), + })? + .cursor; + + // 2. Get start entity index + let mut current_entity_index = self + .get_index_from_cursor(start_cursor, PosType::Entity) + .unwrap(); + // We need end entity index for style iterator + let end_entity_index = self + .get_index_from_cursor(end_cursor, PosType::Entity) + .unwrap(); + + // 3. Setup style iterator + let mut style_range_iter: Box, &Styles)>> = + match &self.style_ranges { + Some(s) => { + let mut idx = current_entity_index; + Box::new(s.iter_range(current_entity_index..end_entity_index).map( + move |elem_slice| { + let len = elem_slice.end.unwrap_or(elem_slice.elem.len) + - elem_slice.start.unwrap_or(0); + let range = idx..idx + len; + idx += len; + (range, &elem_slice.elem.styles) + }, + )) + } + None => Box::new(Some((0..usize::MAX / 2, &*EMPTY_STYLES)).into_iter()), + }; + + let mut cur_style_range = style_range_iter.next(); + let mut cur_styles: Option = + cur_style_range.as_ref().map(|x| x.1.clone().into()); + + let mut ans: Vec<(String, StyleMeta)> = Vec::new(); + + // 4. Iterate tree range + for span in self.tree.iter_range(start_cursor..end_cursor) { + match &span.elem { + RichtextStateChunk::Text(t) => { + let chunk_len = + span.end.unwrap_or(span.elem.rle_len()) - span.start.unwrap_or(0); // length in rle_len (unicode_len) + let mut processed_len = 0; + + while processed_len < chunk_len { + while let Some((inner_cur_range, _)) = cur_style_range.as_ref() { + if current_entity_index >= inner_cur_range.end { + cur_style_range = style_range_iter.next(); + cur_styles = + cur_style_range.as_ref().map(|x| x.1.clone().into()); + } else { + break; + } + } + + if cur_style_range.is_none() { + cur_styles = Some(StyleMeta::default()); + cur_style_range = + Some((current_entity_index..usize::MAX, &*EMPTY_STYLES)); + } + + let (inner_cur_range, _) = cur_style_range.as_ref().unwrap(); + + let remaining_text = chunk_len - processed_len; + let remaining_style = inner_cur_range.end - current_entity_index; + let take_len = remaining_text.min(remaining_style); + + let slice_start = span.start.unwrap_or(0) + processed_len; + let slice_end = slice_start + take_len; + + let text_content = unicode_slice(t.as_str(), slice_start, slice_end) + .map_err(|_| LoroError::OutOfBound { + pos: slice_end, + len: t.unicode_len() as usize, + info: "Slice delta out of bound".into(), + })?; + + let styles = cur_styles.as_ref().unwrap(); + if let Some(last) = ans.last_mut() { + if &last.1 == styles { + last.0.push_str(text_content); + processed_len += take_len; + current_entity_index += take_len; + continue; + } + } + + ans.push((text_content.to_string(), styles.clone())); + + processed_len += take_len; + current_entity_index += take_len; + } + } + RichtextStateChunk::Style { .. } => { + current_entity_index += 1; + } + } + } + + Ok(ans) + }; + self.check_cache(); + result + } + + // PERF: can be splitted into two methods. One is without cursor_to_event_index + // PERF: can be speed up a lot by detecting whether the range is in a single leaf first + /// This is used to accept changes from DiffCalculator + #[instrument(skip(self, f))] + pub(crate) fn drain_by_entity_index( + &mut self, + pos: usize, + len: usize, + mut f: Option<&mut dyn FnMut(RichtextStateChunk)>, + ) -> DrainInfo { + let result = { + assert!( + pos + len <= self.len_entity(), + "pos: {}, len: {}, self.len(): {}", + pos, + len, + &self.len_entity(), + ); + + self.clear_cache(); + // PERF: may use cache to speed up + let range = pos..pos + len; + let (start, start_f) = self + .tree + .query_with_finder_return::(&range.start); + let start_cursor = start.unwrap().cursor(); + let elem = self.tree.get_elem(start_cursor.leaf).unwrap(); + + /// This struct remove the corresponding style ranges if the start style anchor is removed + struct StyleRangeUpdater<'a> { + style_ranges: Option<&'a mut StyleRangeMap>, + current_index: usize, + start: usize, + end: usize, + } + + impl<'a> StyleRangeUpdater<'a> { + fn update(&mut self, elem: &RichtextStateChunk) { + match &elem { + RichtextStateChunk::Text(t) => { + self.current_index += t.unicode_len() as usize; + } + RichtextStateChunk::Style { style, anchor_type } => { + if matches!(anchor_type, AnchorType::End) { + self.end = self.end.max(self.current_index); + if let Some(s) = self.style_ranges.as_mut() { + let start = + s.remove_style_scanning_backward(style, self.current_index); + self.start = self.start.min(start); + } + } + + self.current_index += 1; + } + } + } + + fn new( + style_ranges: Option<&'a mut Box>, + start_index: usize, + ) -> Self { + Self { + style_ranges: style_ranges.map(|x| &mut **x), + current_index: start_index, + end: 0, + start: usize::MAX, + } + } + + fn get_affected_range(&self, pos: usize) -> Option> { + if self.start == usize::MAX { + None + } else { + let start = self.start.min(pos); + let end = self.end.min(pos); + if start == end { + None + } else { + Some(start..end) + } + } + } + } + + if elem.rle_len() >= start_cursor.offset + len { + // drop in place + let mut event_len = 0; + let mut updater = StyleRangeUpdater::new(self.style_ranges.as_mut(), pos); + self.tree.update_leaf(start_cursor.leaf, |elem| { + updater.update(&*elem); + match elem { + RichtextStateChunk::Text(text) => { + if let Some(f) = f { + let span = + text.slice(start_cursor.offset..start_cursor.offset + len); + f(RichtextStateChunk::Text(span)); + } + let (next, event_len_) = + text.delete_by_entity_index(start_cursor.offset, len); + event_len = event_len_; + (true, next.map(RichtextStateChunk::Text), None) + } + RichtextStateChunk::Style { .. } => { + if let Some(f) = f { + let v = std::mem::replace( + elem, + RichtextStateChunk::Text(TextChunk::new_empty()), + ); + f(v); + } else { + *elem = RichtextStateChunk::Text(TextChunk::new_empty()); + } + (true, None, None) + } + } + }); + + let affected_range = updater.get_affected_range(pos); + if let Some(s) = self.style_ranges.as_mut() { + s.delete(pos..pos + len); + } + + DrainInfo { + start_event_index: start_f.event_index, + end_event_index: (start_f.event_index + event_len), + affected_style_range: affected_range.map(|entity_range| { + ( + entity_range.clone(), + self.entity_index_to_event_index(entity_range.start) + ..self.entity_index_to_event_index(entity_range.end), + ) + }), + } + } else { + let (end, end_f) = self + .tree + .query_with_finder_return::(&range.end); + let mut updater = StyleRangeUpdater::new(self.style_ranges.as_mut(), pos); + for iter in generic_btree::iter::Drain::new(&mut self.tree, start, end) { + updater.update(&iter); + if let Some(f) = f.as_mut() { + f(iter) + } + } + + let affected_range = updater.get_affected_range(pos); + if let Some(s) = self.style_ranges.as_mut() { + s.delete(pos..pos + len); + } + + DrainInfo { + start_event_index: start_f.event_index, + end_event_index: end_f.event_index, + affected_style_range: affected_range.map(|entity_range| { + ( + entity_range.clone(), + self.entity_index_to_event_index(entity_range.start) + ..self.entity_index_to_event_index(entity_range.end), + ) + }), + } + } + }; + result + } + + pub fn entity_index_to_event_index(&self, index: usize) -> usize { + if index == 0 { + // the tree maybe empty + return 0; + } + let cursor = self.tree.query::(&index).unwrap(); + self.cursor_to_event_index(cursor.cursor) + } + + pub fn index_to_event_index(&self, index: usize, pos_type: PosType) -> usize { + if self.tree.is_empty() { + return 0; + } + + let cursor = match pos_type { + PosType::Entity => self.tree.query::(&index).unwrap(), + PosType::Utf16 => self.tree.query::(&index).unwrap(), + PosType::Bytes => self.tree.query::(&index).unwrap(), + PosType::Event => return index, + PosType::Unicode => self.tree.query::(&index).unwrap(), + }; + + self.cursor_to_event_index(cursor.cursor) + } + + pub fn event_index_to_unicode_index(&self, index: usize) -> usize { + if !cfg!(feature = "wasm") { + return index; + } + + let Some(cursor) = self.tree.query::(&index) else { + return 0; + }; + + self.cursor_to_unicode_index(cursor.cursor) + } + + #[allow(unused)] + pub(crate) fn check(&self) { + if !cfg!(any(debug_assertions, test)) { + return; + } + + self.tree.check(); + self.check_consistency_between_content_and_style_ranges(); + self.check_style_anchors_appear_in_pairs(); + } + + pub(crate) fn mark_with_entity_index(&mut self, range: Range, style: Arc) { + self.check_cache(); + self.clear_cache(); + if self.tree.is_empty() { + panic!("Cannot mark an empty tree"); + } + + self.insert_elem_at_entity_index( + range.end, + RichtextStateChunk::from_style(style.clone(), AnchorType::End), + ); + self.insert_elem_at_entity_index( + range.start, + RichtextStateChunk::from_style(style.clone(), AnchorType::Start), + ); + // end_entity_index + 2, because + // 1. We inserted a start anchor before end_entity_index, so we need to +1 + // 2. We need to include the end anchor in the range, so we need to +1 + self.ensure_style_ranges_mut() + .annotate(range.start..range.end + 2, style, None); + self.check_cache(); + } + + pub fn iter(&self) -> impl Iterator + '_ { + let mut entity_index = 0; + let mut style_range_iter: Box, &Styles)>> = + match &self.style_ranges { + Some(s) => Box::new(s.iter()), + None => Box::new(Some((0..usize::MAX / 2, &*EMPTY_STYLES)).into_iter()), + }; + let mut cur_style_range = style_range_iter.next(); + let mut cur_styles: Option = + cur_style_range.as_ref().map(|x| x.1.clone().into()); + + self.tree.iter().filter_map(move |x| match x { + RichtextStateChunk::Text(s) => { + let mut styles = Default::default(); + while let Some((inner_cur_range, _)) = cur_style_range.as_ref() { + if entity_index < inner_cur_range.start { + break; + } + + if entity_index < inner_cur_range.end { + styles = cur_styles.as_ref().unwrap().clone(); + break; + } else { + cur_style_range = style_range_iter.next(); + cur_styles = cur_style_range.as_ref().map(|x| x.1.clone().into()); + } + } + + entity_index += s.rle_len(); + Some(RichtextSpan { + text: s.bytes().clone().into(), + attributes: styles, + }) + } + RichtextStateChunk::Style { .. } => { + entity_index += 1; + None + } + }) + } + + #[inline] + pub fn iter_chunk(&self) -> impl Iterator { + self.tree.iter() + } + + pub fn get_richtext_value(&self) -> LoroValue { + let mut builder = RichtextValueBuilder::default(); + self.for_each_richtext_span(&mut |text, styles| builder.push(text, styles)); + builder.finish() + } + + /// §act-eleven A11.10 (flowstate vendor patch): visit every text span of the + /// built state as `(text, styles)` WITHOUT allocating the intermediate + /// `Vec` delta value. [`Self::get_richtext_value`] is expressed on + /// top of this walker via [`RichtextValueBuilder`], so every existing + /// value-equivalence net (T1 verify, corpus, fuzz oracles) transitively pins + /// the walk. Spans arrive UNMERGED (one per text chunk); callers that need + /// delta segmentation merge on the same visible-attribute equality the + /// builder uses ([`StyleMeta::visible_eq`]). + pub fn for_each_richtext_span(&self, f: &mut dyn FnMut(&str, &StyleMeta)) { + self.check_cache(); + for span in self.iter() { + f(span.text.as_str(), &span.attributes); + } + self.check_cache(); + } + + /// §perf-heaven T1 (flowstate vendor patch): produce the richtext value + /// DIRECTLY from a decoded-snapshot loader's `elements` + resolved + /// `style_ranges`, WITHOUT building the text B-tree (`from_chunks`) — the + /// dominant cost of the cold projection body decode (`get_richtext_value` + /// on an attached text forces the whole `LazyLoad::Src -> Dst` conversion). + /// + /// The style map is built with the EXACT same `annotate_style_range` calls + /// `into_state` uses, so style resolution is bit-identical; only the text + /// tree is skipped. Text chunks never straddle a style boundary (style + /// anchors are their own chunks), so the no-split walk here matches + /// [`Self::iter`] chunk-for-chunk. The output is bit-identical to + /// [`Self::get_richtext_value`] on the fully built state — an equivalence a + /// debug/verify pass asserts on every call (see the caller). + pub(crate) fn richtext_value_from_src( + elements: &[RichtextStateChunk], + style_ranges: &[(Arc, std::ops::Range)], + ) -> LoroValue { + let mut builder = RichtextValueBuilder::default(); + Self::for_each_span_from_src(elements, style_ranges, &mut |text, styles| { + builder.push(text, styles) + }); + builder.finish() + } + + /// §act-eleven A11.10 (flowstate vendor patch): the streaming twin of + /// [`Self::richtext_value_from_src`] — same loader walk, but each text chunk + /// is handed to `f` instead of being folded into a `LoroValue` list. + /// `richtext_value_from_src` is expressed on top of THIS function, so the T1 + /// fast-path verify (fast == built-state value) transitively validates the + /// walk order and style resolution here too. + pub(crate) fn for_each_span_from_src( + elements: &[RichtextStateChunk], + style_ranges: &[(Arc, std::ops::Range)], + f: &mut dyn FnMut(&str, &StyleMeta), + ) { + // Style map ONLY (no text tree): reuses the exact merge semantics. + let mut style_state = Self::from_chunks(std::iter::empty::()); + for (style, range) in style_ranges { + style_state.annotate_style_range(range.clone(), style.clone()); + } + + let mut entity_index = 0usize; + let mut style_range_iter: Box, &Styles)>> = + match &style_state.style_ranges { + Some(s) => Box::new(s.iter()), + None => Box::new(Some((0..usize::MAX / 2, &*EMPTY_STYLES)).into_iter()), + }; + let mut cur_style_range = style_range_iter.next(); + let mut cur_styles: Option = cur_style_range.as_ref().map(|x| x.1.clone().into()); + + for chunk in elements { + match chunk { + RichtextStateChunk::Text(s) => { + let mut styles: StyleMeta = Default::default(); + while let Some((inner_cur_range, _)) = cur_style_range.as_ref() { + if entity_index < inner_cur_range.start { + break; + } + if entity_index < inner_cur_range.end { + styles = cur_styles.as_ref().unwrap().clone(); + break; + } else { + cur_style_range = style_range_iter.next(); + cur_styles = cur_style_range.as_ref().map(|x| x.1.clone().into()); + } + } + entity_index += s.rle_len(); + f(s.as_str(), &styles); + } + RichtextStateChunk::Style { .. } => { + entity_index += 1; + } + } + } + } + + #[allow(unused)] + pub fn to_vec(&self) -> Vec { + self.iter().collect() + } + + #[cfg(test)] + #[allow(unused)] + pub(crate) fn debug(&self) { + dbg!(&self.tree); + dbg!(&self.style_ranges); + } + + #[inline(always)] + pub fn len_unicode(&self) -> usize { + self.tree.root_cache().unicode_len as usize + } + + #[inline(always)] + pub fn len_utf16(&self) -> usize { + self.tree.root_cache().utf16_len as usize + } + + #[inline(always)] + pub fn len_utf8(&self) -> usize { + self.tree.root_cache().bytes as usize + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.tree.root_cache().entity_len == 0 + } + + #[inline(always)] + pub fn len_entity(&self) -> usize { + self.tree.root_cache().entity_len as usize + } + + #[inline(always)] + pub(crate) fn content_node_len(&self) -> usize { + self.tree.node_len() + } + + pub fn diagnose(&self) { + println!( + "rope_nodes: {}, style_nodes: {}, text_len: {}", + self.tree.node_len(), + self.style_ranges + .as_ref() + .map(|x| x.tree.node_len()) + .unwrap_or(0), + self.tree.root_cache().bytes + ); + } + + /// Check if the content and style ranges are consistent. + /// + /// Panic if inconsistent. + pub(crate) fn check_consistency_between_content_and_style_ranges(&self) { + if !cfg!(debug_assertions) { + return; + } + + let mut entity_index_to_style_anchor: FxHashMap = + FxHashMap::default(); + let mut index = 0; + for c in self.iter_chunk() { + if matches!(c, RichtextStateChunk::Style { .. }) { + entity_index_to_style_anchor.insert(index, c); + } + + index += c.length() + } + + if let Some(s) = &self.style_ranges { + for IterAnchorItem { + index, + op, + anchor_type: iter_anchor_type, + } in s.iter_anchors() + { + let c = entity_index_to_style_anchor + .remove(&index) + .unwrap_or_else(|| { + panic!( + "Inconsistency found {} {:?} {:?}", + index, iter_anchor_type, &op + ); + }); + + match c { + RichtextStateChunk::Text(_) => { + unreachable!() + } + RichtextStateChunk::Style { style, anchor_type } => { + assert_eq!(style, &op); + assert_eq!(&iter_anchor_type, anchor_type); + } + } + } + } + + assert!( + entity_index_to_style_anchor.is_empty(), + "Inconsistency found. Some anchors are not reflected in style ranges {:#?}", + &entity_index_to_style_anchor + ); + } + + /// Allow StyleAnchors to appear in pairs, so that there won't be unmatched single StyleAnchors. + pub(crate) fn check_style_anchors_appear_in_pairs(&self) { + if !cfg!(any(debug_assertions, test)) { + return; + } + + let mut start_ops: FxHashSet<&Arc> = Default::default(); + for item in self.iter_chunk() { + match item { + RichtextStateChunk::Text(_) => {} + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + AnchorType::Start => { + start_ops.insert(style); + } + AnchorType::End => { + assert!(start_ops.remove(style), "End anchor without start anchor"); + } + }, + } + } + assert!( + start_ops.is_empty(), + "Only has start anchors {:#?}", + &start_ops + ); + } + + /// Iter style ranges in the given range in entity index + pub(crate) fn iter_range( + &self, + range: impl RangeBounds, + ) -> impl Iterator> + '_ { + self.check_cache(); + let result = { + let start = match range.start_bound() { + Bound::Included(x) => *x, + Bound::Excluded(x) => x + 1, + Bound::Unbounded => 0, + }; + let end = match range.end_bound() { + Bound::Included(x) => x + 1, + Bound::Excluded(x) => *x, + Bound::Unbounded => self.len_entity(), + }; + assert!(end > start); + assert!(end <= self.len_entity()); + + let mut style_iter = self + .style_ranges + .as_ref() + .map(|x| x.iter_range(range)) + .into_iter() + .flatten(); + + let start = self.tree.query::(&start).unwrap(); + let end = self.tree.query::(&end).unwrap(); + let mut content_iter = self.tree.iter_range(start.cursor..end.cursor); + let mut style_left_len = usize::MAX; + let mut cur_style = style_iter + .next() + .map(|x| { + style_left_len = x.elem.len - x.start.unwrap_or(0); + &x.elem.styles + }) + .unwrap_or(&*EMPTY_STYLES); + let mut chunk = content_iter.next(); + let mut offset = 0; + let mut chunk_left_len = chunk + .as_ref() + .map(|x| { + let len = x.elem.rle_len(); + offset = x.start.unwrap_or(0); + x.end.map(|v| v.min(len)).unwrap_or(len) - offset + }) + .unwrap_or(0); + std::iter::from_fn(move || { + if chunk_left_len == 0 { + chunk = content_iter.next(); + chunk_left_len = chunk + .as_ref() + .map(|x| { + let len = x.elem.rle_len(); + x.end.map(|v| v.min(len)).unwrap_or(len) + }) + .unwrap_or(0); + offset = 0; + } + + let iter_chunk = chunk.as_ref()?; + + let styles = cur_style; + let iter_len; + let event_range; + if chunk_left_len >= style_left_len { + iter_len = style_left_len; + event_range = iter_chunk + .elem + .entity_range_to_event_range(offset..offset + iter_len); + chunk_left_len -= style_left_len; + offset += style_left_len; + style_left_len = 0; + } else { + iter_len = chunk_left_len; + event_range = iter_chunk + .elem + .entity_range_to_event_range(offset..offset + iter_len); + style_left_len -= chunk_left_len; + chunk_left_len = 0; + } + + if style_left_len == 0 { + cur_style = style_iter + .next() + .map(|x| { + style_left_len = x.elem.len; + &x.elem.styles + }) + .unwrap_or(&*EMPTY_STYLES); + } + + Some(IterRangeItem { + chunk: iter_chunk.elem, + styles, + entity_len: iter_len, + event_len: event_range.len(), + }) + }) + }; + self.check_cache(); + result + } + + pub(crate) fn get_stable_position_at_event_index( + &self, + pos: usize, + kind: PosType, + ) -> Option { + self.check_cache(); + let result = { + let v = &self.get_text_entity_ranges(pos, 1, kind).unwrap(); + let a = v.first()?; + Some(a.id_start) + }; + self.check_cache(); + result + } + + pub(crate) fn len_event(&self) -> usize { + self.check_cache(); + let result = { + if cfg!(feature = "wasm") { + self.len_utf16() + } else { + self.len_unicode() + } + }; + self.check_cache(); + result + } + + pub(crate) fn len(&self, pos_type: PosType) -> usize { + self.check_cache(); + let result = { + match pos_type { + PosType::Unicode => self.len_unicode(), + PosType::Utf16 => self.len_utf16(), + PosType::Entity => self.len_entity(), + PosType::Event => self.len_event(), + PosType::Bytes => self.len_utf8(), + } + }; + self.check_cache(); + result + } +} + +fn entity_offset_to_pos_type_offset( + pos_type: PosType, + elem: &RichtextStateChunk, + offset: usize, +) -> usize { + match pos_type { + PosType::Bytes => match elem { + RichtextStateChunk::Text(t) => unicode_to_utf8_index(t.as_str(), offset).unwrap(), + RichtextStateChunk::Style { .. } => 0, + }, + PosType::Unicode => match elem { + RichtextStateChunk::Text(_) => offset, + RichtextStateChunk::Style { .. } => 0, + }, + PosType::Utf16 => match elem { + RichtextStateChunk::Text(t) => unicode_to_utf16_index(t.as_str(), offset).unwrap(), + RichtextStateChunk::Style { .. } => 0, + }, + PosType::Entity => offset, + PosType::Event => match elem { + RichtextStateChunk::Text(t) => { + if cfg!(feature = "wasm") { + unicode_to_utf16_index(t.as_str(), offset).unwrap() + } else { + offset + } + } + RichtextStateChunk::Style { .. } => 0, + }, + } +} + +fn pos_type_offset_to_entity_offset( + pos_type: PosType, + elem: &RichtextStateChunk, + offset: usize, +) -> Option { + match pos_type { + PosType::Bytes => match elem { + RichtextStateChunk::Text(t) => utf8_to_unicode_index(t.as_str(), offset).ok(), + RichtextStateChunk::Style { .. } => { + if offset > 0 { + None + } else { + Some(0) + } + } + }, + PosType::Unicode => Some(offset), + PosType::Utf16 => match elem { + RichtextStateChunk::Text(t) => utf16_to_unicode_index(t.as_str(), offset).ok(), + RichtextStateChunk::Style { .. } => { + if offset > 0 { + None + } else { + Some(0) + } + } + }, + PosType::Entity => { + if offset > elem.rle_len() { + None + } else { + Some(offset) + } + } + PosType::Event => match elem { + RichtextStateChunk::Text(t) => { + if cfg!(feature = "wasm") { + utf16_to_unicode_index(t.as_str(), offset).ok() + } else if offset < t.unicode_len() as usize { + Some(offset) + } else { + None + } + } + RichtextStateChunk::Style { .. } => { + if offset > 0 { + None + } else { + Some(0) + } + } + }, + } +} + +pub(crate) struct DrainInfo { + pub start_event_index: usize, + pub end_event_index: usize, + // entity range, event range + pub affected_style_range: Option<(Range, Range)>, +} + +pub(crate) struct IterRangeItem<'a> { + pub(crate) chunk: &'a RichtextStateChunk, + pub(crate) styles: &'a Styles, + pub(crate) entity_len: usize, + pub(crate) event_len: usize, +} + +use converter::ContinuousIndexConverter; +mod converter { + use generic_btree::{rle::HasLength, Cursor}; + + use super::{query::EntityQuery, RichtextState}; + + /// Convert entity index into event index. + /// It assumes the entity_index are in ascending order + pub(super) struct ContinuousIndexConverter<'a> { + state: &'a RichtextState, + last_entity_index_cache: Option, + } + + struct ConverterCache { + entity_index: usize, + cursor: Cursor, + event_index: usize, + cursor_elem_len: usize, + } + + impl<'a> ContinuousIndexConverter<'a> { + pub fn new(state: &'a RichtextState) -> Self { + Self { + state, + last_entity_index_cache: None, + } + } + + pub fn convert_entity_index_to_event_index(&mut self, entity_index: usize) -> usize { + self.state.check_cache(); + let result = { + if let Some(last) = self.last_entity_index_cache.as_ref() { + if last.entity_index == entity_index { + return last.event_index; + } + + assert!(entity_index > last.entity_index); + if last.cursor.offset + entity_index - last.entity_index < last.cursor_elem_len + { + // in the same cursor + return self.state.cursor_to_event_index(Cursor { + leaf: last.cursor.leaf, + offset: last.cursor.offset + entity_index - last.entity_index, + }); + } + } + + let cursor = self + .state + .tree + .query::(&entity_index) + .unwrap() + .cursor; + let ans = self.state.cursor_to_event_index(cursor); + let len = self.state.tree.get_elem(cursor.leaf).unwrap().rle_len(); + self.last_entity_index_cache = Some(ConverterCache { + entity_index, + cursor, + event_index: ans, + cursor_elem_len: len, + }); + ans + }; + self.state.check_cache(); + result + } + } +} + +#[cfg(test)] +mod test { + use append_only_bytes::AppendOnlyBytes; + use serde_json::json; + + use crate::{container::richtext::TextStyleInfoFlag, ToJson}; + + use super::*; + + #[derive(Debug, Default, Clone)] + struct SimpleWrapper { + state: RichtextState, + bytes: AppendOnlyBytes, + } + + impl SimpleWrapper { + fn insert(&mut self, pos: usize, text: &str) { + let start = self.bytes.len(); + self.bytes.push_str(text); + { + let state = &mut self.state; + let text = self.bytes.slice(start..); + let (entity_index, _) = state + .get_entity_index_for_text_insert(pos, PosType::Unicode) + .unwrap(); + state.insert_at_entity_index(entity_index, text, IdFull::new(0, 0, 0)); + }; + } + + fn delete(&mut self, pos: usize, len: usize) { + let ranges = self + .state + .get_text_entity_ranges(pos, len, PosType::Unicode) + .unwrap(); + for range in ranges.into_iter().rev() { + self.state.drain_by_entity_index( + range.entity_start, + range.entity_end - range.entity_start, + None, + ); + } + } + + fn mark(&mut self, range: Range, style: Arc) { + let (start, _) = self + .state + .get_entity_index_for_text_insert(range.start, PosType::Unicode) + .unwrap(); + let (end, _) = self + .state + .get_entity_index_for_text_insert(range.end, PosType::Unicode) + .unwrap(); + self.state.mark_with_entity_index(start..end, style); + } + } + + fn bold(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + "bold", + true.into(), + TextStyleInfoFlag::BOLD, + )) + } + + fn comment(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + &format!("comment:{}", n), + "comment".into(), + TextStyleInfoFlag::COMMENT, + )) + } + + fn unbold(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + "bold", + LoroValue::Null, + TextStyleInfoFlag::BOLD.to_delete(), + )) + } + + fn link(n: isize) -> Arc { + Arc::new(StyleOp::new_for_test( + n, + "link", + true.into(), + TextStyleInfoFlag::LINK, + )) + } + + #[test] + fn test() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "bold": true + } + }, + { + "insert": " World!" + } + ]) + ); + wrapper.mark(2..7, link(1)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "He", + "attributes": { + "bold": true + } + }, + { + "insert": "llo", + "attributes": { + "bold": true, + "link": true + } + }, + { + "insert": " W", + "attributes": { + "link": true + } + }, + { + "insert": "orld!" + } + + ]) + ); + } + + #[test] + fn delete_text() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + assert_eq!(wrapper.state.len_unicode(), 12); + assert_eq!(wrapper.state.len_entity(), 12); + wrapper.delete(0, 5); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": " World!" + } + ]) + ); + + assert_eq!(wrapper.state.len_unicode(), 7); + assert_eq!(wrapper.state.len_entity(), 7); + wrapper.delete(1, 1); + + assert_eq!(wrapper.state.len_unicode(), 6); + assert_eq!(wrapper.state.len_entity(), 6); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": " orld!" + } + ]) + ); + + wrapper.delete(5, 1); + assert_eq!(wrapper.state.len_unicode(), 5); + assert_eq!(wrapper.state.len_entity(), 5); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": " orld" + } + ]) + ); + + wrapper.delete(0, 5); + assert_eq!(wrapper.state.len_unicode(), 0); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([]) + ); + } + + #[test] + #[ignore] + fn insert_cache_hit() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "H"); + wrapper.insert(1, "H"); + dbg!(&wrapper); + wrapper.insert(2, "e"); + dbg!(&wrapper); + wrapper.insert(3, "k"); + wrapper.state.diagnose(); + } + + #[test] + fn bold_should_expand() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + wrapper.insert(5, " Test"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello Test", + "attributes": { + "bold": true + } + }, + { + "insert": " World!" + } + ]) + ); + } + + #[test] + fn link_should_not_expand() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, link(0)); + wrapper.insert(5, " Test"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "link": true + } + }, + { + "insert": " Test World!" + }, + ]) + ); + } + + #[test] + fn continuous_text_insert_should_be_merged() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.insert(5, " World!"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello World!" + }, + ]) + ); + } + + #[test] + fn continuous_text_insert_should_be_merged_and_have_bold() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.mark(0..5, bold(0)); + wrapper.insert(5, " World!"); + dbg!(&wrapper.state); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello World!", + "attributes": { + "bold": true + } + }, + ]) + ); + } + + #[test] + fn continuous_text_insert_should_not_be_merged_when_prev_is_link() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.mark(0..5, link(0)); + wrapper.insert(5, " World!"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "link": true + } + }, + { + + "insert": " World!", + } + ]) + ); + } + + #[test] + fn delete_bold() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..12, bold(0)); + wrapper.mark(5..12, unbold(1)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "bold": true + } + }, + { + "insert": " World!" + } + ]) + ); + wrapper.insert(5, "A"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "HelloA", + "attributes": { + "bold": true + } + }, + { + "insert": " World!", + } + ]) + ); + + wrapper.insert(0, "A"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "A", + }, + { + "insert": "HelloA", + "attributes": { + "bold": true + } + }, + { + "insert": " World!", + } + ]) + ); + } + + #[test] + fn bold_and_link_at_the_same_place() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello"); + wrapper.mark(0..5, link(0)); + wrapper.mark(0..5, bold(1)); + wrapper.insert(5, "A"); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "Hello", + "attributes": { + "bold": true, + "link": true + } + }, + { + "insert": "A", + "attributes": { + "bold": true, + } + }, + ]) + ); + } + + #[test] + fn test_comments() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, comment(0)); + wrapper.mark(1..6, comment(1)); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([ + { + "insert": "H", + "attributes": { + "comment:0": "comment", + }, + }, + { + "insert": "ello", + "attributes": { + "comment:0": "comment", + "comment:1": "comment", + }, + }, + + { + "insert": " ", + "attributes": { + "comment:1": "comment", + }, + }, + { + "insert": "World!", + } + ]) + ); + } + + #[test] + fn remove_style_anchors_should_also_delete_style() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + let mut count = 0; + wrapper.state.drain_by_entity_index( + 0, + 7, + Some(&mut |span| { + if matches!(span, RichtextStateChunk::Style { .. }) { + count += 1; + } + }), + ); + + assert_eq!(count, 2); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([{ + "insert": " World!" + }]) + ); + } + + #[test] + fn remove_start_anchor_should_remove_style() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(0..5, bold(0)); + wrapper.state.drain_by_entity_index(6, 1, None); + wrapper.state.drain_by_entity_index(0, 1, None); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([{ + "insert": "Hello World!" + }]) + ); + } + + #[test] + fn remove_start_anchor_in_the_middle_should_remove_style() { + let mut wrapper = SimpleWrapper::default(); + wrapper.insert(0, "Hello World!"); + wrapper.mark(2..5, bold(0)); + wrapper.state.drain_by_entity_index(6, 1, None); + wrapper.state.drain_by_entity_index(1, 2, None); + assert_eq!( + wrapper.state.get_richtext_value().to_json_value(), + json!([{ + "insert": "Hllo World!" + }]) + ); + } +} diff --git a/vendor/loro-internal/src/container/richtext/str_slice.rs b/vendor/loro-internal/src/container/richtext/str_slice.rs new file mode 100644 index 00000000..fc816cdf --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/str_slice.rs @@ -0,0 +1,77 @@ +use append_only_bytes::{AppendOnlyBytes, BytesSlice}; + +/// It's just a wrapper around [BytesSlice] that makes sure +/// the content of the bytes is a valid utf-8 string +pub(crate) struct StrSlice(BytesSlice); + +impl StrSlice { + #[allow(unused)] + pub fn new(bytes: BytesSlice) -> Option { + let _str = std::str::from_utf8(&bytes).ok()?; + Some(Self(bytes)) + } + + pub fn new_from_str(str: &str) -> Self { + let mut a = AppendOnlyBytes::new(); + a.push_str(str); + Self(a.slice(..)) + } + + pub fn bytes(&self) -> &BytesSlice { + &self.0 + } + + pub fn as_str(&self) -> &str { + // SAFETY: We ensure that the content is always valid utf8 + unsafe { std::str::from_utf8_unchecked(&self.0) } + } + + /// §flowstate vendor patch (oom-leads #2): split at `pos` unicode chars, + /// or `None` when the slice holds FEWER than `pos` chars — one walk of at + /// most `pos + 1` chars. Replaces `decode_snapshot_fast`'s per-span + /// `chars().count()` bounds check over the ENTIRE remaining document + /// string, which made snapshot decode O(spans × chars) — 84% of a + /// 2.6M-char document's cold open. + pub fn try_split_at_unicode_pos(&self, pos: usize) -> Option<(Self, Self)> { + let s = self.as_str(); + let mut split_at = None; + let mut chars_seen = 0usize; + for (u, (i, _)) in s.char_indices().enumerate() { + if u == pos { + split_at = Some(i); + break; + } + chars_seen = u + 1; + } + let split_at = match split_at { + Some(byte_ix) => byte_ix, + None => { + if chars_seen < pos { + return None; + } + self.0.len() + } + }; + Some(( + Self(self.0.slice_clone(..split_at)), + Self(self.0.slice_clone(split_at..)), + )) + } + + #[allow(dead_code, reason = "upstream API kept verbatim; flowstate callers use try_split_at_unicode_pos")] + pub fn split_at_unicode_pos(&self, pos: usize) -> (Self, Self) { + let s = self.as_str(); + let mut split_at = self.0.len(); + for (u, (i, _)) in s.char_indices().enumerate() { + if u == pos { + split_at = i; + break; + } + } + + ( + Self(self.0.slice_clone(..split_at)), + Self(self.0.slice_clone(split_at..)), + ) + } +} diff --git a/vendor/loro-internal/src/container/richtext/style_range_map.rs b/vendor/loro-internal/src/container/richtext/style_range_map.rs new file mode 100644 index 00000000..e7b7f6a9 --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/style_range_map.rs @@ -0,0 +1,1027 @@ +//! This map a Range to a set of style +//! + +use std::{ + ops::{ControlFlow, Deref, DerefMut, Range, RangeBounds}, + sync::Arc, +}; + +use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, ElemSlice, LengthFinder, UseLengthFinder, +}; +use rustc_hash::FxHashMap; + +use once_cell::sync::Lazy; + +use crate::delta::StyleMeta; + +use super::{AnchorType, StyleKey, StyleOp}; + +/// This struct keep the mapping of ranges to numbers +/// +/// It's initialized with usize::MAX/2 length. +#[derive(Debug, Clone)] +pub(super) struct StyleRangeMap { + pub(super) tree: BTree, + has_style: bool, + /// §flowstate stylemap patch: newest `(lamport, peer)` op ever inserted + /// into this map. An annotate whose op is newer than EVERYTHING ever + /// inserted cannot be a duplicate anywhere — the whole covered-elem walk + /// skips per-elem membership checks on one O(1) compare. (The duplicate + /// case is real: styled-RETAIN diffs re-assert ops already present.) + max_op_ever: Option>, +} + +#[derive(Debug, Clone)] +pub(super) struct RangeNumMapTrait; + +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(crate) struct Styles { + pub(crate) styles: FxHashMap, +} + +impl Styles { + pub(crate) fn has_key_value(&self, key: &str, value: &loro_common::LoroValue) -> bool { + match self.get(&StyleKey::Key(key.into())) { + Some(v) => match v.get() { + Some(v) => &v.value == value, + _ => false, + }, + _ => false, + } + } + + /// Infer the anchors between the neighbor styles. + /// Returns the last anchor of the left style and the first anchor of the right style. + fn infer_anchors(&self, next: &Self) -> (Option>, Option>) { + let mut left_anchor = None; + let mut right_anchor = None; + for (key, set) in self.styles.iter() { + let right_value = next.styles.get(key); + for diff in set + .iter_ascending() + .filter(|x| !right_value.is_some_and(|right| right.contains(x))) + { + assert!(left_anchor.is_none(), "left anchor should be unique"); + left_anchor = Some(diff.clone()); + } + } + + for (key, set) in next.styles.iter() { + let left_value = self.styles.get(key); + for diff in set + .iter_ascending() + .filter(|x| !left_value.is_some_and(|left| left.contains(x))) + { + assert!(right_anchor.is_none(), "right anchor should be unique"); + right_anchor = Some(diff.clone()); + } + } + + (left_anchor, right_anchor) + } +} + +impl Deref for Styles { + type Target = FxHashMap; + + fn deref(&self) -> &Self::Target { + &self.styles + } +} + +impl DerefMut for Styles { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.styles + } +} + +pub(super) static EMPTY_STYLES: Lazy = Lazy::new(Default::default); + +#[derive(Debug, Clone)] +pub(crate) struct Elem { + pub(crate) styles: Styles, + pub(crate) len: usize, +} + +/// §flowstate stylemap patch (A11.5 follow-on pass): a LAYERED op set — +/// a persistent shared `base` (`im::OrdSet`, O(1) clone, structural sharing) +/// plus a tiny owned `overlay` absorbing writes. The two failure modes this +/// reconciles, both measured on the styled-divergence repro: +/// * plain `BTreeSet` deep-copied the whole set on every boundary insert and +/// split (the edit-side quadratic constant, 5.2s); +/// * pure sharing (`Arc`, then `im::OrdSet`) made `annotate`'s +/// per-covered-elem insert pay COW/path-copy costs — import degraded 85×/12× +/// (an annotate legitimately walks EVERY covered elem; membership is +/// load-bearing per elem, so that walk only gets cheap per-visit work, and +/// `overlay.push` is O(1) where persistent insert was not). +/// The set semantics are exactly `base ∪ overlay` with `overlay ∩ base = ∅` +/// (enforced on insert); the overlay folds into the base past a small bound so +/// clones stay O(1)+ε. Pruning is NOT sound (`infer_anchors` set-differences +/// and `remove_style_scanning_backward`'s early-break walk need exact +/// membership), so this changes representation only, never the logical set. +#[derive(Clone, Debug, Default)] +pub(crate) struct StyleValue { + // we need a set here because we need to calculate the intersection of styles when + // users insert new text between two style sets + /// `None` until the overlay first overflows: real-world sets (a few + /// overlapping ops) live ENTIRELY in the inline overlay and never touch + /// `im` — an empty `im::OrdSet` allocates a ~1KB root chunk, which showed + /// up as +44% alloc on the whole-paragraph mark-application path. + base: Option, + overlay: smallvec::SmallVec<[Arc; 8]>, + /// Cached `base.get_max()` — `im`'s spine walk is too hot for the + /// per-covered-elem dedup gate in [`Self::insert`]. Invariant: + /// `base_max == base.get_max().cloned()`. + base_max: Option>, +} + +/// Shared empty set for the `base: None` read paths (never mutated). +static EMPTY_OP_SET: Lazy = Lazy::new(StyleOpSet::new); + +type StyleOpSet = im::OrdSet>; + +/// Overlay entries beyond this fold into the shared base (amortized O(log)). +const OVERLAY_BOUND: usize = 8; + +impl PartialEq for StyleValue { + fn eq(&self, other: &Self) -> bool { + if self.len() != other.len() { + return false; + } + // Pointer fast path: same-provenance shares compare O(overlay), which + // also lets `Elem::can_merge` re-merge split fragments cheaply. + if self.base_ref().ptr_eq(other.base_ref()) { + return match (self.overlay.len(), other.overlay.len()) { + (0, 0) => true, + _ => { + self.overlay.iter().all(|op| other.overlay.contains(op)) + && other.overlay.iter().all(|op| self.overlay.contains(op)) + } + }; + } + self.iter_ascending() + .zip(other.iter_ascending()) + .all(|(a, b)| a == b) + } +} + +impl Eq for StyleValue {} + +impl StyleValue { + pub fn insert(&mut self, value: Arc) { + if self.overlay.contains(&value) { + return; + } + // Dedup against the base only when the op COULD be there: anything + // newer than the base's cached max cannot be a duplicate. + if self.base_max.as_ref().is_some_and(|max| value <= *max) + && self.base_ref().contains(&value) + { + return; + } + self.insert_unchecked(value); + } + + /// Insert an op PROVEN absent from the logical set (the caller holds a + /// freshness proof — see `StyleRangeMap::annotate`'s map-level gate). + fn insert_unchecked(&mut self, value: Arc) { + self.overlay.push(value); + if self.overlay.len() > OVERLAY_BOUND { + self.normalize(); + } + } + + /// Fold the overlay into the base. After this the overlay is empty and + /// the base owns the full logical set. + fn normalize(&mut self) { + let base = self.base.get_or_insert_with(StyleOpSet::new); + for op in self.overlay.drain(..) { + if self.base_max.as_ref().is_none_or(|max| op > *max) { + self.base_max = Some(op.clone()); + } + base.insert(op); + } + } + + /// The base set for read paths (`None` reads as the shared empty set). + fn base_ref(&self) -> &StyleOpSet { + self.base.as_ref().unwrap_or(&EMPTY_OP_SET) + } + + fn len(&self) -> usize { + self.base_ref().len() + self.overlay.len() + } + + pub fn get(&self) -> Option<&Arc> { + let base_max = self.base_max.as_ref(); + let overlay_max = self.overlay.iter().max(); + match (base_max, overlay_max) { + (Some(a), Some(b)) => Some(if a >= b { a } else { b }), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } + } + + /// Ascending iteration over the LOGICAL set (base ∪ overlay). + fn iter_ascending(&self) -> impl Iterator> { + let mut overlay: smallvec::SmallVec<[&Arc; 8]> = + self.overlay.iter().collect(); + overlay.sort_unstable(); + MergeAscending { + base: self.base_ref().iter().peekable(), + overlay, + overlay_ix: 0, + } + } + + fn is_empty(&self) -> bool { + self.overlay.is_empty() && self.base.as_ref().is_none_or(|base| base.is_empty()) + } + + fn contains(&self, op: &Arc) -> bool { + self.overlay.contains(op) || self.base_ref().contains(op) + } + + fn remove_op(&mut self, op: &Arc) -> bool { + if let Some(ix) = self.overlay.iter().position(|x| x == op) { + self.overlay.swap_remove(ix); + return true; + } + if let Some(base) = self.base.as_mut() { + if base.remove(op).is_some() { + if self.base_max.as_ref().is_some_and(|max| max == op) { + self.base_max = base.get_max().cloned(); + } + return true; + } + } + false + } + + /// Newest `(lamport, peer)` op present in BOTH logical sets — the visible + /// value of the intersection, found WITHOUT materializing it. This is what + /// the per-keystroke `get_styles_for_insert` actually needs (its result + /// only ever feeds `StyleMeta`, which reads each key's max op). + fn max_common(&self, other: &Self) -> Option> { + let mut best: Option<&Arc> = None; + for op in self.overlay.iter() { + if best.is_none_or(|b| op > b) && other.contains(op) { + best = Some(op); + } + } + for op in other.overlay.iter() { + if best.is_none_or(|b| op > b) && self.contains(op) { + best = Some(op); + } + } + if self.base_ref().ptr_eq(other.base_ref()) { + if let Some(max) = self.base_max.as_ref() { + if best.is_none_or(|b| max > b) { + best = Some(max); + } + } + } else { + // Descending double-walk to the first common base op. + let mut left = self.base_ref().iter().rev().peekable(); + let mut right = other.base_ref().iter().rev().peekable(); + while let (Some(a), Some(b)) = (left.peek(), right.peek()) { + if best.is_some_and(|best_op| best_op >= a) || best.is_some_and(|best_op| best_op >= b) { + break; + } + match a.cmp(b) { + std::cmp::Ordering::Equal => { + if best.is_none_or(|best_op| *a > best_op) { + best = Some(*a); + } + break; + } + std::cmp::Ordering::Greater => { + left.next(); + } + std::cmp::Ordering::Less => { + right.next(); + } + } + } + } + best.cloned() + } + + /// `self ∩= right`, returning whether the result is non-empty. A mark's + /// start/end boundary intersection equals one SIDE (the doc-comment proof + /// on [`StyleRangeMap::insert`]: the differing ops are exactly the anchors + /// at the boundary), so the subset fast paths reuse that side's storage + /// (O(compare), zero copies) and the allocating rebuild only runs for + /// boundaries where ops both start AND end. + fn intersect_with(&mut self, right: &Self) -> bool { + if self.base_ref().ptr_eq(right.base_ref()) + && self.overlay.is_empty() + && right.overlay.is_empty() + { + return !self.is_empty(); + } + // One O(n+m) ordered double-walk over the LOGICAL sets decides both + // subset relations. + let mut left_only = false; + let mut right_only = false; + { + let mut left = self.iter_ascending().peekable(); + let mut right_iter = right.iter_ascending().peekable(); + loop { + match (left.peek(), right_iter.peek()) { + (Some(a), Some(b)) => match a.cmp(b) { + std::cmp::Ordering::Equal => { + left.next(); + right_iter.next(); + } + std::cmp::Ordering::Less => { + left_only = true; + left.next(); + } + std::cmp::Ordering::Greater => { + right_only = true; + right_iter.next(); + } + }, + (Some(_), None) => { + left_only = true; + break; + } + (None, Some(_)) => { + right_only = true; + break; + } + (None, None) => break, + } + } + } + if !right_only { + // right ⊆ self: intersection IS right — O(1) persistent clone. + *self = right.clone(); + } else if left_only { + // Neither side contains the other — materialize the intersection. + let mut result = StyleOpSet::new(); + { + let mut left = self.iter_ascending().peekable(); + let mut right_iter = right.iter_ascending().peekable(); + while let (Some(a), Some(b)) = (left.peek(), right_iter.peek()) { + match a.cmp(b) { + std::cmp::Ordering::Equal => { + result.insert((*a).clone()); + left.next(); + right_iter.next(); + } + std::cmp::Ordering::Less => { + left.next(); + } + std::cmp::Ordering::Greater => { + right_iter.next(); + } + } + } + } + self.base_max = result.get_max().cloned(); + self.base = Some(result); + self.overlay.clear(); + } + // else: self ⊆ right — intersection is self, keep as-is. + !self.is_empty() + } +} + +/// Ascending merge of a sorted persistent base and a small sorted overlay. +struct MergeAscending<'value> { + base: std::iter::Peekable>>, + overlay: smallvec::SmallVec<[&'value Arc; 8]>, + overlay_ix: usize, +} + +impl<'value> Iterator for MergeAscending<'value> { + type Item = &'value Arc; + + fn next(&mut self) -> Option { + let overlay_next = self.overlay.get(self.overlay_ix).copied(); + match (self.base.peek(), overlay_next) { + (Some(b), Some(o)) => { + if *b <= o { + self.base.next() + } else { + self.overlay_ix += 1; + Some(o) + } + } + (Some(_), None) => self.base.next(), + (None, Some(o)) => { + self.overlay_ix += 1; + Some(o) + } + (None, None) => None, + } + } +} + +impl Default for StyleRangeMap { + fn default() -> Self { + Self::new() + } +} + +type YieldStyle<'a> = Option<&'a mut dyn FnMut(&Styles, usize)>; + +impl StyleRangeMap { + /// §flowstate style-map probe (diagnostic, env-gated `FLOWSTATE_STYLE_MAP_PROBE`): + /// sampled census of the range map — element (boundary-fragment) count, total + /// `StyleOp` set entries, and the largest single set. Distinguishes the two + /// candidate blowups behind the styled-divergence quadratic: FRAGMENTATION + /// (elems explode because adjacent same-visible-style elems differ by dead + /// ops and `can_merge`'s deep set equality never fires) vs ACCUMULATION + /// (individual sets explode). Sampled every 8192 calls; the O(elems) walk is + /// paid only on samples, and never when the env flag is absent. + fn probe_sample(&self, op: &'static str) { + use std::sync::atomic::{AtomicU64, Ordering}; + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + if !*ENABLED.get_or_init(|| std::env::var_os("FLOWSTATE_STYLE_MAP_PROBE").is_some()) { + return; + } + static CALLS: AtomicU64 = AtomicU64::new(0); + let call = CALLS.fetch_add(1, Ordering::Relaxed); + if call % 8192 != 0 { + return; + } + let mut elems = 0usize; + let mut set_entries = 0usize; + let mut max_set = 0usize; + let mut styled_elems = 0usize; + for elem in self.tree.iter() { + elems += 1; + let mut elem_entries = 0usize; + for value in elem.styles.values() { + elem_entries += value.len(); + max_set = max_set.max(value.len()); + } + if elem_entries > 0 { + styled_elems += 1; + } + set_entries += elem_entries; + } + eprintln!( + "[flowstate-style-map] op={op} calls={call} elems={elems} styled_elems={styled_elems} set_entries={set_entries} max_set={max_set}" + ); + } + + pub fn new() -> Self { + let mut tree = BTree::new(); + tree.push(Elem { + styles: Default::default(), + len: usize::MAX / 4, + }); + + Self { + tree, + has_style: false, + max_op_ever: None, + } + } + + pub fn annotate( + &mut self, + range: Range, + style: Arc, + mut yield_style: YieldStyle, + ) { + self.probe_sample("annotate"); + let range = self.tree.range::(range); + if range.is_none() { + unreachable!(); + } + + self.has_style = true; + let range = range.unwrap(); + // §flowstate stylemap patch: an annotate legitimately visits every + // covered elem (per-elem membership is load-bearing), so per-visit + // work must stay O(1) — `StyleValue::insert` lands in the small owned + // overlay, never copy-on-writing the shared base. + let style_key = style.get_style_key(); + let fresh = self.max_op_ever.as_ref().is_none_or(|max| style > *max); + if fresh { + self.max_op_ever = Some(style.clone()); + } + self.tree + .update(range.start.cursor..range.end.cursor, &mut |x| { + if let Some(value) = x.styles.get_mut(&style_key) { + if fresh { + value.insert_unchecked(style.clone()); + } else { + value.insert(style.clone()); + } + } else { + let mut value = StyleValue::default(); + value.insert_unchecked(style.clone()); + x.styles.insert(style_key.clone(), value); + } + + if let Some(y) = yield_style.as_mut() { + y(&x.styles, x.len); + } + None + }); + } + + /// Get the styles of the range. If the range is not in the same leaf, return None. + pub(crate) fn get_styles_of_range(&self, range: Range) -> Option<&Styles> { + if !self.has_style { + return None; + } + + let right = self + .tree + .query::(&(range.end - 1)) + .unwrap() + .cursor; + let left = self + .tree + .query::(&range.start) + .unwrap() + .cursor; + if left.leaf == right.leaf { + Some(&self.tree.get_elem(left.leaf).unwrap().styles) + } else { + None + } + } + + pub(crate) fn range_contains_key(&self, range: Range, key: &StyleKey) -> bool { + if range.is_empty() || !self.has_style { + return false; + } + + let mut query = self.tree.query::(&range.start).unwrap(); + let mut pos = range.start; + loop { + let elem = self.tree.get_elem(query.cursor.leaf).unwrap(); + if elem.styles.contains_key(key) { + return true; + } + + let remaining_in_elem = elem.len - query.cursor.offset; + let next_pos = pos + remaining_in_elem; + if next_pos >= range.end { + break; + } + + match self.tree.next_elem(query.cursor) { + Some(next_cursor) => { + pos = next_pos; + query.cursor = next_cursor; + } + None => break, + } + } + + false + } + + /// Insert entities at `pos` with length of `len` + /// + /// # Internal + /// + /// When inserting new text, we need to calculate the StyleSet of the new text based on the StyleSet before and after the insertion position. + /// (It should be the intersection of the StyleSet before and after). The proof is as follows: + /// + /// Suppose when inserting text at position pos, the style set at positions pos - 1 and pos are called leftStyleSet and rightStyleSet respectively. + /// + /// - If there is a style x that exists in leftStyleSet but not in rightStyleSet, it means that the position pos - 1 is the end anchor of x. + /// The newly inserted text is after the end anchor of x, so the StyleSet of the new text should not include this style. + /// - If there is a style x that exists in rightStyleSet but not in leftStyleSet, it means that the position pos is the start anchor of x. + /// The newly inserted text is before the start anchor of x, so the StyleSet of the new text should not include this style. + /// - If both leftStyleSet and rightStyleSet contain style x, it means that the newly inserted text is within the style range, so the StyleSet should include x. + pub fn insert(&mut self, pos: usize, len: usize) -> &Styles { + if !self.has_style { + return &EMPTY_STYLES; + } + self.probe_sample("insert"); + + if pos == 0 { + self.tree.prepend(Elem { + len, + styles: Default::default(), + }); + return &EMPTY_STYLES; + } + + if pos as isize == *self.tree.root_cache() { + self.tree.push(Elem { + len, + styles: Default::default(), + }); + return &EMPTY_STYLES; + } + + let right = self.tree.query::(&pos).unwrap().cursor; + let left = self.tree.query::(&(pos - 1)).unwrap().cursor; + if left.leaf == right.leaf { + // left and right are in the same element, we can increase the length of the element directly + self.tree.update_leaf(left.leaf, |x| { + x.len += len; + (true, None, None) + }); + return &self.tree.get_elem(left.leaf).unwrap().styles; + } + + // insert by the intersection of left styles and right styles + // (§flowstate stylemap patch: the map clone is Arc bumps per key, and + // `intersect_with` reuses one side's storage on the common + // pure-start/pure-end boundary — see its doc comment.) + let mut styles = self.tree.get_elem(left.leaf).unwrap().styles.clone(); + let right_styles = &self.tree.get_elem(right.leaf).unwrap().styles; + styles.retain(|key, value| { + if let Some(right_value) = right_styles.get(key) { + return value.intersect_with(right_value); + } + + false + }); + + let (target, _) = self.tree.insert_by_path(right, Elem { len, styles }); + &self.tree.get_elem(target.leaf).unwrap().styles + } + + /// Return the style sets beside `index` and get the intersection of them. + pub fn get_styles_for_insert(&self, index: usize) -> StyleMeta { + if index == 0 || !self.has_style { + return StyleMeta::default(); + } + + let left = self + .tree + .query::(&(index - 1)) + .unwrap() + .cursor; + let right = self.tree.shift_path_by_one_offset(left).unwrap(); + if left.leaf == right.leaf { + let styles = &self.tree.get_elem(left.leaf).unwrap().styles; + styles.into() + } else { + // §flowstate stylemap patch: this result only ever feeds + // `StyleMeta` (per-key max op), so build it via `max_common` — a + // descending double-walk per key, ZERO set materialization. This + // is the per-keystroke path (`get_styles_at_entity_index_for_insert`). + let left_styles = &self.tree.get_elem(left.leaf).unwrap().styles; + let right_styles = &self.tree.get_elem(right.leaf).unwrap().styles; + let mut meta = StyleMeta::default(); + for (key, left_value) in left_styles.iter() { + if let Some(right_value) = right_styles.get(key) { + if let Some(op) = left_value.max_common(right_value) { + meta.insert( + key.key().clone(), + crate::delta::StyleMetaItem { + value: op.to_value(), + lamport: op.lamport, + peer: op.peer, + }, + ); + } + } + } + + meta + } + } + + pub fn iter(&self) -> impl Iterator, &Styles)> + '_ { + let mut index = 0; + self.tree.iter().filter_map(move |elem| { + let len = elem.len; + let value = &elem.styles; + let range = index..index + len; + index += len; + if elem.styles.is_empty() { + return None; + } + + Some((range, value)) + }) + } + + /// Update the styles from `pos` to the start of the document. + fn update_styles_scanning_backward( + &mut self, + pos: usize, + mut f: impl FnMut(&mut Elem) -> ControlFlow<()>, + ) { + let mut cursor = self.tree.query::(&pos).map(|x| x.cursor); + while let Some(inner_cursor) = cursor { + cursor = self.tree.prev_elem(inner_cursor); + let node = self.tree.get_elem_mut(inner_cursor.leaf).unwrap(); + match f(node) { + ControlFlow::Continue(_) => {} + ControlFlow::Break(_) => { + break; + } + } + } + } + + pub(crate) fn iter_range( + &self, + range: impl RangeBounds, + ) -> impl Iterator> + '_ { + let start = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => *x + 1, + std::ops::Bound::Unbounded => 0, + }; + + let end = match range.end_bound() { + std::ops::Bound::Included(x) => *x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => usize::MAX, + }; + + let start = self.tree.query::(&start).unwrap(); + let end = self.tree.query::(&end).unwrap(); + self.tree.iter_range(start.cursor..end.cursor) + } + + /// Return the expected style anchors with their indexes. + pub(super) fn iter_anchors(&self) -> impl Iterator + '_ { + let mut index = 0; + let empty_styles = &EMPTY_STYLES; + let mut last: Option<&Elem> = None; + let mut vec = Vec::new(); + for cur in self.tree.iter() { + let last_styles = last.map(|x| &x.styles).unwrap_or(empty_styles); + let (left_anchor, right_anchor) = last_styles.infer_anchors(&cur.styles); + if let Some(left) = left_anchor { + vec.push(IterAnchorItem { + index: index - 1, + op: left.clone(), + anchor_type: AnchorType::End, + }); + } + if let Some(right) = right_anchor { + vec.push(IterAnchorItem { + index, + op: right.clone(), + anchor_type: AnchorType::Start, + }); + } + + last = Some(cur); + index += cur.len; + } + + let last_styles = last.map(|x| &x.styles).unwrap_or(empty_styles); + let (left_anchor, right_anchor) = last_styles.infer_anchors(empty_styles); + if let Some(left) = left_anchor { + vec.push(IterAnchorItem { + index: index - 1, + op: left.clone(), + anchor_type: AnchorType::End, + }); + } + if let Some(right) = right_anchor { + vec.push(IterAnchorItem { + index, + op: right.clone(), + anchor_type: AnchorType::Start, + }); + } + + vec.into_iter() + } + + /// Remove the style scanning backward, return the start_entity_index + pub fn remove_style_scanning_backward( + &mut self, + to_remove: &Arc, + last_index: usize, + ) -> usize { + let mut removed_len = 0; + let key = to_remove.get_style_key(); + self.update_styles_scanning_backward(last_index, |elem| { + removed_len += elem.len; + let styles = &mut elem.styles; + let mut has_removed = false; + if let Some(value) = styles.get_mut(&key) { + has_removed = value.remove_op(to_remove); + if value.is_empty() { + styles.remove(&key); + } + } + + if has_removed { + ControlFlow::Continue(()) + } else { + ControlFlow::Break(()) + } + }); + + last_index + 1 - removed_len + } + + pub fn delete(&mut self, range: Range) { + if !self.has_style { + return; + } + + let start = self.tree.query::(&range.start).unwrap(); + let end = self.tree.query::(&range.end).unwrap(); + if start.cursor.leaf == end.cursor.leaf { + // delete in the same element + self.tree.update_leaf(start.cursor.leaf, |x| { + x.len -= range.len(); + (true, None, None) + }); + return; + } + + self.tree.drain(start..end); + } + + pub(crate) fn has_style(&self) -> bool { + self.has_style + } +} + +pub(super) struct IterAnchorItem { + pub(super) index: usize, + pub(super) op: Arc, + pub(super) anchor_type: AnchorType, +} + +impl UseLengthFinder for RangeNumMapTrait { + fn get_len(cache: &isize) -> usize { + *cache as usize + } +} + +impl HasLength for Elem { + fn rle_len(&self) -> usize { + self.len + } +} + +impl Mergeable for Elem { + fn can_merge(&self, rhs: &Self) -> bool { + self.styles == rhs.styles || rhs.len == 0 + } + + fn merge_right(&mut self, rhs: &Self) { + self.len += rhs.len + } + + fn merge_left(&mut self, left: &Self) { + self.len += left.len; + } +} + +impl Sliceable for Elem { + fn _slice(&self, range: std::ops::Range) -> Self { + let len = range.len(); + Elem { + styles: self.styles.clone(), + len, + } + } +} + +impl TryInsert for Elem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + if self.styles == elem.styles { + self.len += elem.len; + Ok(()) + } else { + Err(elem) + } + } +} + +impl CanRemove for Elem { + fn can_remove(&self) -> bool { + self.len == 0 + } +} + +impl BTreeTrait for RangeNumMapTrait { + type Elem = Elem; + type Cache = isize; + type CacheDiff = isize; + const USE_DIFF: bool = true; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> isize { + let new_cache = caches.iter().map(|c| c.cache).sum(); + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2; + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += diff; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + elem.len as isize + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } +} + +#[cfg(test)] +mod test { + use loro_common::PeerID; + + use crate::{change::Lamport, container::richtext::TextStyleInfoFlag}; + + use super::*; + + fn new_style(n: i32) -> Arc { + Arc::new(StyleOp { + lamport: n as Lamport, + peer: n as PeerID, + cnt: n, + key: n.to_string().into(), + info: TextStyleInfoFlag::default(), + value: loro_common::LoroValue::Bool(true), + }) + } + + #[test] + fn test_basic_insert() { + let mut map = StyleRangeMap::default(); + map.annotate(1..10, new_style(1), None); + { + map.insert(0, 1); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 2..11); + assert_eq!(map.len(), 1); + } + } + { + map.insert(11, 1); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 2..11); + assert_eq!(map.len(), 1); + } + } + { + map.insert(10, 1); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 2..12); + assert_eq!(map.len(), 1); + } + } + } + + #[test] + fn delete_style() { + let mut map = StyleRangeMap::default(); + map.annotate(1..10, new_style(1), None); + { + map.delete(0..2); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 0..8); + assert_eq!(map.len(), 1); + } + } + { + map.delete(2..4); + for (range, map) in map.iter() { + assert_eq!(range, 0..6); + assert_eq!(map.len(), 1); + } + assert_eq!(map.iter().count(), 1); + } + { + map.delete(6..8); + assert_eq!(map.iter().count(), 1); + for (range, map) in map.iter() { + assert_eq!(range, 0..6); + assert_eq!(map.len(), 1); + } + } + } +} diff --git a/vendor/loro-internal/src/container/richtext/tinyvec.rs b/vendor/loro-internal/src/container/richtext/tinyvec.rs new file mode 100644 index 00000000..6f852fe1 --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/tinyvec.rs @@ -0,0 +1,289 @@ +use std::{fmt::Debug, mem::MaybeUninit}; + +pub(crate) struct TinyVec { + len: u8, + data: [MaybeUninit; N], +} + +impl Debug for TinyVec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.iter()).finish() + } +} + +impl Clone for TinyVec { + fn clone(&self) -> Self { + let mut result = Self::new(); + for item in self.iter() { + result + .push(item.clone()) + .ok() + .expect("TinyVec clone should fit in the same capacity"); + } + result + } +} + +impl std::ops::Index for TinyVec { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + assert!(index < self.len as usize); + // SAFETY: This is safe because we know that the data is initialized + unsafe { self.data[index].assume_init_ref() } + } +} + +impl TinyVec { + #[inline] + pub fn new() -> Self { + if N > u8::MAX as usize { + panic!("TinyVec size too large"); + } + + Self { + len: 0, + // SAFETY: This initialization is copied from std + // SAFETY: An uninitialized `[MaybeUninit<_>; LEN]` is valid. + data: unsafe { MaybeUninit::<[MaybeUninit; N]>::uninit().assume_init() }, + } + } + + pub fn push(&mut self, value: T) -> Result<(), T> { + if self.len == N as u8 { + return Err(value); + } + + self.data[self.len as usize] = MaybeUninit::new(value); + self.len += 1; + Ok(()) + } + + #[inline(always)] + pub fn get(&self, index: usize) -> &T { + &self[index] + } + + pub fn pop(&mut self) -> Option { + if self.len == 0 { + None + } else { + self.len -= 1; + // SAFETY: This is safe because we know that the last element is initialized + Some(unsafe { self.data[self.len as usize].assume_init_read() }) + } + } + + #[inline(always)] + pub fn len(&self) -> usize { + self.len as usize + } + + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn iter(&self) -> impl Iterator { + let mut index = 0; + std::iter::from_fn(move || { + if index < self.len as usize { + let result = &self[index]; + index += 1; + Some(result) + } else { + None + } + }) + } + + pub fn can_merge(&self, other: &Self) -> bool { + if self.len() + other.len() > N { + return false; + } + + self.len() + other.len() <= self.data.len() + } + + pub fn merge(&mut self, other: &Self) + where + T: Clone, + { + if !self.can_merge(other) { + panic!("TinyVec cannot merge"); + } + + for item in other.iter() { + self.push(item.clone()) + .ok() + .expect("TinyVec merge should fit after can_merge"); + } + } + + pub fn merge_left(&mut self, left: &Self) + where + T: Clone, + { + if !self.can_merge(left) { + panic!("TinyVec cannot merge"); + } + + let mut result = Self::new(); + for item in left.iter().chain(self.iter()) { + result + .push(item.clone()) + .ok() + .expect("TinyVec merge_left should fit after can_merge"); + } + *self = result; + } + + pub fn slice(&self, start: usize, end: usize) -> Self + where + T: Clone, + { + assert!(start <= end && end <= self.len as usize); + let mut result = Self::new(); + for item in self.iter().skip(start).take(end - start) { + result + .push(item.clone()) + .ok() + .expect("TinyVec slice should fit in the same capacity"); + } + result + } + + #[inline] + pub fn split(&mut self, pos: usize) -> Self { + assert!(pos <= self.len()); + let old_len = self.len(); + let mut result = Self::new(); + for i in pos..old_len { + // SAFETY: indexes below old_len are initialized. The value is moved + // into result and self.len is truncated below before self is dropped. + let value = unsafe { self.data[i].assume_init_read() }; + result + .push(value) + .ok() + .expect("TinyVec split should fit in the same capacity"); + } + self.len = pos as u8; + result + } +} + +impl TinyVec { + pub fn to_vec(&self) -> Vec { + let mut result = Vec::with_capacity(self.len as usize); + for i in 0..self.len as usize { + result.push(self[i].clone()); + } + result + } +} + +impl PartialEq for TinyVec { + fn eq(&self, other: &Self) -> bool { + if self.len != other.len { + return false; + } + + for i in 0..self.len as usize { + if self[i] != other[i] { + return false; + } + } + + true + } +} + +impl Eq for TinyVec {} + +impl Drop for TinyVec { + fn drop(&mut self) { + for i in 0..self.len as usize { + // SAFETY: This is safe because we know that the element is initialized + unsafe { std::ptr::drop_in_place(self.data[i].as_mut_ptr()) }; + } + } +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use super::TinyVec; + + #[test] + fn pop_should_drop() { + let mut arr: TinyVec<_, 2> = TinyVec::new(); + let elem = Arc::new(10); + assert_eq!(Arc::strong_count(&elem), 1); + arr.push(elem.clone()).unwrap(); + assert_eq!(Arc::strong_count(&elem), 2); + arr.pop(); + assert_eq!(Arc::strong_count(&elem), 1); + } + + #[test] + fn dropping_should_drop_all_elem() { + let mut arr: TinyVec<_, 2> = TinyVec::new(); + let elem = Arc::new(10); + assert_eq!(Arc::strong_count(&elem), 1); + arr.push(elem.clone()).unwrap(); + assert_eq!(Arc::strong_count(&elem), 2); + drop(arr); + assert_eq!(Arc::strong_count(&elem), 1); + } + + #[test] + fn split() { + let mut arr: TinyVec = TinyVec::new(); + arr.push(1).unwrap(); + arr.push(2).unwrap(); + arr.push(3).unwrap(); + arr.push(4).unwrap(); + let new = arr.split(2); + assert_eq!(arr.to_vec(), vec![1, 2]); + assert_eq!(new.to_vec(), vec![3, 4]); + } + + #[test] + fn slice() { + let mut arr: TinyVec = TinyVec::new(); + arr.push(1).unwrap(); + arr.push(2).unwrap(); + arr.push(3).unwrap(); + arr.push(4).unwrap(); + arr.push(5).unwrap(); + let arr = arr; + let new = arr.slice(2, 4); + assert_eq!(new.to_vec(), vec![3, 4]); + } + + #[test] + fn clone() { + let mut arr: TinyVec = TinyVec::new(); + arr.push(1).unwrap(); + arr.push(2).unwrap(); + arr.push(3).unwrap(); + arr.push(4).unwrap(); + arr.push(5).unwrap(); + let arr = arr; + let new = arr.clone(); + assert_eq!(arr, new); + assert_eq!(new.to_vec(), [1, 2, 3, 4, 5]); + } + + #[cfg(miri)] + #[test] + fn clone_of_owned_values_does_not_duplicate_ownership_with_raw_copy() { + let mut arr: TinyVec, 2> = TinyVec::new(); + arr.push(Box::new(1)).unwrap(); + + let cloned = arr.clone(); + drop(arr); + drop(cloned); + } +} diff --git a/vendor/loro-internal/src/container/richtext/tracker.rs b/vendor/loro-internal/src/container/richtext/tracker.rs new file mode 100644 index 00000000..70cab4f1 --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/tracker.rs @@ -0,0 +1,784 @@ +use std::ops::ControlFlow; + +use generic_btree::{ + rle::{HasLength as _, Sliceable}, + LeafIndex, +}; +use loro_common::{Counter, HasId, HasIdSpan, IdFull, IdSpan, Lamport, PeerID, ID}; +use rle::HasLength as _; +use tracing::instrument; + +use crate::{cursor::AbsolutePosition, VersionVector}; + +use self::{crdt_rope::CrdtRope, id_to_cursor::IdToCursor}; + +use super::{ + fugue_span::{FugueSpan, Status}, + RichtextChunk, +}; + +mod crdt_rope; +mod id_to_cursor; +pub(crate) use crdt_rope::CrdtRopeDelta; + +#[derive(Debug)] +pub(crate) struct Tracker { + applied_vv: VersionVector, + current_vv: VersionVector, + rope: CrdtRope, + id_to_cursor: IdToCursor, +} + +impl Default for Tracker { + fn default() -> Self { + Self::new_with_unknown() + } +} + +pub(super) const UNKNOWN_PEER_ID: PeerID = u64::MAX; +impl Tracker { + pub fn new_with_unknown() -> Self { + let mut this = Self { + rope: CrdtRope::new(), + id_to_cursor: IdToCursor::default(), + applied_vv: Default::default(), + current_vv: Default::default(), + }; + + let result = this.rope.tree.push(FugueSpan { + content: RichtextChunk::new_unknown(u32::MAX / 4), + id: IdFull::new(UNKNOWN_PEER_ID, 0, 0), + real_id: None, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + }); + this.id_to_cursor.insert_without_split( + ID::new(UNKNOWN_PEER_ID, 0), + id_to_cursor::Cursor::new_insert(result.leaf, u32::MAX as usize / 4), + ); + this + } + + #[allow(unused)] + fn new() -> Self { + Self { + rope: CrdtRope::new(), + id_to_cursor: IdToCursor::default(), + applied_vv: Default::default(), + current_vv: Default::default(), + } + } + + #[inline] + pub fn all_vv(&self) -> &VersionVector { + &self.applied_vv + } + + #[inline] + pub fn current_vv(&self) -> &VersionVector { + &self.current_vv + } + + pub(crate) fn insert(&mut self, mut op_id: IdFull, mut pos: usize, mut content: RichtextChunk) { + // trace!( + // "TrackerInsert op_id = {:#?}, pos = {:#?}, content = {:#?}", + // op_id, + // &pos, + // &content + // ); + // tracing::span!(tracing::Level::INFO, "TrackerInsert"); + if let ControlFlow::Break(_) = + self.skip_applied(op_id.id(), content.len(), |applied_counter_end| { + // the op is partially included, need to slice the content + let start = (applied_counter_end - op_id.counter) as usize; + op_id.lamport += (applied_counter_end - op_id.counter) as Lamport; + op_id.counter = applied_counter_end; + pos += start; + content = content.slice(start..); + }) + { + return; + } + + // { + // tracing::span!(tracing::Level::INFO, "before insert {} pos={}", op_id, pos); + // debug_log::debug_dbg!(&self); + // } + self._insert(pos, content, op_id); + } + + fn _insert(&mut self, pos: usize, content: RichtextChunk, op_id: IdFull) { + let result = self.rope.insert( + pos, + FugueSpan { + content, + id: op_id, + real_id: if op_id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(op_id.id().try_into().unwrap()) + }, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + }, + |id| self.id_to_cursor.get_insert(id).unwrap(), + ); + self.id_to_cursor.insert( + op_id.id(), + id_to_cursor::Cursor::new_insert(result.leaf, content.len()), + ); + + self.update_insert_by_split(&result.splitted.arr); + + let end_id = op_id.inc(content.len() as Counter); + self.current_vv.extend_to_include_end_id(end_id.id()); + self.applied_vv.extend_to_include_end_id(end_id.id()); + } + + fn update_insert_by_split(&mut self, split: &[LeafIndex]) { + match split.len() { + 0 => {} + 1 => { + let new_leaf_idx = split[0]; + let leaf = self.rope.tree().get_elem(new_leaf_idx).unwrap(); + self.id_to_cursor + .update_insert(leaf.id_span(), new_leaf_idx); + } + _ => { + let mut updates = Vec::with_capacity(split.len()); + for &new_leaf_idx in split { + let leaf = self.rope.tree().get_elem(new_leaf_idx).unwrap(); + updates.push((leaf.id_span(), new_leaf_idx)); + } + self.id_to_cursor.update_insert_batch(&mut updates); + } + } + } + + /// Delete the element from pos..pos+len + /// + /// If `reverse` is true, the deletion happens from the end of the range to the start. + /// So the first op is the one that deletes element at `pos+len-1`, the last op + /// is the one that deletes element at `pos`. + /// + /// - op_id: the first op that perform the deletion + /// - target_start_id: in the target deleted span, it's the first id of the span + /// - pos: the start pos of the deletion in the target span + /// - len: the length of the deletion span + /// - reverse: if true, the kth op delete the last kth element of the span + pub(crate) fn delete( + &mut self, + mut op_id: ID, + mut target_start_id: ID, + pos: usize, + mut len: usize, + reverse: bool, + ) { + if let ControlFlow::Break(_) = self.skip_applied(op_id, len, |applied_counter_end: i32| { + // the op is partially included, need to slice the op + let start = (applied_counter_end - op_id.counter) as usize; + op_id.counter = applied_counter_end; + if !reverse { + target_start_id = target_start_id.inc(start as i32); + } + // Okay, this looks pretty weird, but it's correct. + // If it's reverse, we don't need to change the target_start_id, because target_start_id always pointing towards the + // leftmost element of the span. After applying the initial part of the deletion, which starts from the right side, + // the target_start_id will be still pointing towards the same leftmost element, thus no need to change. + len -= start; + // If reverse, don't need to change the pos, because it's deleting backwards. + // If not reverse, we don't need to change the pos either, because the `start` chars after it are already deleted + }) { + return; + } + + // tracing::info!("after forwarding pos={} len={}", pos, len); + + self._delete(target_start_id, pos, len, reverse, op_id); + } + + fn _delete(&mut self, target_start_id: ID, pos: usize, len: usize, reverse: bool, op_id: ID) { + let mut ans = Vec::new(); + let split = self + .rope + .delete(target_start_id, pos, len, reverse, &mut |span| { + let mut id_span = span.id_span(); + if reverse { + id_span.reverse(); + } + ans.push(id_span); + }); + + let mut cur_id = op_id; + for id_span in ans { + let len = id_span.atom_len(); + self.id_to_cursor + .insert(cur_id, id_to_cursor::Cursor::Delete(id_span)); + cur_id = cur_id.inc(len as Counter); + } + + debug_assert_eq!(cur_id.counter - op_id.counter, len as Counter); + for s in split { + self.update_insert_by_split(&s.arr); + } + + let end_id = op_id.inc(len as Counter); + self.current_vv.extend_to_include_end_id(end_id); + self.applied_vv.extend_to_include_end_id(end_id); + } + + fn skip_applied( + &mut self, + op_id: ID, + len: usize, + mut f: impl FnMut(Counter), + ) -> ControlFlow<()> { + let last_id = op_id.inc(len as Counter - 1); + let applied_counter_end = self.applied_vv.get(&last_id.peer).copied().unwrap_or(0); + if applied_counter_end > op_id.counter { + if !self.current_vv.includes_id(last_id) { + // PERF: may be slow + let mut updates = Default::default(); + let cnt_start = self.current_vv.get(&op_id.peer).copied().unwrap_or(0); + self.forward( + IdSpan::new(op_id.peer, cnt_start, op_id.counter + len as Counter), + &mut updates, + ); + self.batch_update(updates, false); + } + + if applied_counter_end > last_id.counter { + self.current_vv.extend_to_include_last_id(last_id); + return ControlFlow::Break(()); + } + + f(applied_counter_end); + } + ControlFlow::Continue(()) + } + + /// Internally it's delete at the src and insert at the dst. + /// + /// But it needs special behavior for id_to_cursor data structure + /// + #[instrument(skip(self))] + pub(crate) fn move_item( + &mut self, + op_id: IdFull, + deleted_id: ID, + from_pos: usize, + to_pos: usize, + ) { + if let ControlFlow::Break(_) = self.skip_applied(op_id.id(), 1, |_| unreachable!()) { + return; + } + + // We record the **fake** id of the deleted item, and store it in the `id_to_cursor`. + // This is because when we retreat, we need to know the **fake** id of the deleted item, + // so that we can look up the insert pos in `id_to_cursor` + // + // > `id_to_cursor` only stores the mappings from **fake** insert id to the leaf index. + // > **Fake** means the id may be a temporary placeholder, created with UNKNOWN_PEER_ID. + let mut fake_delete_id = None; + let split = self.rope.delete(deleted_id, from_pos, 1, false, &mut |s| { + debug_assert_eq!(s.rle_len(), 1); + fake_delete_id = Some(s.id.id()); + }); + + for s in split { + self.update_insert_by_split(&s.arr); + } + + let result = self.rope.insert( + to_pos, + FugueSpan { + // we need to use the special move content to avoid + // its merging with other [FugueSpan], which will make + // id_to_cursor need to track its split. + // It would be much harder to implement correctly + content: RichtextChunk::new_move(), + id: op_id, + real_id: if op_id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(op_id.id().try_into().unwrap()) + }, + status: Status::default(), + diff_status: None, + origin_left: None, + origin_right: None, + }, + |id| self.id_to_cursor.get_insert(id).unwrap(), + ); + self.update_insert_by_split(&result.splitted.arr); + + self.id_to_cursor.insert( + op_id.id(), + id_to_cursor::Cursor::new_move(result.leaf, fake_delete_id.unwrap()), + ); + + let end_id = op_id.inc(1); + self.current_vv.extend_to_include_end_id(end_id.id()); + self.applied_vv.extend_to_include_end_id(end_id.id()); + } + + #[inline] + pub(crate) fn checkout(&mut self, vv: &VersionVector) { + self._checkout(vv, false); + } + + fn _checkout(&mut self, vv: &VersionVector, on_diff_status: bool) { + // tracing::info!("Checkout to {:?} from {:?}", vv, self.current_vv); + if on_diff_status { + self.rope.clear_diff_status(); + } + + let current_vv = std::mem::take(&mut self.current_vv); + let (retreat, forward) = current_vv.diff_iter(vv); + let mut updates = Vec::new(); + for span in retreat { + for c in self.id_to_cursor.iter(span) { + match c { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: Some(true), + delete_times_diff: 0, + }) + } + id_to_cursor::IterCursor::Delete(span) => { + for to_del in self.id_to_cursor.iter(span) { + match to_del { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: -1, + }) + } + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf, + new_op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to_leaf, + id_span: new_op_id.to_span(1), + set_future: None, + delete_times_diff: -1, + }), + _ => unreachable!(), + } + } + } + id_to_cursor::IterCursor::Move { + from_id: from, + to_leaf: to, + new_op_id: op_id, + } => { + let mut visited = false; + for to_del in self.id_to_cursor.iter(IdSpan::new( + from.peer, + from.counter, + from.counter + 1, + )) { + visited = true; + + match to_del { + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf: to, + new_op_id: op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: op_id.to_span(1), + set_future: None, + delete_times_diff: -1, + }), + // Un delete the from + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + debug_assert_eq!(id_span.atom_len(), 1); + debug_assert_eq!(id_span.counter.start, from.counter); + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: -1, + }) + } + _ => unreachable!(), + } + } + assert!(visited); + // insert the new + updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: IdSpan::new(op_id.peer, op_id.counter, op_id.counter + 1), + set_future: Some(true), + delete_times_diff: 0, + }); + } + } + } + } + + for span in forward { + self.forward(span, &mut updates); + } + + if !on_diff_status { + self.current_vv = vv.clone(); + } else { + self.current_vv = current_vv; + } + + self.batch_update(updates, on_diff_status); + } + + fn batch_update(&mut self, updates: Vec, on_diff_status: bool) { + let leaf_indexes = self.rope.update(updates, on_diff_status); + self.update_insert_by_split(&leaf_indexes); + } + + fn forward(&mut self, span: loro_common::IdSpan, updates: &mut Vec) { + for c in self.id_to_cursor.iter(span) { + match c { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: Some(false), + delete_times_diff: 0, + }) + } + id_to_cursor::IterCursor::Delete(span) => { + for to_del in self.id_to_cursor.iter(span) { + match to_del { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: 1, + }) + } + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf, + new_op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to_leaf, + id_span: new_op_id.to_span(1), + set_future: None, + delete_times_diff: 1, + }), + _ => unreachable!(), + } + } + } + id_to_cursor::IterCursor::Move { + from_id: from, + to_leaf: to, + new_op_id: op_id, + } => { + for to_del in self.id_to_cursor.iter(IdSpan::new( + from.peer, + from.counter, + from.counter + 1, + )) { + match to_del { + id_to_cursor::IterCursor::Move { + from_id: _, + to_leaf: to, + new_op_id: op_id, + } => updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: op_id.to_span(1), + set_future: None, + delete_times_diff: 1, + }), + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + updates.push(crdt_rope::LeafUpdate { + leaf, + id_span, + set_future: None, + delete_times_diff: 1, + }) + } + _ => unreachable!(), + } + } + + updates.push(crdt_rope::LeafUpdate { + leaf: to, + id_span: IdSpan::new(op_id.peer, op_id.counter, op_id.counter + 1), + set_future: Some(false), + delete_times_diff: 0, + }); + } + } + } + } + + #[allow(unused)] + pub(crate) fn check(&self) { + if !cfg!(debug_assertions) { + return; + } + + self.check_vv_correctness(); + self.check_id_to_cursor_insertions_correctness(); + } + + fn check_vv_correctness(&self) { + if !cfg!(debug_assertions) { + return; + } + + for span in self.rope.tree().iter() { + if span.id.peer == UNKNOWN_PEER_ID { + continue; + } + + let id_span = span.id_span(); + assert!(self.all_vv().includes_id(id_span.id_last())); + if span.status.future { + assert!(!self.current_vv.includes_id(id_span.id_start())); + } else { + assert!(self.current_vv.includes_id(id_span.id_last())); + } + } + } + + // It can only check the correctness of insertions in id_to_cursor. + // The deletions are not checked. + fn check_id_to_cursor_insertions_correctness(&self) { + if !cfg!(debug_assertions) { + return; + } + + for rope_elem in self.rope.tree().iter() { + let id_span = rope_elem.id_span(); + let leaf_from_start = self.id_to_cursor.get_insert(id_span.id_start()).unwrap(); + let leaf_from_last = self.id_to_cursor.get_insert(id_span.id_last()).unwrap(); + assert_eq!(leaf_from_start, leaf_from_last); + let elem_from_id_to_cursor_map = self.rope.tree().get_elem(leaf_from_last).unwrap(); + assert_eq!(rope_elem, elem_from_id_to_cursor_map); + } + + for content in self.id_to_cursor.iter_all() { + match content { + id_to_cursor::IterCursor::Insert { leaf, id_span } => { + let leaf = self.rope.tree().get_elem(leaf).unwrap(); + let span = leaf.id_span(); + span.contains(id_span.id_start()); + span.contains(id_span.id_last()); + } + id_to_cursor::IterCursor::Delete(_) => {} + id_to_cursor::IterCursor::Move { .. } => {} + } + } + } + + pub(crate) fn get_target_id_latest_index_at_new_version( + &self, + id: ID, + ) -> Option { + // TODO: PERF this can be sped up from O(n) to O(log(n)) but I'm not sure if it's worth it + let mut index = 0; + for span in self.rope.tree.iter() { + let is_activated = span.is_activated_in_diff(); + let span_id = span.real_id(); + let id_span = span_id.to_span(span.rle_len()); + if id_span.contains(id) { + if is_activated { + index += (id.counter - id_span.counter.start) as usize; + } + + return Some(AbsolutePosition { + pos: index, + side: if is_activated { + crate::cursor::Side::Middle + } else { + crate::cursor::Side::Left + }, + }); + } + + if is_activated { + index += span.rle_len(); + } + } + + None + } + + // #[tracing::instrument(skip(self), level = "info")] + pub(crate) fn diff( + &mut self, + from: &VersionVector, + to: &VersionVector, + ) -> impl Iterator + '_ { + // tracing::info!("Init: {:#?}, ", &self); + self._checkout(from, false); + self._checkout(to, true); + // self.id_to_cursor.diagnose(); + // tracing::trace!("Trace::diff {:#?}, ", &self); + + self.rope.get_diff() + } +} + +#[cfg(test)] +mod test { + use crate::{container::richtext::RichtextChunk, vv}; + use generic_btree::rle::HasLength; + + use super::*; + use std::time::Instant; + + #[test] + fn test_len() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..2)); + assert_eq!(t.rope.len(), 2); + t.checkout(&Default::default()); + assert_eq!(t.rope.len(), 0); + t.insert(IdFull::new(2, 0, 0), 0, RichtextChunk::new_text(2..4)); + let v = vv!(1 => 2, 2 => 2); + t.checkout(&v); + assert_eq!(&t.applied_vv, &v); + assert_eq!(t.rope.len(), 4); + } + + #[test] + fn test_retreat_and_forward_delete() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..10)); + t.delete(ID::new(2, 0), ID::NONE_ID, 0, 10, true); + t.checkout(&vv!(1 => 10, 2=>5)); + assert_eq!(t.rope.len(), 5); + t.checkout(&vv!(1 => 10, 2=>0)); + assert_eq!(t.rope.len(), 10); + t.checkout(&vv!(1 => 10, 2=>10)); + assert_eq!(t.rope.len(), 0); + t.checkout(&vv!(1 => 10, 2=>0)); + assert_eq!(t.rope.len(), 10); + } + + #[test] + fn repeated_tail_splits_keep_id_to_cursor_consistent() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..300)); + + for (i, pos) in [100, 201, 252, 278].into_iter().enumerate() { + let op_id = IdFull::new(2, i as Counter, i as Lamport); + let start = 1000 + i as u32; + t.insert(op_id, pos, RichtextChunk::new_text(start..start + 1)); + } + + t.check(); + } + + #[test] + fn test_checkout_in_doc_with_del_span() { + let mut t = Tracker::new(); + t.insert(IdFull::new(1, 0, 0), 0, RichtextChunk::new_text(0..10)); + t.delete(ID::new(2, 0), ID::NONE_ID, 0, 10, false); + t.checkout(&vv!(1 => 10, 2=>4)); + let v: Vec = t.rope.tree().iter().copied().collect(); + assert_eq!(v.len(), 2); + assert!(!v[0].is_activated()); + assert_eq!(v[0].rle_len(), 4); + assert!(v[1].is_activated()); + assert_eq!(v[1].rle_len(), 6); + } + + #[test] + #[ignore] + fn perf_update_insert_by_split_quadratic() { + // Run with: + // cargo test -p loro-internal perf_update_insert_by_split_quadratic -- --ignored --nocapture + const CHUNK_LEN: usize = 256; + let fragments: usize = std::env::var("LORO_PERF_FRAGMENTS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192); + const PEER_A: PeerID = 1; + const PEER_B: PeerID = 2; + + let doc_len = CHUNK_LEN * fragments; + + let mut t = Tracker::new(); + t.insert( + IdFull::new(PEER_A, 0, 0), + 0, + RichtextChunk::new_text(0..doc_len as u32), + ); + t.id_to_cursor.diagnose(); + + let start = Instant::now(); + let expected_fragment_updates = (fragments as u64) * ((fragments - 1) as u64) / 2; + + for i in 0..(fragments - 1) { + let pos = (i + 1) * CHUNK_LEN + i; + let op_id = IdFull::new(PEER_B, i as Counter, i as Lamport); + let chunk = RichtextChunk::new_text( + (doc_len as u32 + i as u32)..(doc_len as u32 + i as u32 + 1), + ); + t.insert(op_id, pos, chunk); + } + + let elapsed = start.elapsed(); + let before_vv = vv!(PEER_A => doc_len as Counter); + let after_vv = vv!(PEER_A => doc_len as Counter, PEER_B => (fragments - 1) as Counter); + let diff_start = Instant::now(); + let diff_len = t.diff(&before_vv, &after_vv).count(); + let diff_elapsed = diff_start.elapsed(); + assert_eq!(t.rope.tree().iter().count(), 1 + 2 * (fragments - 1)); + println!( + "perf_update_insert_by_split_quadratic: doc_len={}, fragments={}, expected_fragment_updates={}, insert_elapsed={:?}, diff_items={}, diff_elapsed={:?}", + doc_len, fragments, expected_fragment_updates, elapsed, diff_len, diff_elapsed + ); + } + + #[test] + #[ignore] + fn perf_update_insert_by_split_quadratic_unknown() { + // Run with: + // LORO_PERF_FRAGMENTS=8192 cargo test -p loro-internal perf_update_insert_by_split_quadratic_unknown -- --ignored --nocapture + const CHUNK_LEN: usize = 256; + let fragments: usize = std::env::var("LORO_PERF_FRAGMENTS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(8192); + const PEER_B: PeerID = 2; + + let doc_len = CHUNK_LEN * fragments; + + let mut t = Tracker::new_with_unknown(); + t.checkout(&vv!()); + t.id_to_cursor.diagnose(); + + let start = Instant::now(); + for i in 0..(fragments - 1) { + let pos = (i + 1) * CHUNK_LEN + i; + let op_id = IdFull::new(PEER_B, i as Counter, i as Lamport); + let chunk = RichtextChunk::new_text( + (doc_len as u32 + i as u32)..(doc_len as u32 + i as u32 + 1), + ); + t.insert(op_id, pos, chunk); + } + + let elapsed = start.elapsed(); + println!( + "perf_update_insert_by_split_quadratic_unknown: doc_len={}, fragments={}, elapsed={:?}", + doc_len, fragments, elapsed + ); + } +} diff --git a/vendor/loro-internal/src/container/richtext/tracker/crdt_rope.rs b/vendor/loro-internal/src/container/richtext/tracker/crdt_rope.rs new file mode 100644 index 00000000..112a9c4c --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/tracker/crdt_rope.rs @@ -0,0 +1,1109 @@ +use std::cmp::Ordering; + +use generic_btree::{ + rle::{CanRemove, HasLength, Sliceable}, + BTree, BTreeTrait, Cursor, FindResult, LeafIndex, Query, SplittedLeaves, +}; +use itertools::Itertools; +use loro_common::{Counter, HasCounter, HasCounterSpan, HasIdSpan, IdSpan, Lamport, ID}; +use smallvec::{smallvec, SmallVec}; + +use crate::container::richtext::{fugue_span::DiffStatus, FugueSpan, RichtextChunk, Status}; + +use super::UNKNOWN_PEER_ID; + +#[derive(Debug, Default, Clone)] +pub(super) struct CrdtRope { + pub(super) tree: BTree, +} + +pub(super) struct InsertResult { + #[allow(unused)] + pub content: FugueSpan, + pub leaf: LeafIndex, + pub splitted: SplittedLeaves, +} + +impl CrdtRope { + pub fn new() -> Self { + Self { tree: BTree::new() } + } + + #[inline(always)] + #[allow(unused)] + pub fn len(&self) -> usize { + self.tree.root_cache().len as usize + } + + #[inline(always)] + pub(super) fn tree(&self) -> &BTree { + &self.tree + } + + pub(super) fn insert( + &mut self, + pos: usize, + mut content: FugueSpan, + find_elem: impl Fn(ID) -> LeafIndex, + ) -> InsertResult { + if self.tree.is_empty() { + assert_eq!(pos, 0); + let leaf = self.tree.push(content).leaf; + return InsertResult { + content, + leaf, + splitted: SplittedLeaves::default(), + }; + } + + // tracing::span!(tracing::Level::INFO, "Inserting {} len={}", content.id, content.rle_len()); + + let pos = pos as i32; + let start = self.tree.query::(&pos).unwrap(); + + let (parent_right_leaf, in_between, lazy_tail_from) = { + // calculate origin_left and origin_right + // origin_left is the alive op at `pos-1`, origin_right is the first non-future op between `pos-1` and `pos`. + + // `start` may point to a zero len node that's before the active index. + let origin_left = if start.cursor.offset == 0 { + // get left leaf node if offset == 0, so we can calculate the origin_left + if let Some(left) = self.tree.prev_elem(start.cursor) { + let left_node = self.tree.get_leaf(left.leaf.into()); + assert!(left_node.elem().rle_len() > 0); + Some( + left_node + .elem() + .id + .inc(left_node.elem().rle_len() as Counter - 1) + .id(), + ) + } else { + None + } + } else { + let left_node = self.tree.get_leaf(start.leaf().into()); + assert!(left_node.elem().rle_len() >= start.offset()); + Some(left_node.elem().id.inc(start.offset() as Counter - 1).id()) + }; + + let (origin_right, parent_right_leaf, in_between, lazy_tail_from) = { + // §flowstate vendor patch (bimodal-undo fix): upstream scanned + // linearly to the first NON-FUTURE element, collecting every + // future element on the way — O(future-run) per insert, which + // goes O(ops²) when a tracker rebuild retreats a large + // concurrent branch and every insert faces a document-sized + // future run (the 80-195s dirty-history undo regime). Probe a + // BOUNDED prefix (identical to upstream within the bound); + // past it, jump the run in O(log n) via the `non_future_len` + // cache, and let the Fugue walk below stream the rest of the + // run LAZILY (it almost always breaks within a few elements). + fn future_probe_limit() -> usize { + // Kill switch: restores upstream's unbounded linear scan + // (for A/B attribution; the guard test flips quadratic). + static LIMIT: std::sync::OnceLock = std::sync::OnceLock::new(); + *LIMIT.get_or_init(|| { + if std::env::var_os("FLOWSTATE_DISABLE_FUTURE_JUMP").is_some() { + usize::MAX + } else { + 32 + } + }) + } + let mut in_between = Vec::new(); + let mut origin_right = None; + let mut parent_right_idx = None; + let mut lazy_tail_from = None; + let mut overflowed = false; + for iter in self.tree.iter_range(start.cursor..) { + if let Some(offset) = iter.start { + if offset >= iter.elem.rle_len() { + continue; + } + } + + if !iter.elem.status.future { + origin_right = + Some(iter.elem.id.inc(iter.start.unwrap_or(0) as Counter).id()); + let parent_right = match iter.start { + Some(offset) if offset > 0 => { + // It's guaranteed that origin_right's origin_left == this.origin_left. + // Because the first non-future node is just the leaf node in `start.cursor` node (iter.start.is_some()) + // Thus parent_right = origin_right + Some(origin_right) + } + _ => { + // Otherwise, we need to test whether origin_right's origin_left == this.origin_left + if iter.elem.origin_left.map(|x| x.to_id()) == origin_left { + Some(origin_right) + } else { + None + } + } + }; + parent_right_idx = parent_right.map(|_| iter.cursor().leaf); + break; + } + + if in_between.len() >= future_probe_limit() { + overflowed = true; + break; + } + + // elem must be from future + in_between.push((iter.cursor().leaf, *iter.elem)); + } + + { + // §flowstate probe (FLOWSTATE_FUTURE_RUN_PROBE=1): count + // probe overflows + total probed future elems, to verify + // the accelerated path engages on the pathological shape. + static PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + if *PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_FUTURE_RUN_PROBE").is_some()) { + use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; + static INSERTS: AtomicU64 = AtomicU64::new(0); + static PROBED: AtomicU64 = AtomicU64::new(0); + static OVERFLOWS: AtomicU64 = AtomicU64::new(0); + let inserts = INSERTS.fetch_add(1, AtomicOrdering::Relaxed) + 1; + let probed = PROBED.fetch_add(in_between.len() as u64, AtomicOrdering::Relaxed) + in_between.len() as u64; + let overflows = OVERFLOWS.fetch_add(u64::from(overflowed), AtomicOrdering::Relaxed) + u64::from(overflowed); + if inserts.is_multiple_of(4096) || (overflowed && overflows.is_multiple_of(1024)) { + eprintln!("future-run-probe: inserts={inserts} probed_elems={probed} overflows={overflows}"); + } + } + } + if overflowed { + let (from_leaf, from_elem) = *in_between.last().unwrap(); + if let Some(leaf_idx) = self.tree.next_leaf_matching( + Cursor { + leaf: from_leaf, + offset: 0, + }, + &mut |cache: &Cache| cache.non_future_len > 0, + ) { + let elem = self.tree.get_leaf(leaf_idx.into()).elem(); + debug_assert!(!elem.status.future); + // A mid-range element always enters the upstream loop + // with `iter.start == None` (only the cursor's own + // leaf carries an offset, and the probe consumed it), + // so `inc(0)` and the origin_left-equality branch + // mirror upstream exactly. + let or = elem.id.id(); + origin_right = Some(or); + parent_right_idx = if elem.origin_left.map(|x| x.to_id()) == origin_left { + Some(leaf_idx) + } else { + None + }; + } + // The Fugue walk streams the rest of the future run from + // after the last probed element (whether or not a + // non-future element exists to its right). + lazy_tail_from = Some((from_leaf, from_elem)); + + // §debug (FLOWSTATE_FUTURE_JUMP_VERIFY=1): re-run the + // upstream unbounded scan and compare the jump's answers. + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + if *VERIFY.get_or_init(|| std::env::var_os("FLOWSTATE_FUTURE_JUMP_VERIFY").is_some()) { + let mut ref_origin_right = None; + let mut ref_parent_right_idx = None; + let mut ref_run_len = 0usize; + for iter in self.tree.iter_range(start.cursor..) { + if let Some(offset) = iter.start { + if offset >= iter.elem.rle_len() { + continue; + } + } + if !iter.elem.status.future { + ref_origin_right = + Some(iter.elem.id.inc(iter.start.unwrap_or(0) as Counter).id()); + let parent_right = match iter.start { + Some(offset) if offset > 0 => Some(ref_origin_right), + _ => { + if iter.elem.origin_left.map(|x| x.to_id()) == origin_left { + Some(ref_origin_right) + } else { + None + } + } + }; + ref_parent_right_idx = parent_right.map(|_| iter.cursor().leaf); + break; + } + ref_run_len += 1; + } + if (origin_right, parent_right_idx) != (ref_origin_right, ref_parent_right_idx) { + eprintln!("future-jump verify MISMATCH: run_len={ref_run_len}"); + eprintln!(" jump: origin_right={origin_right:?} parent_right={parent_right_idx:?}"); + eprintln!(" linear: origin_right={ref_origin_right:?} parent_right={ref_parent_right_idx:?}"); + // Dump the tree neighborhood: every elem from the cursor + // with id/status, until past the linear answer. + let mut dumped = 0; + for iter in self.tree.iter_range(start.cursor..) { + let e = iter.elem; + eprintln!( + " elem leaf={:?} start={:?} id={:?} len={} future={} del={} ", + iter.cursor().leaf, iter.start, e.id, e.rle_len(), e.status.future, e.status.delete_times + ); + dumped += 1; + if (!e.status.future && dumped > ref_run_len) || dumped > ref_run_len + 8 { + break; + } + } + // Climb from the linear answer's leaf, dumping every + // ancestor level's child caches — shows where the + // `non_future_len` pred loses the subtree. + if let Some(answer_leaf) = ref_parent_right_idx { + self.tree.debug_dump_ancestor_caches(answer_leaf, &mut |cache: &Cache| { + format!("len={} changed={} non_future={}", cache.len, cache.changed_num, cache.non_future_len) + }); + } + panic!("future-jump verify: jump answers diverge from the linear scan"); + } + } + } + + (origin_right, parent_right_idx, in_between, lazy_tail_from) + }; + + content.origin_left = origin_left.map(|x| x.try_into().unwrap()); + content.origin_right = origin_right.map(|x| x.try_into().unwrap()); + (parent_right_leaf, in_between, lazy_tail_from) + }; + + let mut insert_pos = start.cursor; + + if !in_between.is_empty() { + // find insert pos + let mut scanning = false; + let mut visited: SmallVec<[IdSpan; 4]> = Default::default(); + // §flowstate vendor patch: the probed prefix chained with a LAZY + // stream of the remaining future run (upstream pre-collected the + // whole run; the walk visits only what its break conditions need). + let tail = lazy_tail_from.into_iter().flat_map(|(leaf, elem)| { + let resume = Cursor { + leaf, + offset: elem.rle_len(), + }; + self.tree + .iter_range(resume..) + .filter(|it| it.start.is_none_or(|offset| offset < it.elem.rle_len())) + .take_while(|it| it.elem.status.future) + .map(|it| (it.cursor().leaf, *it.elem)) + }); + for (other_leaf, other_elem) in in_between.iter().copied().chain(tail) { + // tracing::info!("Visiting {}", &other_elem.id); + let other_origin_left = other_elem.origin_left; + if other_origin_left != content.origin_left + && other_origin_left + .map(|left| visited.iter().all(|x| !x.contains_id(left.to_id()))) + .unwrap_or(true) + { + // The other_elem's origin_left must be at the left side of content's origin_left. + // So the content must be at the left side of other_elem. + + // tracing::info!("Break because the node's origin_left is at the left side of new_elem's origin left"); + break; + } + + visited.push(IdSpan::new( + other_elem.id.peer, + other_elem.id.counter, + other_elem.id.counter + other_elem.rle_len() as Counter, + )); + + if content.origin_left == other_origin_left { + if other_elem.origin_right == content.origin_right { + // tracing::info!("Same right parent"); + // Same right parent + if other_elem.id.peer > content.id.peer { + // tracing::info!("Break on larger peer"); + break; + } else { + scanning = false; + } + } else { + // tracing::info!("Different right parent"); + // Different right parent, we need to compare the right parents' position + + let other_parent_right_idx = + if let Some(other_origin_right) = other_elem.origin_right { + let elem_idx = find_elem(other_origin_right.to_id()); + let elem = self.tree.get_elem(elem_idx).unwrap(); + // It must be the start of the elem + assert_eq!(elem.id.id(), other_origin_right.to_id()); + if elem.origin_left == content.origin_left { + Some(elem_idx) + } else { + None + } + } else { + None + }; + + match self.cmp_pos(other_parent_right_idx, parent_right_leaf) { + Ordering::Less => { + // tracing::info!("Less"); + scanning = true; + } + Ordering::Equal if other_elem.id.peer > content.id.peer => { + // tracing::info!("Break on eq"); + break; + } + _ => { + // tracing::info!("Scanning"); + scanning = false; + } + } + } + } + + if !scanning { + insert_pos = Cursor { + leaf: other_leaf, + offset: other_elem.rle_len(), + }; + // tracing::info!("updating insert pos {:?}", &insert_pos); + } + } + } + + // tracing::info!("Inserting at {:?}", insert_pos); + // + let (cursor, splitted) = self.tree.insert_by_path(insert_pos, content); + InsertResult { + content, + leaf: cursor.leaf, + splitted, + } + } + + /// Delete the span at `pos` with `len`. + /// + /// The `start_id` is used to mark the real_id in the [FugueSpan]. + /// + /// - If reversed is true, the deletion will be done in reversed order. + /// But the start_id always refers to the first delete op's id. + /// - If reversed is true, the returned `SplittedLeaves` will be in reversed order. + pub(super) fn delete( + &mut self, + mut start_id: ID, + pos: usize, + len: usize, + reversed: bool, + notify_deleted_span: &mut dyn FnMut(&FugueSpan), + ) -> SmallVec<[SplittedLeaves; 1]> { + if len == 0 { + return Default::default(); + } + + if reversed && len > 1 { + // TODO: PERF: optimize this + let mut ans = SmallVec::with_capacity(len); + for i in (0..len).rev() { + let a = self.delete( + start_id.inc(i as i32), + pos + i, + 1, + false, + notify_deleted_span, + ); + + ans.extend(a); + } + + return ans; + } + + let start = self + .tree + .query::(&(pos as i32)) + .unwrap(); + // avoid pointing to the end of the node + let start = start.cursor; + let elem = self.tree.get_elem_mut(start.leaf).unwrap(); + if elem.rle_len() >= start.offset + len { + // tracing::info!("len={} offset={} l={} ", elem.rle_len(), start.offset, len,); + let (_, splitted) = self.tree.update_leaf(start.leaf, |elem| { + let (a, b) = elem.update_with_split(start.offset..start.offset + len, |elem| { + assert!(elem.is_activated()); + debug_assert_eq!(len, elem.rle_len()); + notify_deleted_span(elem); + elem.status.delete_times += 1; + if elem.real_id.is_none() { + elem.real_id = Some(start_id.try_into().unwrap()); + } + + start_id = start_id.inc(elem.rle_len() as i32); + }); + + (true, a, b) + }); + + return smallvec![splitted]; + } + + let end = self + .tree + .query::(&((pos + len) as i32)) + .unwrap(); + smallvec![self.tree.update(start..end.cursor(), &mut |elem| { + if elem.is_activated() { + notify_deleted_span(elem); + elem.status.delete_times += 1; + if elem.real_id.is_none() { + elem.real_id = Some(start_id.try_into().unwrap()); + } + + start_id = start_id.inc(elem.rle_len() as i32); + Some(Cache { + len: -(elem.rle_len() as i32), + changed_num: 0, + // Deletion flips `delete_times`, never `future`. + non_future_len: 0, + }) + } else { + None + } + })] + } + + #[allow(unused)] + pub(crate) fn diagnose(&self) { + println!("crdt_rope number of tree nodes = {}", self.tree.node_len()); + } + + /// Update the leaf with given `id_span` + /// + /// Return the new leaf indexes that are created by splitting the old leaf nodes + pub(super) fn update( + &mut self, + mut updates: Vec, + on_diff_status: bool, + ) -> Vec { + updates.sort_by_key(|x| x.leaf); + let mut tree_update_info = Vec::with_capacity(updates.len()); + for (leaf, group) in &updates.into_iter().group_by(|x| x.leaf) { + let elem = self.tree.get_elem(leaf).unwrap(); + for u in group { + debug_assert_eq!(u.id_span.peer, elem.id.peer); + let start = (u.id_span.ctr_start() - elem.id.counter).max(0); + let end = (u.id_span.ctr_end() - elem.id.counter).max(0); + let len = elem.rle_len(); + + tree_update_info.push((leaf, (start as usize).min(len)..(end as usize).min(len), u)) + } + } + + self.tree + .update_leaves_with_arg_in_ranges(tree_update_info, |elem, arg| { + let status = if on_diff_status { + if elem.diff_status.is_none() { + elem.diff_status = Some(elem.status); + } + elem.diff_status.as_mut().unwrap() + } else { + &mut elem.status + }; + + if let Some(f) = arg.set_future { + status.future = f; + } + + status.delete_times += arg.delete_times_diff; + }) + } + + pub(super) fn clear_diff_status(&mut self) { + self.tree.update_cache_and_elem_with_filter( + |cache| { + let drill = cache.changed_num > 0; + cache.changed_num = 0; + drill + }, + |elem| { + elem.diff_status = None; + }, + ); + } + + pub(super) fn get_diff(&self) -> impl Iterator + '_ { + let mut last_pos = 0; + let mut iter = self + .tree + .iter_with_filter(|cache| (cache.changed_num > 0, cache.len)); + let mut next = None; + std::iter::from_fn(move || { + if let Some(next) = next.take() { + return Some(next); + } + + #[allow(clippy::while_let_on_iterator)] + while let Some((index, elem)) = iter.next() { + // The elements will not be changed by this method. + // This index is current index of the elem (calculated by `status` field rather than `diff_status` field) + match elem.diff() { + DiffStatus::NotChanged => {} + DiffStatus::Created => { + let rt = Some(CrdtRopeDelta::Insert { + chunk: elem.content, + id: elem.real_id.map(|x| x.to_id()).unwrap(), + lamport: if elem.id.peer == UNKNOWN_PEER_ID { + None + } else { + Some(elem.id.lamport) + }, + }); + if index > last_pos { + next = rt; + let len = index - last_pos; + // last pos = index, because the creation has not been applied to the elem + last_pos = index; + return Some(CrdtRopeDelta::Retain(len as usize)); + } else { + return rt; + } + } + DiffStatus::Deleted => { + let rt = Some(CrdtRopeDelta::Delete(elem.rle_len())); + if index > last_pos { + next = rt; + let len = index - last_pos; + // last pos = index + len, because the deletion has not been applied to the elem + last_pos = index + elem.rle_len() as i32; + return Some(CrdtRopeDelta::Retain(len as usize)); + } else { + last_pos = index + elem.rle_len() as i32; + return rt; + } + } + } + } + + None + }) + } + + fn cmp_pos( + &self, + parent_right: Option, + other_parent_right: Option, + ) -> Ordering { + match (parent_right, other_parent_right) { + (Some(a), Some(b)) => self + .tree + .compare_pos(Cursor { leaf: a, offset: 0 }, Cursor { leaf: b, offset: 0 }), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(crate) enum CrdtRopeDelta { + Retain(usize), + Insert { + chunk: RichtextChunk, + id: ID, + /// This is a optional field, because we may not know the correct lamport + /// for chunk id with UNKNOWN_PEER_ID. + /// + /// This case happens when the chunk is created by default placeholder and + /// the deletion happens that marks the chunk with its start_id. But it doesn't + /// know the correct lamport for the chunk. + lamport: Option, + }, + Delete(usize), +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct CrdtRopeTrait; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Copy)] +pub(super) struct Cache { + pub(super) len: i32, + pub(super) changed_num: i32, + /// §flowstate patch (bimodal-undo fix): total `rle_len` of the subtree's + /// NON-`future` elements. `CrdtRope::insert` must find the first + /// non-future element to the right of every fed op (its `origin_right`); + /// this lets `next_leaf_matching` skip whole future subtrees in O(log n) + /// where the upstream linear scan went O(ops²) whenever a tracker rebuild + /// retreated a large concurrent branch (the 80-195s dirty-history undo + /// regime). A LENGTH, not an element count, because generic-btree + /// propagates split accounting assuming elem caches are ADDITIVE UNDER + /// SLICING (`split_leaf_if_needed` recomputes the halves but the ancestor + /// diff only carries the inserted elem) — an element count drifts negative + /// on splits (caught by the fuzz soak as a projection divergence); lengths + /// sum exactly. Every live elem has `rle_len ≥ 1` (zero-len spans are + /// `can_remove`), so `> 0` still means "contains a non-future element". + pub(super) non_future_len: i32, +} + +impl CanRemove for Cache { + fn can_remove(&self) -> bool { + self.len == 0 && self.changed_num == 0 && self.non_future_len == 0 + } +} + +impl BTreeTrait for CrdtRopeTrait { + type Elem = FugueSpan; + type Cache = Cache; + type CacheDiff = Cache; + const USE_DIFF: bool = true; + + #[inline(always)] + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new_len = caches.iter().map(|x| x.cache.len).sum(); + let new_changed_num = caches.iter().map(|x| x.cache.changed_num).sum(); + let new_non_future_len = caches.iter().map(|x| x.cache.non_future_len).sum(); + let len_diff = new_len - cache.len; + let changed_num_diff = new_changed_num - cache.changed_num; + let non_future_len_diff = new_non_future_len - cache.non_future_len; + cache.len = new_len; + cache.changed_num = new_changed_num; + cache.non_future_len = new_non_future_len; + Cache { + len: len_diff, + changed_num: changed_num_diff, + non_future_len: non_future_len_diff, + } + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + cache.len += diff.len; + cache.changed_num += diff.changed_num; + cache.non_future_len += diff.non_future_len; + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + diff1.len += diff2.len; + diff1.changed_num += diff2.changed_num; + diff1.non_future_len += diff2.non_future_len; + } + + #[inline(always)] + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + Cache { + len: elem.activated_len() as i32, + changed_num: if elem.diff_status.is_some() { 1 } else { 0 }, + non_future_len: if elem.status.future { 0 } else { elem.rle_len() as i32 }, + } + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + Cache { + len: cache_lhs.len - cache_rhs.len, + changed_num: cache_lhs.changed_num - cache_rhs.changed_num, + non_future_len: cache_lhs.non_future_len - cache_rhs.non_future_len, + } + } +} + +/// Query for start position, prefer left. +/// +/// If there are zero length spans (deleted, or spans from future) at the +/// active index, the query will return the position of the first non-zero length +/// content before them. +/// +/// NOTE: it may points to the end of a leaf node (with offset = rle_len) while the next leaf node is available. +struct ActiveLenQueryPreferLeft { + left: i32, +} + +impl Query for ActiveLenQueryPreferLeft { + type QueryArg = i32; + + fn init(target: &Self::QueryArg) -> Self { + debug_assert!(*target >= 0); + ActiveLenQueryPreferLeft { left: *target } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut left = self.left; + for (i, child) in child_caches.iter().enumerate() { + let cache = &child.cache; + if left <= cache.len { + // Prefer left. So if both `cache.len` and `left` equal zero, return here. + self.left = left; + return generic_btree::FindResult::new_found(i, left as usize); + } + + left -= cache.len; + } + + if let Some(last) = child_caches.last() { + left += last.cache.len; + self.left = left; + FindResult::new_missing(child_caches.len() - 1, left as usize) + } else { + // TODO: this should be impossible + unreachable!() + } + } + + fn confirm_elem( + &mut self, + _: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + if elem.is_activated() { + (self.left as usize, (self.left as usize) < elem.rle_len()) + } else { + // prefer left on zero length spans + (0, self.left == 0) + } + } +} + +/// Query for start position, prefer right +/// +/// If there are zero length spans (deleted, or spans from future) at the +/// active index, the query will return the position of the first non-zero length +/// content after them. +struct ActiveLenQueryPreferRight { + left: i32, +} + +impl Query for ActiveLenQueryPreferRight { + type QueryArg = i32; + + fn init(target: &Self::QueryArg) -> Self { + debug_assert!(*target >= 0); + Self { left: *target } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut left = self.left; + for (i, child) in child_caches.iter().enumerate() { + let cache = &child.cache; + if left < cache.len { + // Prefer left. So if both `cache.len` and `left` equal zero, return here. + self.left = left; + return generic_btree::FindResult::new_found(i, left as usize); + } + + left -= cache.len; + } + + if let Some(last) = child_caches.last() { + left += last.cache.len; + self.left = left; + FindResult::new_missing(child_caches.len() - 1, left as usize) + } else { + // TODO: this should be impossible + unreachable!() + } + } + + fn confirm_elem( + &mut self, + _: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + if elem.is_activated() { + (self.left as usize, (self.left as usize) < elem.rle_len()) + } else { + (self.left as usize, self.left == 0) + } + } +} + +/// This struct describe the ways to update a leaf's status +#[derive(Clone, Debug, Copy)] +pub(super) struct LeafUpdate { + pub leaf: LeafIndex, + /// `id_span` should only contains a subset of the leaf content + pub id_span: IdSpan, + /// if `set_future` is `None`, the `future` field will not be changed + pub set_future: Option, + pub delete_times_diff: i16, +} + +impl LeafUpdate { + #[allow(unused)] + fn apply_to(&self, s: &mut Status) { + s.delete_times += self.delete_times_diff; + if let Some(f) = self.set_future { + s.future = f; + } + } +} + +#[cfg(test)] +mod test { + use std::ops::Range; + + use loro_common::{CompactId, Counter, IdFull, PeerID, ID}; + + use crate::container::richtext::RichtextChunk; + + use super::*; + + fn span(id: u32, range: Range) -> FugueSpan { + FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_text(range), + ) + } + + #[allow(unused)] + fn unknown_span(id: u32, len: usize) -> FugueSpan { + FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_unknown(len as u32), + ) + } + + fn future_span(id: u32, range: Range) -> FugueSpan { + let mut fugue = FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_text(range), + ); + + fugue.status.future = true; + fugue + } + + fn dead_span(id: u32, range: Range) -> FugueSpan { + let mut span = FugueSpan::new( + IdFull::new(id as PeerID, 0 as Counter, 0), + RichtextChunk::new_text(range), + ); + + span.status.delete_times += 1; + span + } + + #[test] + fn len_test() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + assert_eq!(rope.len(), 10); + rope.insert(5, span(1, 0..10), |_| panic!()); + assert_eq!(rope.len(), 20); + rope.insert(20, span(1, 0..10), |_| panic!()); + assert_eq!(rope.len(), 30); + for i in 3..30 { + assert_eq!(rope.len(), i * 10); + rope.insert( + i, + span(i as u32, i as u32 * 10..(i as u32 + 1) * 10), + |_| panic!(), + ); + } + } + + #[test] + fn content_insert_middle() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + rope.insert(5, span(1, 10..20), |_| panic!()); + let arr: Vec<_> = rope.tree.iter().collect(); + assert_eq!(arr.len(), 3); + assert_eq!(arr[0].rle_len(), 5); + assert_eq!(arr[1].rle_len(), 10); + assert_eq!(arr[2].rle_len(), 5); + + assert_eq!(arr[0].id.counter, 0); + assert_eq!(arr[0].id.peer, 0); + assert_eq!(arr[1].id.counter, 0); + assert_eq!(arr[1].id.peer, 1); + assert_eq!(arr[2].id.counter, 5); + assert_eq!(arr[2].id.peer, 0); + } + + #[test] + fn content_insert_should_ignore_tombstone() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + // 0..10 + + rope.insert(5, dead_span(1, 10..20), |_| panic!()); + // 0..5, 10..20(dead), 5..10 + + rope.insert(10, span(0, 20..30), |_| panic!()); + // 0..5, 10..20(dead), 5..10, 20..30 + + let arr: Vec<_> = rope.tree.iter().collect(); + assert_eq!(arr.len(), 4); + assert_eq!(arr[0].rle_len(), 5); + assert_eq!(arr[1].rle_len(), 10); + assert_eq!(arr[1].activated_len(), 0); + assert_eq!(arr[2].rle_len(), 5); + assert_eq!(arr[3].rle_len(), 10); + } + + #[test] + fn get_origin_left_and_right() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + let fugue = rope.insert(5, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 5))); + } + + #[test] + fn get_origin_left_and_right_among_tombstones() { + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + assert_eq!(rope.len(), 10); + rope.delete(ID::NONE_ID, 5, 2, false, &mut |_| {}); + assert_eq!(rope.len(), 8); + let fugue = rope.insert(6, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 7))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 8))); + let fugue = rope.insert(5, span(1, 10..11), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 5))); + } + + #[test] + fn should_ignore_future_spans_when_getting_origin_left() { + { + // insert future + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + rope.insert(5, future_span(1, 10..20), |_| panic!()); + let fugue = rope.insert(5, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(0, 5))); + } + { + // insert deleted + let mut rope = CrdtRope::new(); + rope.insert(0, span(0, 0..10), |_| panic!()); + rope.insert(5, dead_span(1, 10..20), |_| panic!()); + let fugue = rope.insert(5, span(1, 10..20), |_| panic!()).content; + assert_eq!(fugue.origin_left, Some(CompactId::new(0, 4))); + assert_eq!(fugue.origin_right, Some(CompactId::new(1, 0))); + } + } + + #[test] + fn update() { + let mut rope = CrdtRope::new(); + let result = rope.insert(0, span(0, 0..10), |_| panic!()); + let split = rope.update( + vec![LeafUpdate { + leaf: result.leaf, + id_span: IdSpan::new(0, 2, 8), + set_future: None, + delete_times_diff: 1, + }], + false, + ); + + assert_eq!(rope.len(), 4); + assert_eq!(split.len(), 2); + let split = rope.update( + vec![LeafUpdate { + leaf: split[0], + id_span: IdSpan::new(0, 2, 8), + set_future: None, + delete_times_diff: -1, + }], + false, + ); + + assert_eq!(rope.len(), 10); + assert_eq!(split.len(), 0); + } + + #[test] + fn checkout() { + let mut rope = CrdtRope::new(); + let result1 = rope.insert(0, span(0, 0..10), |_| panic!()); + let result2 = rope.insert(10, dead_span(1, 10..20), |_| panic!()); + rope.update( + vec![ + LeafUpdate { + leaf: result1.leaf, + id_span: IdSpan::new(0, 2, 8), + set_future: None, + delete_times_diff: 1, + }, + LeafUpdate { + leaf: result2.leaf, + id_span: IdSpan::new(1, 0, 3), + set_future: None, + delete_times_diff: -1, + }, + ], + true, + ); + let vec: Vec<_> = rope.get_diff().collect(); + assert_eq!( + vec![ + CrdtRopeDelta::Retain(2), + CrdtRopeDelta::Delete(6), + CrdtRopeDelta::Retain(2), + CrdtRopeDelta::Insert { + chunk: RichtextChunk::new_text(10..13), + id: ID::new(1, 0), + lamport: Some(0) + } + ], + vec, + ); + } + + #[test] + fn checkout_future() { + let mut rope = CrdtRope::new(); + let result = rope.insert(0, future_span(0, 0..10), |_| panic!()); + rope.update( + vec![LeafUpdate { + leaf: result.leaf, + id_span: IdSpan::new(0, 2, 10), + set_future: Some(false), + delete_times_diff: 0, + }], + true, + ); + let vec: Vec<_> = rope.get_diff().collect(); + assert_eq!( + vec![CrdtRopeDelta::Insert { + chunk: RichtextChunk::new_text(2..10), + id: ID::new(0, 2), + lamport: Some(2) + }], + vec, + ); + } + + #[test] + fn checkout_future_with_delete() { + let mut rope = CrdtRope::new(); + let result = rope.insert(0, future_span(0, 0..10), |_| panic!()); + rope.update( + vec![LeafUpdate { + leaf: result.leaf, + id_span: IdSpan::new(0, 2, 10), + set_future: Some(false), + delete_times_diff: 1, + }], + true, + ); + let vec: Vec<_> = rope.get_diff().collect(); + assert!(vec.is_empty()); + } +} diff --git a/vendor/loro-internal/src/container/richtext/tracker/id_to_cursor.rs b/vendor/loro-internal/src/container/richtext/tracker/id_to_cursor.rs new file mode 100644 index 00000000..cf477a98 --- /dev/null +++ b/vendor/loro-internal/src/container/richtext/tracker/id_to_cursor.rs @@ -0,0 +1,1248 @@ +use generic_btree::{ + rle::{HasLength, Mergeable}, + LeafIndex, +}; +use loro_common::{Counter, IdSpan, PeerID, ID}; +use rle::{HasLength as RHasLength, Mergable as RMergeable, Sliceable}; +use rustc_hash::FxHashMap; +use smallvec::smallvec; + +use self::insert_set::InsertSet; + +// If we make this too large, we may have too many cursors inside a fragment +// and trigger the worst case +const MAX_FRAGMENT_LEN: usize = 256; +#[cfg(not(test))] +const SMALL_SET_MAX_LEN: usize = 32; +#[cfg(test)] +const SMALL_SET_MAX_LEN: usize = 4; + +/// This struct maintains the mapping of Op `ID` to +/// +/// - `LeafIndex` from crdt_rope, if the Op is an Insert +/// - The IdSpan deleted by the Op, if the Op is a Delete +#[derive(Debug, Default)] +pub(super) struct IdToCursor { + map: FxHashMap>, +} + +static EMPTY_VEC: Vec = vec![]; +impl IdToCursor { + pub fn insert_without_split(&mut self, id: ID, cursor: Cursor) { + let list = self.map.entry(id.peer).or_default(); + list.push(Fragment { + counter: id.counter, + cursor, + }); + } + + pub fn insert(&mut self, id: ID, cursor: Cursor) { + let list = self.map.entry(id.peer).or_default(); + if let Some(last) = list.last_mut() { + let last_end = last.counter + last.cursor.rle_len() as Counter; + debug_assert!(last_end <= id.counter, "id:{}, {:#?}", id, &self); + if last_end == id.counter + && last.cursor.can_merge(&cursor) + && last.cursor.rle_len() + cursor.rle_len() < MAX_FRAGMENT_LEN + { + last.cursor.merge_right(&cursor); + return; + } + } + + if let Cursor::Insert(InsertSet::Small(set)) = cursor { + if set.len > MAX_FRAGMENT_LEN as u32 { + assert!(set.set.len() == 1); + let insert = set.set[0]; + let mut counter = id.counter; + for start in (0..set.len).step_by(MAX_FRAGMENT_LEN) { + let end = (start + MAX_FRAGMENT_LEN as u32).min(set.len); + let len = (end - start) as usize; + list.push(Fragment { + counter, + cursor: Cursor::new_insert(insert.leaf, len), + }); + counter += len as Counter; + } + } else { + list.push(Fragment { + counter: id.counter, + cursor: Cursor::Insert(InsertSet::Small(set)), + }); + } + } else { + list.push(Fragment { + counter: id.counter, + cursor, + }); + } + } + + /// Update the given id_span to the new_leaf + /// + /// id_span should be within the same `Cursor` and should be a `Insert` + pub fn update_insert(&mut self, id_span: IdSpan, new_leaf: LeafIndex) { + debug_assert!(!id_span.is_reversed()); + let list = self.map.get_mut(&id_span.peer).unwrap(); + let last = list.last().unwrap(); + debug_assert!(last.counter + last.cursor.rle_len() as Counter > id_span.counter.max()); + let mut index = match list.binary_search_by_key(&id_span.counter.start, |x| x.counter) { + Ok(index) => index, + Err(index) => index.saturating_sub(1), + }; + + let mut start_counter = id_span.counter.start; + // A split of a large merged rope leaf redirects a WIDE counter range + // to one new leaf. Rewriting every fragment's insert-set in that range + // (the old behavior) made replaying a large linear history quadratic: + // each split walked ~all of the peer's fragments. Instead, update the + // partially-covered head/tail fragments in place and COLLAPSE the + // fully-covered middle run into a single fragment pointing at the new + // leaf — the list shrinks after every wide update, so the total work + // stays linear in the number of fragments ever created. + let mut collapse_lo: Option = None; + let mut collapse_hi = index; + while start_counter < id_span.counter.end + && index < list.len() + && start_counter < list[index].counter_end() + { + let fragment = &mut list[index]; + let from = (start_counter - fragment.counter) as usize; + let to = + ((id_span.counter.end - fragment.counter) as usize).min(fragment.cursor.rle_len()); + + if from == 0 && to == fragment.cursor.rle_len() && matches!(fragment.cursor, Cursor::Insert(_)) { + collapse_lo.get_or_insert(index); + collapse_hi = index + 1; + } else { + fragment.cursor.update_insert(from, to, new_leaf); + } + start_counter += (to - from) as Counter; + index += 1; + } + + assert_eq!(start_counter, id_span.counter.end); + if let Some(lo) = collapse_lo { + let counter = list[lo].counter; + let len = (list[collapse_hi - 1].counter_end() - counter) as usize; + list.splice( + lo..collapse_hi, + std::iter::once(Fragment { + counter, + cursor: Cursor::new_insert(new_leaf, len), + }), + ); + } + } + + pub fn update_insert_batch(&mut self, updates: &mut [(IdSpan, LeafIndex)]) { + if updates.is_empty() { + return; + } + + if updates.len() == 1 { + let (span, leaf) = updates[0]; + self.update_insert(span, leaf); + return; + } + + let mut per_peer: FxHashMap> = FxHashMap::default(); + for &(id_span, new_leaf) in updates.iter() { + per_peer + .entry(id_span.peer) + .or_default() + .push((id_span, new_leaf)); + } + + for (peer, peer_updates) in per_peer { + let Some(list) = self.map.get_mut(&peer) else { + continue; + }; + + let mut per_fragment: FxHashMap> = + FxHashMap::default(); + + for (id_span, new_leaf) in peer_updates { + debug_assert!(!id_span.is_reversed()); + let mut index = + match list.binary_search_by_key(&id_span.counter.start, |x| x.counter) { + Ok(index) => index, + Err(index) => index.saturating_sub(1), + }; + + let mut start_counter = id_span.counter.start; + while start_counter < id_span.counter.end + && index < list.len() + && start_counter < list[index].counter_end() + { + let fragment = &list[index]; + let from = (start_counter - fragment.counter) as usize; + let to = ((id_span.counter.end - fragment.counter) as usize) + .min(fragment.cursor.rle_len()); + + if from != to { + per_fragment + .entry(index) + .or_default() + .push((from, to, new_leaf)); + } + + start_counter += (to - from) as Counter; + index += 1; + } + + assert_eq!(start_counter, id_span.counter.end); + } + + let mut fragment_indexes: Vec = per_fragment.keys().copied().collect(); + fragment_indexes.sort_unstable(); + for index in fragment_indexes { + let updates = per_fragment.get(&index).unwrap(); + list[index].cursor.update_insert_many(updates); + } + } + } + + pub fn iter_all(&self) -> impl Iterator + '_ { + self.map.iter().flat_map(|(peer, list)| { + list.iter() + .flat_map(move |f| -> Box> { + match &f.cursor { + Cursor::Insert(set) => set.iter_all(*peer, f.counter), + Cursor::Delete(span) => { + let start_counter = f.counter; + let end_counter = f.counter + span.atom_len() as Counter; + let id_span = IdSpan::new(*peer, start_counter, end_counter); + Box::new(std::iter::once(IterCursor::Delete(id_span))) + } + Cursor::Move { from, to } => Box::new(std::iter::once(IterCursor::Move { + from_id: *from, + to_leaf: *to, + new_op_id: ID::new(*peer, f.counter), + })), + } + }) + }) + } + + pub fn iter(&self, mut iter_id_span: IdSpan) -> impl Iterator + '_ { + iter_id_span.normalize_(); + let list = self.map.get(&iter_id_span.peer).unwrap_or(&EMPTY_VEC); + // Index in the list + let mut index = 0; + let mut insert_set_iter: Option>> = None; + + if !list.is_empty() { + index = match list.binary_search_by_key(&iter_id_span.counter.start, |x| x.counter) { + Ok(index) => index, + Err(index) => index.saturating_sub(1), + }; + } + + std::iter::from_fn(move || loop { + if index >= list.len() { + return None; + } + + // Always iterate the insert set's iterator first if it exists + // It's on the top of the stack + if let Some(iter) = insert_set_iter.as_mut() { + let Some(next) = iter.next() else { + index += 1; + insert_set_iter = None; + continue; + }; + + return Some(next); + } + + let f = &list[index]; + let iter_counter = f.counter; + if iter_counter >= iter_id_span.counter.end { + return None; + } + + if iter_counter + f.cursor.rle_len() as Counter <= iter_id_span.counter.start { + index += 1; + continue; + } + + match &f.cursor { + Cursor::Insert(set) => { + insert_set_iter = Some(set.iter_range( + ID::new(iter_id_span.peer, iter_counter), + iter_id_span.counter.start, + iter_id_span.counter.end, + )); + continue; + } + Cursor::Delete(span) => { + index += 1; + let from = (iter_id_span.counter.start - iter_counter) + .max(0) + .min(span.atom_len() as Counter); + let to = (iter_id_span.counter.end - iter_counter) + .max(0) + .min(span.atom_len() as Counter); + if from == to { + continue; + } + + return Some(IterCursor::Delete(span.slice(from as usize, to as usize))); + } + Cursor::Move { from, to } => { + index += 1; + let op_id = ID::new(iter_id_span.peer, f.counter); + return Some(IterCursor::Move { + from_id: *from, + to_leaf: *to, + new_op_id: op_id, + }); + } + } + }) + } + + pub fn get_insert(&self, id: ID) -> Option { + let list = self.map.get(&id.peer)?; + let index = match list.binary_search_by_key(&id.counter, |x| x.counter) { + Ok(index) => index, + Err(index) => index - 1, + }; + + list[index] + .cursor + .get_insert((id.counter - list[index].counter) as usize) + } + + #[allow(unused)] + pub fn diagnose(&self) { + let fragment_num = self.map.iter().map(|x| x.1.len()).sum::(); + let insert_pieces = self + .map + .iter() + .flat_map(|x| x.1.iter()) + .map(|x| match &x.cursor { + Cursor::Insert(set) => set.len(), + Cursor::Delete(_) => 0, + Cursor::Move { .. } => 0, + }) + .sum::(); + let max_insert_len = self + .map + .iter() + .map(|x| { + x.1.iter() + .filter_map(|x| match &x.cursor { + Cursor::Insert(set) => Some(set.len()), + _ => None, + }) + .max() + .unwrap_or(0) + }) + .max() + .unwrap_or(0); + println!( + "fragments:{}, insert_pieces:{}, max_insert_len:{}", + fragment_num, insert_pieces, max_insert_len + ); + } +} + +#[derive(Debug)] +pub(super) struct Fragment { + pub(super) counter: Counter, + pub(super) cursor: Cursor, +} + +impl PartialEq for Fragment { + fn eq(&self, other: &Self) -> bool { + self.counter == other.counter + } +} + +impl Eq for Fragment {} + +impl PartialOrd for Fragment { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Fragment { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.counter.cmp(&other.counter) + } +} + +impl Fragment { + fn counter_end(&self) -> Counter { + self.counter + self.cursor.rle_len() as Counter + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum IterCursor { + Insert { + leaf: LeafIndex, + id_span: IdSpan, + }, + // deleted id_span, the start may be greater than the end + Delete(IdSpan), + // Move from `from ID` to `to LeafIndex` with `op_id` + Move { + from_id: ID, + to_leaf: LeafIndex, + new_op_id: ID, + }, +} + +#[derive(Debug)] +pub(super) enum Cursor { + Insert(InsertSet), + Delete(IdSpan), + Move { + from: ID, + to: LeafIndex, + // to id is the same as the current op_id + }, +} + +mod insert_set { + + use super::*; + use generic_btree::{ + rle::{CanRemove, TryInsert}, + BTree, BTreeTrait, LengthFinder, UseLengthFinder, + }; + use smallvec::SmallVec; + + #[derive(Debug)] + pub(crate) enum InsertSet { + Small(SmallInsertSet), + Large(LargeInsertSet), + } + + impl InsertSet { + pub(crate) fn new(leaf: LeafIndex, len: usize) -> Self { + Self::Small(SmallInsertSet { + set: smallvec![Insert { + leaf, + len: len as u32 + }], + len: len as u32, + }) + } + + pub(crate) fn update(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + self.upgrade_if_needed(); + match self { + Self::Small(set) => { + set.update(from, to, new_leaf); + } + Self::Large(set) => { + set.update(from, to, new_leaf); + } + } + } + + pub(crate) fn update_many(&mut self, updates: &[(usize, usize, LeafIndex)]) { + if updates.is_empty() { + return; + } + + if updates.len() == 1 { + let (from, to, leaf) = updates[0]; + self.update(from, to, leaf); + return; + } + + let len = self.len(); + if len > MAX_FRAGMENT_LEN { + for &(from, to, leaf) in updates { + self.update(from, to, leaf); + } + return; + } + + let mut dense: SmallVec<[LeafIndex; MAX_FRAGMENT_LEN]> = SmallVec::with_capacity(len); + match self { + InsertSet::Small(set) => { + for insert in set.set.iter() { + for _ in 0..insert.len { + dense.push(insert.leaf); + } + } + } + InsertSet::Large(set) => { + for insert in set.tree.iter() { + for _ in 0..insert.len { + dense.push(insert.leaf); + } + } + } + } + debug_assert_eq!(dense.len(), len); + + for &(from, to, leaf) in updates { + debug_assert!(from <= to); + debug_assert!(to <= len); + for i in from..to { + dense[i] = leaf; + } + } + + let mut new_set: SmallVec<[Insert; 1]> = SmallVec::new(); + if !dense.is_empty() { + let mut cur_leaf = dense[0]; + let mut cur_len: usize = 1; + for &leaf in dense.iter().skip(1) { + if leaf == cur_leaf { + cur_len += 1; + } else { + new_set.push(Insert { + leaf: cur_leaf, + len: cur_len as u32, + }); + cur_leaf = leaf; + cur_len = 1; + } + } + new_set.push(Insert { + leaf: cur_leaf, + len: cur_len as u32, + }); + } + + *self = InsertSet::Small(SmallInsertSet { + set: new_set, + len: len as u32, + }); + } + + pub(crate) fn get_insert(&self, pos: usize) -> Option { + match self { + Self::Small(set) => { + let mut index = 0; + for insert in set.set.iter() { + if index + insert.len as usize > pos { + return Some(insert.leaf); + } + index += insert.len as usize; + } + + unreachable!() + } + Self::Large(set) => set.get_insert(pos), + } + } + + pub(crate) fn len(&self) -> usize { + match self { + Self::Small(set) => set.len as usize, + Self::Large(set) => *set.tree.root_cache() as usize, + } + } + + pub(crate) fn iter_all( + &self, + peer: PeerID, + counter: Counter, + ) -> Box + '_> { + match self { + InsertSet::Small(set) => { + let mut offset = 0; + Box::new(set.set.iter().map(move |elem| { + let ans = IterCursor::Insert { + leaf: elem.leaf, + id_span: IdSpan::new( + peer, + counter + offset as Counter, + counter + offset as Counter + elem.len as Counter, + ), + }; + + offset += elem.len; + ans + })) + } + InsertSet::Large(set) => { + let mut offset = 0; + Box::new(set.tree.iter().map(move |elem| { + let ans = IterCursor::Insert { + leaf: elem.leaf, + id_span: IdSpan::new( + peer, + counter + offset as Counter, + counter + offset as Counter + elem.len as Counter, + ), + }; + + offset += elem.len; + ans + })) + } + } + } + + /// Iterate the given target span range, the start id of the InsertSet is cur_id + pub(crate) fn iter_range( + &self, + cur_id: ID, + target_start_counter: i32, + target_end_counter: i32, + ) -> Box + '_> { + match self { + InsertSet::Small(set) => { + let mut offset = 0; + Box::new(set.set.iter().filter_map(move |elem| { + let id_span = IdSpan::new( + cur_id.peer, + (cur_id.counter + offset as Counter) + .max(target_start_counter) + .min(target_end_counter), + (cur_id.counter + offset as Counter + elem.len as Counter) + .max(target_start_counter) + .min(target_end_counter), + ); + + offset += elem.len; + if id_span.atom_len() == 0 { + return None; + } + + let ans = IterCursor::Insert { + leaf: elem.leaf, + id_span, + }; + + Some(ans) + })) + } + InsertSet::Large(set) => { + // let mut offset = 0; + // Box::new(set.tree.iter().filter_map(move |elem| { + // let id_span = IdSpan::new( + // cur_id.peer, + // (cur_id.counter + offset as Counter) + // .max(target_start_counter) + // .min(target_end_counter), + // (cur_id.counter + offset as Counter + elem.len as Counter) + // .max(target_start_counter) + // .min(target_end_counter), + // ); + + // offset += elem.len; + // if id_span.atom_len() == 0 { + // return None; + // } + + // let ans = IterCursor::Insert { + // leaf: elem.leaf, + // id_span, + // }; + + // Some(ans) + // })); + let offset = (target_start_counter - cur_id.counter).max(0); + let (start, mut start_counter) = if offset > 0 { + match set.tree.query::(&(offset as usize)) { + Some(start) => ( + std::ops::Bound::Included(start.cursor), + // NOTE: Can this be wrong? + target_start_counter - start.cursor.offset as Counter, + ), + _ => (std::ops::Bound::Unbounded, cur_id.counter), + } + } else { + (std::ops::Bound::Unbounded, cur_id.counter) + }; + + Box::new( + set.tree + .iter_range((start, std::ops::Bound::Unbounded)) + .map(move |b| { + let id_span = IdSpan::new( + cur_id.peer, + (start_counter) + .max(target_start_counter) + .min(target_end_counter), + (start_counter + b.elem.rle_len() as Counter) + .max(target_start_counter) + .min(target_end_counter), + ); + + start_counter += b.elem.rle_len() as Counter; + if id_span.atom_len() == 0 { + return None; + } + + let ans = IterCursor::Insert { + leaf: b.elem.leaf, + id_span, + }; + + Some(ans) + }) + .take_while(|x| x.is_some()) + .map(|x| x.unwrap()), + ) + } + } + } + + fn upgrade_if_needed(&mut self) { + match self { + InsertSet::Small(set) => { + if set.set.len() < SMALL_SET_MAX_LEN { + return; + } + + let set = std::mem::take(&mut set.set); + let tree_set = LargeInsertSet::init_from_small(set); + *self = InsertSet::Large(tree_set); + } + InsertSet::Large(_) => {} + } + } + } + + #[derive(Debug)] + pub(crate) struct SmallInsertSet { + pub set: SmallVec<[Insert; 1]>, + pub len: u32, + } + + impl SmallInsertSet { + fn update(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + if from == to { + return; + } + + let mut cur_scan_index: usize = 0; + let old_set = std::mem::take(&mut self.set); + let mut new_set = SmallVec::with_capacity(old_set.len() + 2); + let mut new_leaf_inserted = false; + for insert in old_set.iter() { + if new_leaf_inserted { + let end = cur_scan_index + insert.len as usize; + if end <= to { + cur_scan_index = end; + continue; + } + + if cur_scan_index >= to { + new_set.push(*insert); + } else { + new_set.push(Insert { + leaf: insert.leaf, + len: (end - to) as u32, + }); + } + + cur_scan_index = end; + continue; + } + + if cur_scan_index + insert.len as usize <= from { + new_set.push(*insert); + cur_scan_index += insert.len as usize; + } else { + debug_assert!(!new_leaf_inserted); + let elem_end = cur_scan_index + insert.len as usize; + if cur_scan_index < from { + new_set.push(Insert { + leaf: insert.leaf, + len: (from - cur_scan_index) as u32, + }); + cur_scan_index = from; + } + + if elem_end > to { + new_set.push(Insert { + leaf: new_leaf, + len: (to - cur_scan_index) as u32, + }); + new_set.push(Insert { + leaf: insert.leaf, + len: (elem_end - to) as u32, + }); + } else { + new_set.push(Insert { + leaf: new_leaf, + len: (to - cur_scan_index) as u32, + }); + } + + new_leaf_inserted = true; + cur_scan_index = elem_end; + } + } + + self.set = new_set; + debug_assert_eq!( + self.len, + self.set.iter().map(|x| x.len as usize).sum::() as u32 + ); + } + } + + pub(super) struct InsertSetBTreeTrait; + impl CanRemove for Insert { + fn can_remove(&self) -> bool { + self.len == 0 + } + } + + impl generic_btree::rle::Mergeable for Insert { + fn can_merge(&self, rhs: &Self) -> bool { + self.leaf == rhs.leaf + } + + fn merge_right(&mut self, rhs: &Self) { + self.len += rhs.len; + } + + fn merge_left(&mut self, left: &Self) { + self.len += left.len; + } + } + + impl HasLength for Insert { + fn rle_len(&self) -> usize { + self.len as usize + } + } + + impl generic_btree::rle::Sliceable for Insert { + fn _slice(&self, range: std::ops::Range) -> Self { + Insert { + leaf: self.leaf, + len: range.len() as u32, + } + } + } + + impl TryInsert for Insert { + fn try_insert(&mut self, _: usize, elem: Self) -> Result<(), Self> { + if self.leaf == elem.leaf { + self.len += elem.len; + Ok(()) + } else { + Err(elem) + } + } + } + + impl BTreeTrait for InsertSetBTreeTrait { + type Elem = Insert; + + type Cache = i32; + + type CacheDiff = i32; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new: Self::Cache = caches.iter().map(|x| *x.cache()).sum(); + let diff = new - *cache; + *cache = new; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + elem.len as i32 + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } + } + + impl UseLengthFinder for InsertSetBTreeTrait { + fn get_len(cache: &i32) -> usize { + *cache as usize + } + } + + #[derive(Debug)] + pub(crate) struct LargeInsertSet { + tree: Box>, + } + + impl LargeInsertSet { + pub fn new() -> Self { + Self { + tree: Box::new(BTree::new()), + } + } + + fn update(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + if from == to { + return; + } + + let Some(from) = self.tree.query::(&from) else { + return; + }; + let to = self + .tree + .query::(&to) + .map(|x| x.cursor) + .or_else(|| self.tree.end_cursor()) + .unwrap(); + + self.tree.update(from.cursor..to, &mut |x| { + x.leaf = new_leaf; + None + }); + } + + fn get_insert(&self, pos: usize) -> Option { + let c = self.tree.query::(&pos)?.cursor; + Some(self.tree.get_elem(c.leaf)?.leaf) + } + + fn init_from_small(set: SmallVec<[Insert; 1]>) -> LargeInsertSet { + let mut tree_set = LargeInsertSet::new(); + for item in set.into_iter() { + tree_set.tree.push(item); + } + + tree_set + } + } +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct Insert { + leaf: LeafIndex, + len: u32, +} + +impl Cursor { + pub fn new_insert(leaf: LeafIndex, len: usize) -> Self { + Self::Insert(InsertSet::new(leaf, len)) + } + + pub fn new_move(leaf: LeafIndex, from_id: ID) -> Self { + Self::Move { + to: leaf, + from: from_id, + } + } + + #[allow(unused)] + pub fn new_delete(id_span: IdSpan) -> Self { + Self::Delete(id_span) + } + + fn update_insert(&mut self, from: usize, to: usize, new_leaf: LeafIndex) { + // tracing::info!( + // "set_insert: from={}, to={}, new_leaf={:?}", + // from, + // to, + // &new_leaf + // ); + + assert!(from <= to); + assert!(to <= self.rle_len()); + match self { + Self::Insert(set) => { + set.update(from, to, new_leaf); + } + Self::Move { .. } => { + unreachable!("update_insert on Move") + } + _ => unreachable!(), + } + } + + fn update_insert_many(&mut self, updates: &[(usize, usize, LeafIndex)]) { + match self { + Self::Insert(set) => { + set.update_many(updates); + } + Self::Move { .. } => { + unreachable!("update_insert_many on Move") + } + _ => unreachable!(), + } + } + + fn get_insert(&self, pos: usize) -> Option { + if pos >= self.rle_len() { + return None; + } + + match self { + Cursor::Insert(set) => set.get_insert(pos), + Cursor::Move { to, .. } => { + assert!(pos == 0); + Some(*to) + } + Cursor::Delete(_) => unreachable!(), + } + } +} + +impl HasLength for Cursor { + fn rle_len(&self) -> usize { + match self { + Cursor::Insert(set) => set.len(), + Cursor::Delete(d) => d.atom_len(), + Cursor::Move { .. } => 1, + } + } +} + +impl Mergeable for Cursor { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + (Self::Insert(InsertSet::Small(a)), Self::Insert(InsertSet::Small(b))) => { + a.set.last().unwrap().leaf == b.set.first().unwrap().leaf && b.len == 1 + } + (Self::Delete(a), Self::Delete(b)) => a.is_mergable(b, &()), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + (Self::Insert(InsertSet::Small(a)), Self::Insert(InsertSet::Small(b))) => { + assert!(b.len == 1); + a.set.last_mut().unwrap().len += b.set.first().unwrap().len; + a.len += b.len; + } + (Self::Delete(a), Self::Delete(b)) => { + a.merge(b, &()); + } + _ => unreachable!(), + } + } + + fn merge_left(&mut self, _: &Self) { + unreachable!(); + } +} + +#[cfg(test)] +mod test { + use std::{any::Any, ops::Range, panic::AssertUnwindSafe}; + + use super::*; + use generic_btree::{ + rle::{CanRemove, TryInsert}, + BTree, BTreeTrait, UseLengthFinder, + }; + use smallvec::smallvec; + + #[derive(Debug, Clone)] + struct DummyElem; + + impl generic_btree::rle::HasLength for DummyElem { + fn rle_len(&self) -> usize { + 1 + } + } + + impl generic_btree::rle::Sliceable for DummyElem { + fn _slice(&self, range: Range) -> Self { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + DummyElem + } + + fn split(&mut self, _pos: usize) -> Self { + unreachable!() + } + } + + impl generic_btree::rle::Mergeable for DummyElem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } + } + + impl TryInsert for DummyElem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> { + Err(elem) + } + } + + impl CanRemove for DummyElem { + fn can_remove(&self) -> bool { + false + } + } + + struct DummyTree; + + impl BTreeTrait for DummyTree { + type Elem = DummyElem; + type Cache = isize; + type CacheDiff = isize; + const USE_DIFF: bool = true; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new_cache = caches.iter().map(|child| child.cache).sum::(); + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2; + } + + fn get_elem_cache(_elem: &Self::Elem) -> Self::Cache { + 1 + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + cache_lhs - cache_rhs + } + } + + impl UseLengthFinder for DummyTree { + fn get_len(cache: &isize) -> usize { + *cache as usize + } + } + + fn dummy_leaf() -> LeafIndex { + let mut tree = BTree::::new(); + tree.push(DummyElem).leaf + } + + fn dummy_leaves(n: usize) -> Vec { + let mut tree = BTree::::new(); + (0..n).map(|_| tree.push(DummyElem).leaf).collect() + } + + fn panic_message(payload: Box) -> String { + if let Some(msg) = payload.downcast_ref::() { + msg.clone() + } else if let Some(msg) = payload.downcast_ref::<&str>() { + msg.to_string() + } else { + "".to_string() + } + } + + #[test] + fn test_id_to_cursor() { + let _map = IdToCursor::default(); + } + + #[test] + fn zero_len_fragment_panics_with_left_0_right_3() { + let peer = 1; + let mut map = IdToCursor::default(); + map.map.insert( + peer, + vec![ + Fragment { + counter: 0, + cursor: Cursor::Insert(insert_set::InsertSet::Small( + insert_set::SmallInsertSet { + set: smallvec![], + len: 0, + }, + )), + }, + Fragment { + counter: 3, + cursor: Cursor::new_insert(dummy_leaf(), 1), + }, + ], + ); + + let err = std::panic::catch_unwind(AssertUnwindSafe(|| { + map.update_insert(IdSpan::new(peer, 0, 3), dummy_leaf()); + })) + .expect_err("expected zero-length fragment to trip the invariant"); + let msg = panic_message(err); + assert!(msg.contains("left == right"), "{msg}"); + assert!(msg.contains("left: 0"), "{msg}"); + assert!(msg.contains("right: 3"), "{msg}"); + } + + #[test] + fn zero_width_small_update_keeps_insert_set_non_empty() { + let leaf = dummy_leaf(); + let mut set = insert_set::InsertSet::new(leaf, 3); + set.update(0, 0, dummy_leaf()); + + match set { + insert_set::InsertSet::Small(set) => { + assert_eq!(set.len, 3); + assert_eq!(set.set.len(), 1); + assert_eq!(set.set[0].len, 3); + } + insert_set::InsertSet::Large(_) => panic!("expected small insert set"), + } + } + + #[test] + fn large_update_can_replace_the_tail_range() { + let leaves = dummy_leaves(4); + let mut set = insert_set::InsertSet::Small(insert_set::SmallInsertSet { + set: smallvec![ + Insert { + leaf: leaves[0], + len: 100, + }, + Insert { + leaf: leaves[1], + len: 100, + }, + Insert { + leaf: leaves[2], + len: 100, + } + ], + len: 300, + }); + set.update(0, 1, leaves[3]); + let new_leaf = leaves[3]; + set.update_many(&[(200, 300, new_leaf)]); + + assert_ne!(set.get_insert(199), Some(new_leaf)); + for i in 200..300 { + assert_eq!(set.get_insert(i), Some(new_leaf)); + } + } +} diff --git a/vendor/loro-internal/src/container/tree/bool_rle_vec.rs b/vendor/loro-internal/src/container/tree/bool_rle_vec.rs new file mode 100644 index 00000000..c3609d72 --- /dev/null +++ b/vendor/loro-internal/src/container/tree/bool_rle_vec.rs @@ -0,0 +1,442 @@ +// TODO: use boolrle to encode the has_effects array +#![allow(dead_code)] +use std::io::{Read, Write}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct BoolRleVec { + /// For the number of the nth item (0-indexed): + /// - if n is odd, the item is the run length of consecutive `true` values + /// - if n is even, the item is the run length of consecutive `false` values + rle_vec: Vec, + len: usize, +} + +impl BoolRleVec { + pub fn push(&mut self, value: bool) { + self.len += 1; + if self.rle_vec.is_empty() { + self.rle_vec.push(0); + } + + let last_index = self.rle_vec.len() - 1; + let is_last_run_true = last_index % 2 == 1; + + if (value && is_last_run_true) || (!value && !is_last_run_true) { + // If the new value matches the last run, increment its length + self.rle_vec[last_index] += 1; + } else { + // If the new value doesn't match, start a new run + self.rle_vec.push(1); + } + } + + pub fn pop(&mut self) -> Option { + while let Some(last) = self.rle_vec.last() { + if *last == 0 { + self.rle_vec.pop(); + } else { + break; + } + } + + if self.rle_vec.is_empty() { + return None; + } + + self.len -= 1; + let last_index = self.rle_vec.len() - 1; + let is_last_run_true = last_index % 2 == 1; + self.rle_vec[last_index] -= 1; + if self.rle_vec[last_index] == 0 { + self.rle_vec.pop(); + } + + Some(is_last_run_true) + } + + pub fn drop_last_n(&mut self, n: usize) { + if n > self.len { + panic!("Attempted to drop more elements than exist in the vector"); + } + + let mut remaining = n; + while remaining > 0 { + if let Some(last) = self.rle_vec.last_mut() { + if *last <= remaining as u32 { + remaining -= *last as usize; + self.rle_vec.pop(); + } else { + *last -= remaining as u32; + remaining = 0; + } + } else { + break; + } + } + + self.len -= n; + + // Remove any trailing zero-length runs + while self.rle_vec.last().is_some_and(|&x| x == 0) { + self.rle_vec.pop(); + } + } + + pub fn merge(&mut self, other: &BoolRleVec) { + if self.is_empty() { + self.rle_vec = other.rle_vec.clone(); + self.len = other.len; + return; + } + + if other.is_empty() { + return; + } + + // Align the end of self with the start of other + let self_last_run_true = self.rle_vec.len().is_multiple_of(2); + if self_last_run_true { + self.rle_vec.extend_from_slice(&other.rle_vec); + } else { + *self.rle_vec.last_mut().unwrap() += other.rle_vec[0]; + self.rle_vec.extend_from_slice(&other.rle_vec[1..]); + } + + self.len += other.len; + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn iter(&self) -> BoolRleVecIter<'_> { + BoolRleVecIter { + rle_vec: self, + index: 0, + offset: 0, + } + } + + pub fn new() -> Self { + Self { + rle_vec: Default::default(), + len: 0, + } + } + + pub fn encode(&self, writer: &mut W) -> Result<(), std::io::Error> { + leb128::write::unsigned(writer, self.rle_vec.len() as u64)?; + for item in &self.rle_vec { + leb128::write::unsigned(writer, *item as u64)?; + } + + Ok(()) + } + + pub fn decode(reader: &mut R) -> Result { + let len = leb128::read::unsigned(reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))? + as usize; + let mut rle_vec = Vec::with_capacity(len); + let mut total_len = 0; + for _ in 0..len { + let v = leb128::read::unsigned(reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))? + as u32; + rle_vec.push(v); + total_len += v; + } + + Ok(Self { + rle_vec, + len: total_len as usize, + }) + } +} + +pub(crate) struct BoolRleVecIter<'a> { + rle_vec: &'a BoolRleVec, + index: usize, + offset: u32, +} + +impl Iterator for BoolRleVecIter<'_> { + type Item = bool; + + fn next(&mut self) -> Option { + if self.index >= self.rle_vec.rle_vec.len() { + return None; + } + + let run_length = self.rle_vec.rle_vec[self.index]; + if run_length == 0 { + self.index += 1; + self.offset = 0; + return self.next(); + } + + let value = self.index % 2 == 1; + self.offset += 1; + if self.offset >= run_length { + self.index += 1; + self.offset = 0; + } + + Some(value) + } + + fn nth(&mut self, n: usize) -> Option { + self.offset += n as u32; + while self.index < self.rle_vec.rle_vec.len() + && self.offset >= self.rle_vec.rle_vec[self.index] + { + self.offset -= self.rle_vec.rle_vec[self.index]; + self.index += 1; + } + + self.next() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_bool_rle_vec() { + let truth = [true, true, false, false, true, true]; + let mut rle_vec = BoolRleVec::new(); + for t in truth.iter() { + rle_vec.push(*t); + } + + let iter = rle_vec.iter(); + for (a, b) in truth.iter().zip(iter) { + assert_eq!(*a, b); + } + + let mut encoded = Vec::new(); + rle_vec.encode(&mut encoded).unwrap(); + let decoded = BoolRleVec::decode(&mut encoded.as_slice()).unwrap(); + assert_eq!(rle_vec, decoded); + } + + #[test] + fn test_bool_rle_vec_skip() { + let truth = [true, true, false, false, true, true]; + let mut rle_vec = BoolRleVec::new(); + for t in truth.iter() { + rle_vec.push(*t); + } + + let iter = rle_vec.iter(); + for (a, b) in truth.iter().skip(3).zip(iter.skip(3)) { + assert_eq!(*a, b); + } + } + + #[test] + fn test_bool_rle_vec_empty() { + let rle_vec = BoolRleVec::new(); + assert_eq!(rle_vec.len(), 0); + assert!(rle_vec.iter().next().is_none()); + } + + #[test] + fn test_bool_rle_vec_single_element() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + assert_eq!(rle_vec.len(), 1); + assert_eq!(rle_vec.iter().next(), Some(true)); + } + + #[test] + fn test_bool_rle_vec_alternating() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + rle_vec.push(false); + rle_vec.push(true); + rle_vec.push(false); + assert_eq!(rle_vec.len(), 4); + assert_eq!( + rle_vec.iter().collect::>(), + vec![true, false, true, false] + ); + } + + #[test] + fn test_bool_rle_vec_long_run() { + let mut rle_vec = BoolRleVec::new(); + for _ in 0..1000 { + rle_vec.push(true); + } + rle_vec.push(false); + assert_eq!(rle_vec.len(), 1001); + assert_eq!(rle_vec.iter().filter(|&x| x).count(), 1000); + assert_eq!(rle_vec.iter().filter(|&x| !x).count(), 1); + } + + #[test] + fn test_bool_rle_vec_nth() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + rle_vec.push(true); + rle_vec.push(false); + rle_vec.push(false); + rle_vec.push(true); + let mut iter = rle_vec.iter(); + assert_eq!(iter.nth(2), Some(false)); + assert_eq!(iter.nth(1), Some(true)); + assert_eq!(iter.next(), None); + } + + #[test] + fn test_bool_rle_vec_iter_skip() { + let mut rle_vec = BoolRleVec::new(); + rle_vec.push(true); + rle_vec.push(true); + rle_vec.push(false); + rle_vec.push(false); + rle_vec.push(true); + rle_vec.push(false); + + // Test skipping zero elements + let iter = rle_vec.iter(); + assert_eq!( + iter.collect::>(), + vec![true, true, false, false, true, false] + ); + + // Test skipping some elements + let iter = rle_vec.iter().skip(2); + assert_eq!(iter.collect::>(), vec![false, false, true, false]); + + // Test skipping all elements + let mut iter = rle_vec.iter().skip(6); + assert_eq!(iter.next(), None); + + // Test skipping more than available elements + let mut iter = rle_vec.iter().skip(10); + assert_eq!(iter.next(), None); + + // Test skipping with a long run + let mut long_rle_vec = BoolRleVec::new(); + for _ in 0..1000 { + long_rle_vec.push(true); + } + long_rle_vec.push(false); + long_rle_vec.push(true); + + let iter = long_rle_vec.iter().skip(999); + assert_eq!(iter.collect::>(), vec![true, false, true]); + } + + #[test] + fn test_bool_rle_vec_merge() { + // Test merging two empty vectors + let mut vec1 = BoolRleVec::new(); + let vec2 = BoolRleVec::new(); + vec1.merge(&vec2); + assert!(vec1.is_empty()); + + // Test merging an empty vector with a non-empty one + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + vec2.push(true); + vec2.push(false); + vec1.merge(&vec2); + assert_eq!(vec1.iter().collect::>(), vec![true, false]); + + // Test merging two non-empty vectors + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + vec1.push(true); + vec1.push(true); + vec2.push(false); + vec2.push(true); + vec1.merge(&vec2); + assert_eq!( + vec1.iter().collect::>(), + vec![true, true, false, true] + ); + + // Test merging with alignment (last run of vec1 matches first run of vec2) + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + vec1.push(true); + vec1.push(false); + vec2.push(false); + vec2.push(true); + vec1.merge(&vec2); + assert_eq!( + vec1.iter().collect::>(), + vec![true, false, false, true] + ); + + // Test merging with long runs + let mut vec1 = BoolRleVec::new(); + let mut vec2 = BoolRleVec::new(); + for _ in 0..100 { + vec1.push(true); + } + for _ in 0..100 { + vec2.push(false); + } + vec1.merge(&vec2); + assert_eq!(vec1.len(), 200); + assert_eq!(vec1.rle_vec.len(), 3); + assert_eq!(vec1.iter().take(100).filter(|&x| x).count(), 100); + assert_eq!(vec1.iter().skip(100).filter(|&x| !x).count(), 100); + } + + #[test] + fn test_drop_last_n() { + // Test dropping from a single run + let mut vec = BoolRleVec::new(); + for _ in 0..5 { + vec.push(true); + } + vec.drop_last_n(3); + assert_eq!(vec.iter().collect::>(), vec![true, true]); + + // Test dropping across multiple runs + let mut vec = BoolRleVec::new(); + vec.push(true); + vec.push(true); + vec.push(false); + vec.push(false); + vec.push(true); + vec.drop_last_n(3); + assert_eq!(vec.iter().collect::>(), vec![true, true]); + + // Test dropping entire vector + let mut vec = BoolRleVec::new(); + vec.push(true); + vec.push(false); + vec.drop_last_n(2); + assert!(vec.is_empty()); + + // Test dropping from long runs + let mut vec = BoolRleVec::new(); + for _ in 0..100 { + vec.push(true); + } + for _ in 0..100 { + vec.push(false); + } + vec.drop_last_n(150); + assert_eq!(vec.len(), 50); + assert!(vec.iter().all(|x| x)); + + // Test dropping zero elements + let mut vec = BoolRleVec::new(); + vec.push(true); + vec.push(false); + vec.drop_last_n(0); + assert_eq!(vec.iter().collect::>(), vec![true, false]); + } +} diff --git a/vendor/loro-internal/src/container/tree/mod.rs b/vendor/loro-internal/src/container/tree/mod.rs new file mode 100644 index 00000000..d9f97a10 --- /dev/null +++ b/vendor/loro-internal/src/container/tree/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod bool_rle_vec; +pub(crate) mod tree_ds; +pub(crate) mod tree_op; diff --git a/vendor/loro-internal/src/container/tree/tree_ds.rs b/vendor/loro-internal/src/container/tree/tree_ds.rs new file mode 100644 index 00000000..01912995 --- /dev/null +++ b/vendor/loro-internal/src/container/tree/tree_ds.rs @@ -0,0 +1,115 @@ +#![allow(unused)] + +use loro_common::{IdFull, IdLp, TreeID}; +use rustc_hash::FxHashMap; + +use crate::VersionVector; + +use super::bool_rle_vec::BoolRleVec; + +pub(crate) struct TreeLinearHistory { + ops: Vec, + has_effect: BoolRleVec, + target_to_op_idx: FxHashMap, +} + +struct TreeOpWrap { + op: TreeOp, + last_effective_update_on_target: Option, +} + +pub(crate) struct TreeOp { + pub id: IdFull, + pub target: TreeID, +} + +pub(crate) struct PoppedTreeOp { + pub op: TreeOp, + pub has_effect: bool, + pub last_update_on_target: Option, +} + +impl TreeLinearHistory { + pub fn new() -> Self { + Self { + ops: Vec::new(), + has_effect: BoolRleVec::new(), + target_to_op_idx: FxHashMap::default(), + } + } + + pub fn from_vec(ops: Vec, has_effect: BoolRleVec) -> Self { + let mut this = Self::new(); + for (op, e) in ops.into_iter().zip(has_effect.iter()) { + this.push(op, e); + } + + this + } + + pub fn push(&mut self, op: TreeOp, has_effect: bool) { + if has_effect { + let last = self.target_to_op_idx.insert(op.target, self.ops.len()); + self.ops.push(TreeOpWrap { + op, + last_effective_update_on_target: last, + }); + } else { + self.ops.push(TreeOpWrap { + op, + last_effective_update_on_target: None, + }); + } + + self.has_effect.push(has_effect); + } + + pub fn pop_util(&mut self, threshold: IdLp) -> Vec { + let index = match self + .ops + .binary_search_by(|op_wrap| op_wrap.op.id.idlp().cmp(&threshold)) + { + Ok(index) => index, + Err(index) => index, + }; + + self._pop_util(index) + } + + // Some of them need to be pushed back later if they are inside the vv + #[must_use] + pub fn pop_until_all_inside_vv(&mut self, vv: &VersionVector) -> Vec { + for (i, op_wrap) in self.ops.iter().enumerate() { + if vv.get(&op_wrap.op.id.peer).unwrap() <= &op_wrap.op.id.counter { + return self._pop_util(i); + } + } + + vec![] + } + + fn _pop_util(&mut self, index: usize) -> Vec { + todo!() + } + + pub fn last_update_on(&self, target: TreeID) -> Option { + self.target_to_op_idx + .get(&target) + .and_then(|&idx| self.ops.get(idx)) + .map(|op_wrap| op_wrap.op.id) + } + + pub fn greatest_idlp(&self) -> Option { + self.ops.last().map(|op| op.op.id.idlp()) + } + + #[inline] + pub fn len(&self) -> usize { + self.ops.len() + } + + #[inline] + pub fn get_bool_rle_vec(&self) -> &BoolRleVec { + &self.has_effect + } +} diff --git a/vendor/loro-internal/src/container/tree/tree_op.rs b/vendor/loro-internal/src/container/tree/tree_op.rs new file mode 100644 index 00000000..5f7fe3c2 --- /dev/null +++ b/vendor/loro-internal/src/container/tree/tree_op.rs @@ -0,0 +1,75 @@ +use fractional_index::FractionalIndex; +use loro_common::TreeID; +use rle::{HasLength, Mergable}; +use serde::{Deserialize, Serialize}; + +use crate::state::TreeParentId; + +/// The operation of movable tree. +/// +/// In the movable tree, there are three actions: +/// - **Create**: target tree id will be generated by [`Transaction`], and parent tree id is `None`. +/// - **Move**: move target tree node a child node of the specified parent node. +/// - **Delete**: move target tree node to [`loro_common::DELETED_TREE_ROOT`]. +/// +/// +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum TreeOp { + Create { + target: TreeID, + parent: Option, + position: FractionalIndex, + }, + Move { + target: TreeID, + parent: Option, + position: FractionalIndex, + }, + Delete { + target: TreeID, + }, +} + +impl TreeOp { + pub(crate) fn target(&self) -> TreeID { + match self { + TreeOp::Create { target, .. } => *target, + TreeOp::Move { target, .. } => *target, + TreeOp::Delete { target, .. } => *target, + } + } + + #[allow(unused)] + pub(crate) fn parent(&self) -> Option { + match self { + TreeOp::Create { parent, .. } => *parent, + TreeOp::Move { parent, .. } => *parent, + TreeOp::Delete { .. } => Some(TreeID::delete_root()), + } + } + + pub(crate) fn parent_id(&self) -> TreeParentId { + match self { + TreeOp::Create { parent, .. } => TreeParentId::from(*parent), + TreeOp::Move { parent, .. } => TreeParentId::from(*parent), + TreeOp::Delete { .. } => TreeParentId::Deleted, + } + } + + pub(crate) fn fractional_index(&self) -> Option { + match self { + TreeOp::Create { position, .. } | TreeOp::Move { position, .. } => { + Some(position.clone()) + } + TreeOp::Delete { .. } => None, + } + } +} + +impl HasLength for TreeOp { + fn content_len(&self) -> usize { + 1 + } +} + +impl Mergable for TreeOp {} diff --git a/vendor/loro-internal/src/cursor.rs b/vendor/loro-internal/src/cursor.rs new file mode 100644 index 00000000..f3c35585 --- /dev/null +++ b/vendor/loro-internal/src/cursor.rs @@ -0,0 +1,97 @@ +use loro_common::{ContainerID, ID}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Cursor { + // It's option because it's possible that the given container is empty. + pub id: Option, + pub container: ContainerID, + /// The target position is at the left, middle, or right of the given id. + /// + /// Side info can help to model the selection + pub side: Side, + /// The position of the cursor in the container when the cursor is created. + /// For text, this is the unicode codepoint index + /// This value is not encoded + pub(crate) origin_pos: usize, +} + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Side { + Left = -1, + #[default] + Middle = 0, + Right = 1, +} + +impl Side { + pub fn from_i32(i: i32) -> Option { + match i { + -1 => Some(Side::Left), + 0 => Some(Side::Middle), + 1 => Some(Side::Right), + _ => None, + } + } + + pub fn to_i32(&self) -> i32 { + *self as i32 + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PosQueryResult { + pub update: Option, + pub current: AbsolutePosition, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AbsolutePosition { + pub pos: usize, + /// The target position is at the left, middle, or right of the given pos. + pub side: Side, +} + +#[derive(Debug, Clone, Copy, thiserror::Error)] +pub enum CannotFindRelativePosition { + #[error("Cannot find relative position. The container is deleted.")] + ContainerDeleted, + #[error("Cannot find relative position. It may be that the given id is deleted and the relative history is cleared.")] + HistoryCleared, + #[error("Cannot find relative position. The id is not found.")] + IdNotFound, +} + +impl Cursor { + pub fn new(id: Option, container: ContainerID, side: Side, origin_pos: usize) -> Self { + Self { + id, + container, + side, + origin_pos, + } + } + + pub fn encode(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + pub fn decode(data: &[u8]) -> Result { + postcard::from_bytes(data) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PosType { + /// The index is based on the length of the text in bytes(UTF-8). + Bytes, + /// The index is based on the length of the text in Unicode Code Points. + Unicode, + /// The index is based on the length of the text in UTF-16 code units. + Utf16, + /// The index is based on the length of the text in events. + /// It is determined by the `wasm` feature. + Event, + /// The index is based on the entity index. + Entity, +} diff --git a/vendor/loro-internal/src/dag.rs b/vendor/loro-internal/src/dag.rs new file mode 100644 index 00000000..fc7ee9c9 --- /dev/null +++ b/vendor/loro-internal/src/dag.rs @@ -0,0 +1,922 @@ +//! DAG (Directed Acyclic Graph) is a common data structure in distributed system. +//! +//! This mod contains the DAGs in our CRDT. It's not a general DAG, it has some specific properties that +//! we used to optimize the speed: +//! - Each node has lamport clock. +//! - Each node has its ID (client_id, counter). +//! - We use ID to refer to node rather than content addressing (hash) +//! +use std::{ + borrow::Cow, + collections::{BinaryHeap, HashMap}, + fmt::Debug, +}; + +use loro_common::IdSpanVector; +use rle::{HasLength, Sliceable}; +use rustc_hash::{FxHashMap, FxHashSet}; +use smallvec::SmallVec; +mod iter; +mod mermaid; +#[cfg(feature = "test_utils")] +mod test; +#[cfg(feature = "test_utils")] +pub use test::{fuzz_alloc_tree, Interaction}; + +use crate::{ + change::Lamport, + diff_calc::DiffMode, + id::{Counter, PeerID, ID}, + span::{CounterSpan, HasId, HasIdSpan, HasLamport, HasLamportSpan, IdSpan}, + version::{Frontiers, VersionVector, VersionVectorDiff}, +}; + +use self::{ + iter::{iter_dag, iter_dag_with_vv, DagCausalIter, DagIterator, DagIteratorVV}, + mermaid::dag_to_mermaid, +}; + +pub(crate) trait DagNode: HasLamport + HasId + HasLength + Debug + Sliceable { + fn deps(&self) -> &Frontiers; + + #[allow(unused)] + #[inline] + fn get_lamport_from_counter(&self, c: Counter) -> Lamport { + self.lamport() + c as Lamport - self.id_start().counter as Lamport + } +} + +/// Dag (Directed Acyclic Graph). +/// +/// We have following invariance in DAG +/// - All deps' lamports are smaller than current node's lamport +pub(crate) trait Dag: Debug { + type Node: DagNode; + + fn get(&self, id: ID) -> Option; + #[allow(unused)] + fn frontier(&self) -> &Frontiers; + fn vv(&self) -> &VersionVector; + fn contains(&self, id: ID) -> bool; +} + +pub(crate) trait DagUtils: Dag { + fn find_common_ancestor(&self, a_id: &Frontiers, b_id: &Frontiers) -> (Frontiers, DiffMode); + /// Slow, should probably only use on dev + #[allow(unused)] + fn get_vv(&self, id: ID) -> VersionVector; + #[allow(unused)] + fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff; + fn iter_causal(&self, from: Frontiers, target: IdSpanVector) -> DagCausalIter<'_, Self> + where + Self: Sized; + #[allow(unused)] + fn iter(&self) -> DagIterator<'_, Self::Node> + where + Self: Sized; + #[allow(unused)] + fn iter_with_vv(&self) -> DagIteratorVV<'_, Self::Node> + where + Self: Sized; + #[allow(unused)] + fn mermaid(&self) -> String + where + Self: Sized; +} + +impl DagUtils for T { + #[inline] + fn find_common_ancestor(&self, a_id: &Frontiers, b_id: &Frontiers) -> (Frontiers, DiffMode) { + // TODO: perf: make it also return the spans to reach common_ancestors + find_common_ancestor(&|id| self.get(id), a_id, b_id) + } + + #[inline] + fn get_vv(&self, id: ID) -> VersionVector { + get_version_vector(&|id| self.get(id), id) + } + + fn find_path(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + let mut ans = VersionVectorDiff::default(); + if from == to { + return ans; + } + + if from.len() == 1 && to.len() == 1 { + let from = from.as_single().unwrap(); + let to = to.as_single().unwrap(); + if from.peer == to.peer { + let from_span = self.get(from).unwrap(); + let to_span = self.get(to).unwrap(); + if from_span.id_start() == to_span.id_start() { + if from.counter < to.counter { + ans.forward.insert( + from.peer, + CounterSpan::new(from.counter + 1, to.counter + 1), + ); + } else { + ans.retreat.insert( + from.peer, + CounterSpan::new(to.counter + 1, from.counter + 1), + ); + } + return ans; + } + + if from_span.deps().len() == 1 + && to_span.contains_id(from_span.deps().as_single().unwrap()) + { + ans.retreat.insert( + from.peer, + CounterSpan::new(to.counter + 1, from.counter + 1), + ); + return ans; + } + + if to_span.deps().len() == 1 + && from_span.contains_id(to_span.deps().as_single().unwrap()) + { + ans.forward.insert( + from.peer, + CounterSpan::new(from.counter + 1, to.counter + 1), + ); + return ans; + } + } + } + + _find_common_ancestor( + &|v| self.get(v), + from, + to, + &mut |span, node_type| match node_type { + NodeType::A => ans.merge_left(span), + NodeType::B => ans.merge_right(span), + NodeType::Shared => { + ans.subtract_start_left(span); + ans.subtract_start_right(span); + } + }, + true, + ); + + ans + } + + #[inline(always)] + fn iter_with_vv(&self) -> DagIteratorVV<'_, Self::Node> + where + Self: Sized, + { + iter_dag_with_vv(self) + } + + #[inline(always)] + fn iter_causal(&self, from: Frontiers, target: IdSpanVector) -> DagCausalIter<'_, Self> + where + Self: Sized, + { + DagCausalIter::new(self, from, target) + } + + #[inline(always)] + fn iter(&self) -> DagIterator<'_, Self::Node> + where + Self: Sized, + { + iter_dag(self) + } + + /// You can visualize and generate img link at https://mermaid.live/ + #[inline] + fn mermaid(&self) -> String + where + Self: Sized, + { + dag_to_mermaid(self) + } +} + +#[allow(dead_code)] +fn get_version_vector<'a, Get, D>(get: &'a Get, id: ID) -> VersionVector +where + Get: Fn(ID) -> Option, + D: DagNode + 'a, +{ + let mut vv = VersionVector::new(); + let mut visited: FxHashSet = FxHashSet::default(); + vv.insert(id.peer, id.counter + 1); + let node = get(id).unwrap(); + + if node.deps().is_empty() { + return vv; + } + + let mut stack = Vec::with_capacity(node.deps().len()); + for dep in node.deps().iter() { + stack.push(dep); + } + + while let Some(node_id) = stack.pop() { + let node = get(node_id).unwrap(); + let node_id_start = node.id_start(); + vv.try_update_last(node_id); + if !visited.contains(&node_id_start) { + for dep in node.deps().iter() { + if !visited.contains(&dep) { + stack.push(dep); + } + } + + visited.insert(node_id_start); + } + } + + vv +} + +#[derive(Debug, PartialEq, Eq)] +struct OrdIdSpan<'a> { + id: ID, + lamport: Lamport, + len: usize, + deps: Cow<'a, Frontiers>, +} + +impl HasLength for OrdIdSpan<'_> { + fn content_len(&self) -> usize { + self.len + } +} + +impl HasId for OrdIdSpan<'_> { + fn id_start(&self) -> ID { + self.id + } +} + +impl HasLamport for OrdIdSpan<'_> { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl PartialOrd for OrdIdSpan<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for OrdIdSpan<'_> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport_last() + .cmp(&other.lamport_last()) + .then(self.id.peer.cmp(&other.id.peer)) + // If they have the same last id, we want the shorter one to be greater; + // Otherwise, find_common_ancestor won't work correctly. Because we may + // lazily load the dag node, so sometimes the longer one should be broken + // into smaller pieces but it's already pushed to the queue. + .then(other.len.cmp(&self.len)) + } +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] +enum NodeType { + A, + B, + Shared, +} + +impl<'a> OrdIdSpan<'a> { + #[inline] + fn from_dag_node(id: ID, get: &'a F) -> Option> + where + D: DagNode + 'a, + F: Fn(ID) -> Option, + { + let span = get(id)?; + let span_id = span.id_start(); + Some(OrdIdSpan { + id: span_id, + lamport: span.lamport(), + deps: Cow::Owned(span.deps().clone()), + len: (id.counter - span_id.counter) as usize + 1, + }) + } + + #[inline] + fn get_min(&self) -> OrdIdSpan<'a> { + OrdIdSpan { + id: self.id, + lamport: self.lamport, + deps: Cow::Owned(Default::default()), + len: 1, + } + } +} + +#[inline(always)] +fn find_common_ancestor<'a, F, D>( + get: &'a F, + a_id: &Frontiers, + b_id: &Frontiers, +) -> (Frontiers, DiffMode) +where + D: DagNode + 'a, + F: Fn(ID) -> Option, +{ + if b_id.is_empty() { + return (Default::default(), DiffMode::Checkout); + } + + _find_common_ancestor_new(get, a_id, b_id) +} + +/// - deep whether keep searching until the min of non-shared node is found +fn _find_common_ancestor<'a, F, D, G>( + get: &'a F, + a_ids: &Frontiers, + b_ids: &Frontiers, + notify: &mut G, + find_path: bool, +) -> FxHashMap +where + D: DagNode + 'a, + F: Fn(ID) -> Option, + G: FnMut(IdSpan, NodeType), +{ + let mut ans: FxHashMap = Default::default(); + let mut queue: BinaryHeap<(OrdIdSpan, NodeType)> = BinaryHeap::new(); + for id in a_ids.iter() { + queue.push((OrdIdSpan::from_dag_node(id, get).unwrap(), NodeType::A)); + } + for id in b_ids.iter() { + queue.push((OrdIdSpan::from_dag_node(id, get).unwrap(), NodeType::B)); + } + let mut visited: HashMap = FxHashMap::default(); + // invariants in this method: + // + // - visited's (client, counters) are subset of max(version_vector_a, version_vector_b) + // - visited's node type reflecting whether we found the shared node of this client + // - ans's client id never repeat + // - nodes with the same id will only be visited once + // - we may visit nodes that are before the common ancestors + + // type count in the queue. if both are zero, we can stop + let mut a_count = a_ids.len(); + let mut b_count = b_ids.len(); + let mut min = None; + while let Some((node, mut node_type)) = queue.pop() { + match node_type { + NodeType::A => a_count -= 1, + NodeType::B => b_count -= 1, + NodeType::Shared => {} + } + + if node_type != NodeType::Shared { + if let Some(min) = &mut min { + let node_start = node.get_min(); + if node_start < *min { + *min = node_start; + } + } else { + min = Some(node.get_min()) + } + } + + // pop the same node in the queue + while let Some((other_node, other_type)) = queue.peek() { + if node.id_span() == other_node.id_span() { + if node_type == *other_type { + match node_type { + NodeType::A => a_count -= 1, + NodeType::B => b_count -= 1, + NodeType::Shared => {} + } + } else { + if node_type != NodeType::Shared { + if visited.get(&node.id.peer).map(|(_, t)| *t) != Some(NodeType::Shared) { + ans.insert(node.id.peer, other_node.id_last().counter); + } + node_type = NodeType::Shared; + } + match other_type { + NodeType::A => a_count -= 1, + NodeType::B => b_count -= 1, + NodeType::Shared => {} + } + } + + queue.pop(); + } else { + break; + } + } + + // detect whether client is visited by other + if let Some((ctr, visited_type)) = visited.get_mut(&node.id.peer) { + debug_assert!(*ctr >= node.id_last().counter); + if *visited_type == NodeType::Shared { + node_type = NodeType::Shared; + } else if *visited_type != node_type { + // if node_type is shared, ans should already contains it or its descendance + if node_type != NodeType::Shared { + ans.insert(node.id.peer, node.id_last().counter); + } + *visited_type = NodeType::Shared; + node_type = NodeType::Shared; + } + } else { + visited.insert(node.id.peer, (node.id_last().counter, node_type)); + } + + // if this is not shared, the end of the span must be only reachable from A, or only reachable from B. + // but the begin of the span may be reachable from both A and B + notify(node.id_span(), node_type); + + match node_type { + NodeType::A => a_count += node.deps.len(), + NodeType::B => b_count += node.deps.len(), + NodeType::Shared => {} + } + + if a_count == 0 && b_count == 0 && (min.is_none() || &node <= min.as_ref().unwrap()) { + if node_type != NodeType::Shared { + ans.clear(); + } + + break; + } + + for dep_id in node.deps.as_ref().iter() { + queue.push((OrdIdSpan::from_dag_node(dep_id, get).unwrap(), node_type)); + } + + if node_type != NodeType::Shared { + if queue.is_empty() { + ans.clear(); + break; + } + if node.deps.is_empty() && !find_path { + if node.len == 1 { + ans.clear(); + break; + } + + match node_type { + NodeType::A => a_count += 1, + NodeType::B => b_count += 1, + NodeType::Shared => {} + } + + queue.push(( + OrdIdSpan { + deps: Cow::Owned(Default::default()), + id: node.id, + len: 1, + lamport: node.lamport, + }, + node_type, + )); + } + } + } + + ans +} + +fn _find_common_ancestor_new<'a, F, D>( + get: &'a F, + left: &Frontiers, + right: &Frontiers, +) -> (Frontiers, DiffMode) +where + D: DagNode + 'a, + F: Fn(ID) -> Option, +{ + if right.is_empty() { + return (Default::default(), DiffMode::Checkout); + } + + if left.is_empty() { + if right.len() == 1 { + let mut node_id = right.as_single().unwrap(); + let mut node = get(node_id).unwrap(); + while node.deps().len() == 1 { + node_id = node.deps().as_single().unwrap(); + node = get(node_id).unwrap(); + } + + if node.deps().is_empty() { + return (Default::default(), DiffMode::Linear); + } + } + + return (Default::default(), DiffMode::ImportGreaterUpdates); + } + + if left.len() == 1 && right.len() == 1 { + let left = left.as_single().unwrap(); + let right = right.as_single().unwrap(); + if left.peer == right.peer { + let left_span = get(left).unwrap(); + let right_span = get(right).unwrap(); + if left_span.id_start() == right_span.id_start() { + if left.counter < right.counter { + return (left.into(), DiffMode::Linear); + } else { + return (right.into(), DiffMode::Checkout); + } + } + + if left_span.deps().len() == 1 + && right_span.contains_id(left_span.deps().as_single().unwrap()) + { + return (right.into(), DiffMode::Checkout); + } + + if right_span.deps().len() == 1 + && left_span.contains_id(right_span.deps().as_single().unwrap()) + { + return (left.into(), DiffMode::Linear); + } + } + } + + let mut is_linear = left.len() <= 1 && right.len() == 1; + let mut is_right_greater = true; + let mut ans: Frontiers = Default::default(); + let mut queue: BinaryHeap<(SmallVec<[OrdIdSpan; 1]>, NodeType)> = BinaryHeap::new(); + + fn ids_to_ord_id_spans<'a, D: DagNode + 'a, F: Fn(ID) -> Option>( + ids: &Frontiers, + get: &'a F, + ) -> Option; 1]>> { + let mut ans: SmallVec<[OrdIdSpan<'a>; 1]> = SmallVec::with_capacity(ids.len()); + for id in ids.iter() { + if let Some(node) = OrdIdSpan::from_dag_node(id, get) { + ans.push(node); + } else { + return None; + } + } + + if ans.len() > 1 { + ans.sort_unstable_by(|a, b| b.cmp(a)); + } + + Some(ans) + } + + queue.push((ids_to_ord_id_spans(left, get).unwrap(), NodeType::A)); + queue.push((ids_to_ord_id_spans(right, get).unwrap(), NodeType::B)); + while let Some((mut node, mut node_type)) = queue.pop() { + while let Some((other_node, other_type)) = queue.peek() { + if node == *other_node + || (node.len() == 1 + && other_node.len() == 1 + && node[0].id_last() == other_node[0].id_last()) + { + if node_type != *other_type { + node_type = NodeType::Shared; + } + + queue.pop(); + } else { + break; + } + } + + if queue.is_empty() { + if node_type == NodeType::Shared { + ans = node.into_iter().map(|x| x.id_last()).collect(); + } + + // Iteration is done and no common ancestor is found + // So the ans is empty + is_right_greater = false; + break; + } + + // if node_type is A, then the left side is greater or parallel to the right side + if node_type == NodeType::A { + is_right_greater = false; + } + + if node.len() > 1 { + for node in node.drain(1..node.len()) { + queue.push((smallvec::smallvec![node], node_type)); + } + } + + if let Some(other) = queue.peek() { + if other.0.len() == 1 + && node[0].contains_id(other.0[0].id_last()) + && node_type != other.1 + { + node[0].len = (other.0[0].id_last().counter - node[0].id.counter + 1) as usize; + queue.push((node, node_type)); + continue; + } + + if node[0].len > 1 { + if other.0[0].lamport_last() > node[0].lamport { + node[0].len = (other.0[0].lamport_last() - node[0].lamport) + .min(node[0].len as u32 - 1) as usize; + queue.push((node, node_type)); + continue; + } else { + node[0].len = 1; + queue.push((node, node_type)); + continue; + } + } + } + + if !node[0].deps.is_empty() { + if let Some(deps) = ids_to_ord_id_spans(node[0].deps.as_ref(), get) { + queue.push((deps, node_type)); + } else { + // dep on trimmed history + panic!("deps on trimmed history"); + } + + is_linear = false; + } else { + is_right_greater = false; + break; + } + } + + let mode = if is_right_greater { + if ans.len() <= 1 { + debug_assert_eq!(&ans, left); + } + + if is_linear { + debug_assert!(ans.len() <= 1); + DiffMode::Linear + } else { + DiffMode::ImportGreaterUpdates + } + } else { + DiffMode::Checkout + }; + + (ans, mode) +} + +pub fn remove_included_frontiers(frontiers: &mut VersionVector, new_change_deps: &[ID]) { + for dep in new_change_deps.iter() { + if let Some(last) = frontiers.get_last(dep.peer) { + if last <= dep.counter { + frontiers.remove(&dep.peer); + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + + use super::*; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + len: usize, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + impl Sliceable for TestNode { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id: self.id.inc(from as i32), + len: to - from, + lamport: self.lamport + from as Lamport, + deps: if from == 0 { + self.deps.clone() + } else { + self.id.inc(from as i32 - 1).into() + }, + } + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes + .range(..=id) + .next_back() + .filter(|(_, node)| node.contains_id(id)) + .map(|(_, node)| node.clone()) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.get(id).is_some() + } + } + + fn node( + peer: PeerID, + counter: Counter, + len: usize, + lamport: Lamport, + deps: Frontiers, + ) -> TestNode { + TestNode { + id: ID::new(peer, counter), + len, + lamport, + deps, + } + } + + #[test] + fn common_ancestor_handles_empty_linear_same_span_and_parent_child_cases() { + let first = node(1, 0, 2, 0, Frontiers::default()); + let second = node(1, 2, 2, 2, ID::new(1, 1).into()); + let dag = TestDag::new(vec![first, second], ID::new(1, 3).into()); + + assert_eq!( + dag.find_common_ancestor(&Frontiers::default(), &ID::new(1, 3).into()), + (Frontiers::default(), DiffMode::Linear) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 3).into(), &Frontiers::default()), + (Frontiers::default(), DiffMode::Checkout) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 0).into(), &ID::new(1, 1).into()), + (ID::new(1, 0).into(), DiffMode::Linear) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 1).into(), &ID::new(1, 0).into()), + (ID::new(1, 0).into(), DiffMode::Checkout) + ); + assert_eq!( + dag.find_common_ancestor(&ID::new(1, 1).into(), &ID::new(1, 3).into()), + (ID::new(1, 1).into(), DiffMode::Linear) + ); + } + + #[test] + fn common_ancestor_of_parallel_branches_is_shared_dependency() { + let root = node(1, 0, 1, 0, Frontiers::default()); + let left = node(2, 0, 1, 1, root.id.into()); + let right = node(3, 0, 1, 2, root.id.into()); + let merge = node(4, 0, 1, 3, Frontiers::from([left.id, right.id])); + let dag = TestDag::new( + vec![root.clone(), left.clone(), right.clone(), merge.clone()], + merge.id.into(), + ); + + let (ancestor, mode) = dag.find_common_ancestor(&left.id.into(), &right.id.into()); + assert_eq!(ancestor, root.id.into()); + assert_eq!(mode, DiffMode::Checkout); + + let (ancestor, _) = dag.find_common_ancestor(&root.id.into(), &merge.id.into()); + assert_eq!(ancestor, root.id.into()); + } + + #[test] + fn find_path_reports_forward_and_retreat_spans_for_linear_and_branch_paths() { + let first = node(1, 0, 2, 0, Frontiers::default()); + let second = node(1, 2, 2, 2, ID::new(1, 1).into()); + let side = node(2, 0, 1, 1, ID::new(1, 1).into()); + let dag = TestDag::new( + vec![first, second, side], + Frontiers::from([ID::new(1, 3), ID::new(2, 0)]), + ); + + let forward = dag.find_path(&ID::new(1, 0).into(), &ID::new(1, 3).into()); + assert!(forward.retreat.is_empty()); + assert_eq!(forward.forward.get(&1), Some(&CounterSpan::new(1, 4))); + + let retreat = dag.find_path(&ID::new(1, 3).into(), &ID::new(1, 0).into()); + assert!(retreat.forward.is_empty()); + assert_eq!(retreat.retreat.get(&1), Some(&CounterSpan::new(1, 4))); + + let branch = dag.find_path(&ID::new(2, 0).into(), &ID::new(1, 3).into()); + assert_eq!(branch.retreat.get(&2), Some(&CounterSpan::new(0, 1))); + assert_eq!(branch.forward.get(&1), Some(&CounterSpan::new(2, 4))); + } + + #[test] + fn get_version_vector_walks_dependencies_once_and_tracks_dep_ids_inside_spans() { + let base = node(1, 0, 3, 0, Frontiers::default()); + let left = node(2, 0, 1, 4, ID::new(1, 1).into()); + let right = node(3, 0, 1, 5, ID::new(1, 2).into()); + let merge = node(4, 0, 1, 6, Frontiers::from([left.id, right.id])); + let dag = TestDag::new(vec![base, left, right, merge.clone()], merge.id.into()); + + let vv = dag.get_vv(merge.id); + assert_eq!(vv.get_last(1), Some(2)); + assert_eq!(vv.get_last(2), Some(0)); + assert_eq!(vv.get_last(3), Some(0)); + assert_eq!(vv.get_last(4), Some(0)); + } + + #[test] + fn remove_included_frontiers_drops_only_dependencies_at_or_after_recorded_frontiers() { + let mut vv = VersionVector::default(); + vv.set_last(ID::new(1, 3)); + vv.set_last(ID::new(2, 4)); + vv.set_last(ID::new(3, 1)); + + remove_included_frontiers(&mut vv, &[ID::new(1, 3), ID::new(2, 2), ID::new(4, 0)]); + + assert_eq!(vv.get_last(1), None); + assert_eq!(vv.get_last(2), Some(4)); + assert_eq!(vv.get_last(3), Some(1)); + } + + #[test] + fn ord_id_span_orders_by_last_lamport_peer_and_shorter_overlapping_span() { + let short = OrdIdSpan { + id: ID::new(1, 1), + lamport: 1, + len: 1, + deps: Cow::Owned(Frontiers::default()), + }; + let long = OrdIdSpan { + id: ID::new(1, 0), + lamport: 0, + len: 2, + deps: Cow::Owned(Frontiers::default()), + }; + let later_peer = OrdIdSpan { + id: ID::new(2, 0), + lamport: 1, + len: 1, + deps: Cow::Owned(Frontiers::default()), + }; + + assert!(short > long); + assert!(later_peer > short); + assert_eq!(long.get_min().len, 1); + } +} diff --git a/vendor/loro-internal/src/dag/iter.rs b/vendor/loro-internal/src/dag/iter.rs new file mode 100644 index 00000000..4c824b5b --- /dev/null +++ b/vendor/loro-internal/src/dag/iter.rs @@ -0,0 +1,631 @@ +use crate::version::Frontiers; +use num::Zero; +use std::{collections::BTreeMap, ops::Range}; + +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct IdHeapItem { + id: ID, + lamport: Lamport, +} + +impl PartialOrd for IdHeapItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for IdHeapItem { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport.cmp(&other.lamport).reverse() + } +} + +#[allow(dead_code)] +pub(crate) fn iter_dag_with_vv>(dag: &D) -> DagIteratorVV<'_, T> { + DagIteratorVV { + dag, + vv_map: Default::default(), + heap: BinaryHeap::new(), + } +} + +#[allow(dead_code)] +pub(crate) fn iter_dag(dag: &dyn Dag) -> DagIterator<'_, T> { + DagIterator { + dag, + visited: VersionVector::new(), + heap: BinaryHeap::new(), + } +} + +pub struct DagIterator<'a, T> { + dag: &'a dyn Dag, + /// Because all deps' lamports are smaller than current node's lamport. + /// We can use the lamport to sort the nodes so that each node's deps are processed before itself. + /// + /// The ids in this heap are start ids of nodes. It won't be a id pointing to the middle of a node. + heap: BinaryHeap, + visited: VersionVector, +} + +/// Should only use it on debug, because it's slow and likely to use lots of mem +impl Iterator for DagIterator<'_, T> { + type Item = T; + + fn next(&mut self) -> Option { + if self.visited.is_empty() { + if self.dag.vv().is_empty() { + return None; + } + + for (&client_id, _) in self.dag.vv().iter() { + if let Some(node) = self.dag.get(ID::new(client_id, 0)) { + self.heap.push(IdHeapItem { + id: ID::new(client_id, 0), + lamport: node.lamport(), + }); + } + + self.visited.insert(client_id, 0); + } + } + + if !self.heap.is_empty() { + let item = self.heap.pop().unwrap(); + let id = item.id; + let node = self.dag.get(id).unwrap(); + // push next node from the same client to the heap + let next_id = id.inc(node.content_len() as i32); + if self.dag.contains(next_id) { + let next_node = self.dag.get(next_id).unwrap(); + self.heap.push(IdHeapItem { + id: next_id, + lamport: next_node.lamport(), + }); + } + + return Some(node); + } + + None + } +} + +pub(crate) struct DagIteratorVV<'a, T> { + dag: &'a dyn Dag, + /// we should keep every nodes starting id inside this map + vv_map: FxHashMap, + /// Because all deps' lamports are smaller than current node's lamport. + /// We can use the lamport to sort the nodes so that each node's deps are processed before itself. + /// + /// The ids in this heap are start ids of nodes. It won't be a id pointing to the middle of a node. + heap: BinaryHeap, +} + +/// Should only use it on debug, because it's slow and likely to use lots of mem +impl Iterator for DagIteratorVV<'_, T> { + type Item = (T, VersionVector); + + fn next(&mut self) -> Option { + if self.vv_map.is_empty() { + if self.dag.vv().is_empty() { + return None; + } + + for (&client_id, _) in self.dag.vv().iter() { + let vv = VersionVector::new(); + if let Some(node) = self.dag.get(ID::new(client_id, 0)) { + if node.lamport() == 0 { + self.vv_map.insert(ID::new(client_id, 0), vv.clone()); + } + + self.heap.push(IdHeapItem { + id: ID::new(client_id, 0), + lamport: node.lamport(), + }); + } + } + } + + if !self.heap.is_empty() { + let item = self.heap.pop().unwrap(); + let id = item.id; + let node = self.dag.get(id).unwrap(); + debug_assert_eq!(id, node.id_start()); + let mut vv = { + // calculate vv + let mut vv: Option = None; + for dep_id in node.deps().iter() { + let dep = self.dag.get(dep_id).unwrap(); + let dep_vv = self.vv_map.get(&dep.id_start()).unwrap(); + if let Some(vv) = vv.as_mut() { + vv.merge(dep_vv); + } else { + vv = Some(dep_vv.clone()); + } + + if dep.id_start() != dep_id { + vv.as_mut().unwrap().set_last(dep_id); + } + } + + vv.unwrap_or_default() + }; + + vv.try_update_last(id); + self.vv_map.insert(id, vv.clone()); + + // push next node from the same client to the heap + let next_id = id.inc(node.content_len() as i32); + if self.dag.contains(next_id) { + let next_node = self.dag.get(next_id).unwrap(); + self.heap.push(IdHeapItem { + id: next_id, + lamport: next_node.lamport(), + }); + } + + return Some((node, vv)); + } + + None + } +} + +/// Visit every span in the target IdSpanVector. +/// It's guaranteed that the spans are visited in causal order, and each span is visited only once. +/// When visiting a span, we will checkout to the version where the span was created +pub(crate) struct DagCausalIter<'a, Dag> { + dag: &'a Dag, + frontier: Frontiers, + target: IdSpanVector, + /// how many dependencies are inside target for each id + out_degrees: FxHashMap, + succ: BTreeMap>, + stack: Vec, +} + +#[derive(Debug)] +pub(crate) struct IterReturn { + /// data is a reference, it need to be sliced by the counter_range to get the underlying data + pub data: T, + /// data[slice] is the data we want to return + #[allow(unused)] + pub slice: Range, +} + +impl<'a, T: DagNode, D: Dag + Debug> DagCausalIter<'a, D> { + pub fn new(dag: &'a D, from: Frontiers, target: IdSpanVector) -> Self { + let mut out_degrees: FxHashMap = FxHashMap::default(); + let mut succ: BTreeMap> = BTreeMap::default(); + let mut stack = Vec::new(); + let mut q = vec![]; + for id in target.iter() { + if id.1.content_len() > 0 { + let id = id.id_start(); + q.push(id); + } + } + + // traverse all nodes, calculate the out_degrees + // if out_degree is 0, then it can be iterated directly + while let Some(id) = q.pop() { + let client = id.peer; + let node = dag.get(id).unwrap(); + let deps = node.deps(); + out_degrees.insert( + id, + deps.iter() + .filter(|&dep| { + if let Some(span) = target.get(&dep.peer) { + let included_in_target = + dep.counter >= span.min() && dep.counter <= span.max(); + if included_in_target { + succ.entry(dep).or_default().push(id); + } + included_in_target + } else { + false + } + }) + .count(), + ); + let target_span = target.get(&client).unwrap(); + let last_counter = node.id_last().counter; + if target_span.max() > last_counter { + q.push(ID::new(client, last_counter + 1)) + } + } + + // Enforce per-peer counter order. Within a single peer, the iterator + // must visit tracked node-starts in ascending counter order even when + // the nodes are concurrent (i.e. don't declare each other as deps). + // Without this, two same-peer nodes can both have zero in-degree and + // end up on the stack at once; the LIFO pop order would then violate + // the iterator's invariant that each popped id equals the current + // target_span.min() for its peer. + let mut by_peer: FxHashMap> = FxHashMap::default(); + for id in out_degrees.keys() { + by_peer.entry(id.peer).or_default().push(id.counter); + } + for (peer, mut counters) in by_peer { + if counters.len() < 2 { + continue; + } + counters.sort_unstable(); + for pair in counters.windows(2) { + let prev = ID::new(peer, pair[0]); + let curr = ID::new(peer, pair[1]); + if let Some(deg) = out_degrees.get_mut(&curr) { + *deg += 1; + } + succ.entry(prev).or_default().push(curr); + } + } + + out_degrees.retain(|k, v| { + if v.is_zero() { + stack.push(*k); + return false; + } + true + }); + + Self { + dag, + frontier: from, + target, + out_degrees, + succ, + stack, + } + } +} + +impl> Iterator for DagCausalIter<'_, D> { + type Item = IterReturn; + + fn next(&mut self) -> Option { + loop { + if self.stack.is_empty() { + debug_assert_eq!( + 0, + self.target + .iter() + .map(|x| x.1.content_len() as i32) + .sum::() + ); + return None; + } + + let node_id = self.stack.pop().unwrap(); + let (node, slice_from, slice_end, next_same_peer) = { + let target_span = self.target.get_mut(&node_id.peer).unwrap(); + if node_id.counter != target_span.min() { + debug_assert!( + node_id.counter < target_span.min(), + "{} {:?}", + node_id, + target_span + ); + continue; + } + + // node_id may point into the middle of the node, we need to slice + let node = self.dag.get(node_id).unwrap(); + // node start_id may be smaller than node_id + let counter = node.id_span().counter; + let slice_from = if counter.start < target_span.start { + target_span.start - counter.start + } else { + 0 + }; + let slice_end = if counter.end < target_span.end { + counter.end - counter.start + } else { + target_span.end - counter.start + }; + + if slice_end <= slice_from { + debug_assert_eq!(slice_end, slice_from, "{node_id} {:?}", target_span); + continue; + } + + let consumed_last_counter = counter.start + slice_end - 1; + target_span.set_start(consumed_last_counter + 1); + let next_same_peer = if target_span.content_len() > 0 { + Some(ID::new(node_id.peer, target_span.min())) + } else { + None + }; + + (node, slice_from, slice_end, next_same_peer) + }; + + if let Some(next_id) = next_same_peer { + if !self.out_degrees.contains_key(&next_id) && !self.stack.contains(&next_id) { + self.stack.push(next_id); + } + } + + // NOTE: we expect user to update the tracker, to apply node, after visiting the node + self.frontier = Frontiers::from_id(node.id_start().inc(slice_end - 1)); + + let current_peer = node_id.peer; + let mut keys = Vec::new(); + let mut heap = BinaryHeap::new(); + // The in-degree of the successor node minus 1, and if it becomes 0, it is added to the heap + for (key, succ) in self.succ.range(node.id_start()..node.id_end()) { + keys.push(*key); + for succ_id in succ.iter() { + self.out_degrees.entry(*succ_id).and_modify(|i| *i -= 1); + if let Some(in_degree) = self.out_degrees.get(succ_id) { + if in_degree.is_zero() { + heap.push((succ_id.peer != current_peer, *succ_id)); + self.out_degrees.remove(succ_id); + } + } + } + } + // Nodes that have been traversed are removed from the graph to avoid being covered by other node ranges again + keys.into_iter().for_each(|k| { + self.succ.remove(&k); + }); + while let Some(id) = heap.pop() { + self.stack.push(id.1) + } + + return Some(IterReturn { + data: node, + slice: slice_from..slice_end, + }); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + use crate::{ + change::Lamport, + id::ID, + span::{CounterSpan, HasId, HasLamport}, + version::VersionVector, + }; + use rle::{HasLength, Sliceable}; + + #[derive(Debug, Clone)] + struct DummyNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Frontiers, + } + + impl DagNode for DummyNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl Sliceable for DummyNode { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id: self.id.inc(from as i32), + lamport: self.lamport + from as Lamport, + len: to - from, + deps: if from > 0 { + self.id.inc(from as i32 - 1).into() + } else { + self.deps.clone() + }, + } + } + } + + impl HasLamport for DummyNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasId for DummyNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLength for DummyNode { + fn content_len(&self) -> usize { + self.len + } + } + + #[derive(Debug)] + struct DummyDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl Dag for DummyDag { + type Node = DummyNode; + + fn get(&self, id: ID) -> Option { + self.nodes + .range(..=id) + .next_back() + .filter(|(_, node)| node.contains_id(id)) + .map(|(_, node)| node.clone()) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.nodes + .range(..=id) + .next_back() + .is_some_and(|(_, node)| node.contains_id(id)) + } + } + + #[test] + fn stale_iterator_state_repairs_missing_same_peer_continuation() { + let first = DummyNode { + id: ID::new(1, 0), + lamport: 0, + len: 5, + deps: Frontiers::default(), + }; + let second = DummyNode { + id: ID::new(1, 5), + lamport: 5, + len: 5, + deps: ID::new(1, 4).into(), + }; + let mut vv = VersionVector::default(); + vv.set_end(second.id_end()); + let dag = DummyDag { + frontier: second.id_last().into(), + nodes: [(first.id_start(), first), (second.id_start(), second)] + .into_iter() + .collect(), + vv, + }; + + let mut target = IdSpanVector::default(); + target.insert(1, CounterSpan::new(0, 7)); + let mut iter = DagCausalIter { + dag: &dag, + frontier: Frontiers::default(), + target, + out_degrees: Default::default(), + succ: BTreeMap::default(), + // This models the stale iterator state after `new()` saw one large + // node and queued only the first peer segment. + stack: vec![ID::new(1, 0)], + }; + + let first = iter.next().expect("first segment should be yielded"); + assert_eq!(first.data.id_start(), ID::new(1, 0)); + assert_eq!(first.slice, 0..5); + + let second = iter.next().expect("second segment should be re-queued"); + assert_eq!(second.data.id_start(), ID::new(1, 5)); + assert_eq!(second.slice, 0..2); + + assert!(iter.next().is_none()); + } + + /// Regression: before the fix, `DagCausalIter::new` would push both + /// `(peer=1, counter=0)` and `(peer=1, counter=5)` onto the initial + /// stack because neither node has any *in-target* dependency. The LIFO + /// pop would then surface `(peer=1, counter=5)` first while + /// `target_span.min()` is still `0`, tripping the causal-order + /// invariant (historically: "slice_end > slice_from" / "counter < + /// target_span.min()"). With the fix the iter synthesizes a per-peer + /// ordering edge so the second node is only released after the first is + /// consumed. + #[test] + fn two_concurrent_same_peer_nodes_visit_in_counter_order() { + let first = DummyNode { + id: ID::new(1, 0), + lamport: 10, + len: 5, + deps: Frontiers::default(), + }; + // `second` is concurrent with `first` from peer 1's perspective: + // its only dep lives outside the target span (peer 2), so the BFS + // records zero in-degree and would historically race to the stack + // alongside `first`. + let second = DummyNode { + id: ID::new(1, 5), + lamport: 20, + len: 5, + deps: ID::new(2, 0).into(), + }; + let side = DummyNode { + id: ID::new(2, 0), + lamport: 0, + len: 1, + deps: Frontiers::default(), + }; + + let mut vv = VersionVector::default(); + vv.set_end(first.id_end()); + vv.set_end(second.id_end()); + vv.set_end(side.id_end()); + + let dag = DummyDag { + frontier: second.id_last().into(), + nodes: [ + (first.id_start(), first), + (second.id_start(), second), + (side.id_start(), side), + ] + .into_iter() + .collect(), + vv, + }; + + // Target only peer 1. `second`'s dep on (peer 2, 0) is thus + // *outside* the target vector — the exact condition that made both + // same-peer nodes reach the stack concurrently before the fix. + let mut target = IdSpanVector::default(); + target.insert(1, CounterSpan::new(0, 10)); + + let mut iter = DagCausalIter::new(&dag, Frontiers::default(), target); + + let a = iter.next().expect("first peer-1 segment"); + assert_eq!(a.data.id_start(), ID::new(1, 0)); + assert_eq!(a.slice, 0..5); + + let b = iter.next().expect("second peer-1 segment"); + assert_eq!(b.data.id_start(), ID::new(1, 5)); + assert_eq!(b.slice, 0..5); + + assert!(iter.next().is_none()); + } + + #[test] + fn larger_node_only_advances_consumed_slice() { + let node = DummyNode { + id: ID::new(1, 0), + lamport: 0, + len: 10, + deps: Frontiers::default(), + }; + let mut vv = VersionVector::default(); + vv.set_end(node.id_end()); + let dag = DummyDag { + frontier: node.id_last().into(), + nodes: [(node.id_start(), node)].into_iter().collect(), + vv, + }; + + let mut target = IdSpanVector::default(); + target.insert(1, CounterSpan::new(0, 7)); + let mut iter = DagCausalIter { + dag: &dag, + frontier: Frontiers::default(), + target, + out_degrees: Default::default(), + succ: BTreeMap::default(), + stack: vec![ID::new(1, 0)], + }; + + let item = iter.next().expect("target slice should be yielded"); + assert_eq!(item.slice, 0..7); + assert_eq!(iter.target.get(&1).unwrap().content_len(), 0); + } +} diff --git a/vendor/loro-internal/src/dag/mermaid.rs b/vendor/loro-internal/src/dag/mermaid.rs new file mode 100644 index 00000000..f8d7f403 --- /dev/null +++ b/vendor/loro-internal/src/dag/mermaid.rs @@ -0,0 +1,320 @@ +use super::*; +#[allow(dead_code)] +struct BreakPoints { + break_points: FxHashMap>, + /// start ID to ID. The target ID may be in the middle of an op. + /// + /// only includes links across different clients + links: FxHashMap>, +} + +#[allow(dead_code)] +struct Output { + clients: FxHashMap>, + /// start ID to start ID. + /// + /// only includes links across different clients + links: FxHashMap>, +} + +#[allow(dead_code)] +fn to_str(output: Output) -> String { + let mut s = String::new(); + let mut indent_level = 0; + macro_rules! new_line { + () => { + s += "\n"; + for _ in 0..indent_level { + s += " "; + } + }; + } + s += "flowchart RL"; + indent_level += 1; + new_line!(); + for (client_id, spans) in output.clients.iter() { + s += format!("subgraph peer{client_id}").as_str(); + new_line!(); + let mut is_first = true; + for id_span in spans.iter().rev() { + if !is_first { + s += " --> " + } + + is_first = false; + s += format!( + "{}-{}(\"c{}: [{}, {})\")", + id_span.peer, + id_span.counter.start, + id_span.peer, + id_span.counter.start, + id_span.counter.end, + ) + .as_str(); + } + + new_line!(); + s += "end"; + new_line!(); + new_line!(); + } + + for (id_from, id_tos) in output.links.iter() { + for id_to in id_tos.iter() { + s += format!( + "{}-{} --> {}-{}", + id_from.peer, id_from.counter, id_to.peer, id_to.counter + ) + .as_str(); + new_line!(); + } + } + + s +} + +#[allow(dead_code)] +fn break_points_to_output(input: BreakPoints) -> Output { + let mut output = Output { + clients: FxHashMap::default(), + links: FxHashMap::default(), + }; + let breaks: FxHashMap> = input + .break_points + .into_iter() + .map(|(client_id, set)| { + let mut vec: Vec = set.iter().cloned().collect(); + vec.sort(); + (client_id, vec) + }) + .collect(); + for (client_id, break_points) in breaks.iter() { + let mut spans = Vec::with_capacity(break_points.len()); + for (from, to) in break_points.iter().zip(break_points.iter().skip(1)) { + spans.push(IdSpan::new(*client_id, *from, *to)); + } + output.clients.insert(*client_id, spans); + } + + for (id_from, id_tos) in input.links.iter() { + for id_to in id_tos { + let client_breaks = breaks.get(&id_to.peer).unwrap(); + match client_breaks.binary_search(&id_to.counter) { + Ok(_) => { + output.links.entry(*id_from).or_default().push(*id_to); + } + Err(index) => { + output + .links + .entry(*id_from) + .or_default() + .push(ID::new(id_to.peer, client_breaks[index - 1])); + } + } + } + } + output +} + +#[allow(dead_code)] +fn get_dag_break_points(dag: &impl Dag) -> BreakPoints { + let mut break_points = BreakPoints { + break_points: FxHashMap::default(), + links: FxHashMap::default(), + }; + + for node in dag.iter() { + let id = node.id_start(); + let set = break_points.break_points.entry(id.peer).or_default(); + set.insert(id.counter); + set.insert(id.counter + node.content_len() as Counter); + for dep in node.deps().iter() { + if dep.peer == id.peer { + continue; + } + + break_points + .break_points + .entry(dep.peer) + .or_default() + .insert(dep.counter); + break_points.links.entry(id).or_default().push(dep); + } + } + + break_points +} + +#[allow(dead_code)] +pub(crate) fn dag_to_mermaid(dag: &impl Dag) -> String { + to_str(break_points_to_output(get_dag_break_points(dag))) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use loro_common::{HasId, HasIdSpan}; + use rle::{HasLength, Sliceable}; + + use super::*; + + #[derive(Clone, Debug)] + struct TestNode { + id: ID, + len: usize, + lamport: Lamport, + deps: Frontiers, + } + + impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } + } + + impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } + } + + impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } + } + + impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } + } + + impl Sliceable for TestNode { + fn slice(&self, _from: usize, _to: usize) -> Self { + self.clone() + } + } + + #[derive(Debug)] + struct TestDag { + nodes: BTreeMap, + vv: VersionVector, + frontier: Frontiers, + } + + impl TestDag { + fn new(nodes: impl IntoIterator, frontier: Frontiers) -> Self { + let mut vv = VersionVector::default(); + let nodes = nodes + .into_iter() + .map(|node| { + vv.set_end(node.id_end()); + (node.id_start(), node) + }) + .collect(); + Self { + nodes, + vv, + frontier, + } + } + } + + impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + self.nodes + .range(..=id) + .next_back() + .filter(|(_, node)| node.contains_id(id)) + .map(|(_, node)| node.clone()) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.get(id).is_some() + } + } + + fn node( + peer: PeerID, + counter: Counter, + len: usize, + lamport: Lamport, + deps: Frontiers, + ) -> TestNode { + TestNode { + id: ID::new(peer, counter), + len, + lamport, + deps, + } + } + + #[test] + fn break_points_to_output_splits_spans_and_retargets_middle_links_to_span_starts() { + let mut break_points = FxHashMap::default(); + break_points.insert(1, FxHashSet::from_iter([0, 3])); + break_points.insert(2, FxHashSet::from_iter([0, 5, 10])); + let from = ID::new(1, 0); + let mut links = FxHashMap::default(); + links.insert(from, vec![ID::new(2, 7)]); + + let output = break_points_to_output(BreakPoints { + break_points, + links, + }); + + assert_eq!(output.clients.get(&1).unwrap(), &vec![IdSpan::new(1, 0, 3)]); + assert_eq!( + output.clients.get(&2).unwrap(), + &vec![IdSpan::new(2, 0, 5), IdSpan::new(2, 5, 10)] + ); + assert_eq!(output.links.get(&from).unwrap(), &vec![ID::new(2, 5)]); + } + + #[test] + fn dag_to_mermaid_includes_peer_subgraphs_split_spans_and_cross_peer_edges() { + let first = node(1, 0, 2, 0, Frontiers::default()); + let second = node(2, 0, 1, 3, ID::new(1, 1).into()); + let dag = TestDag::new(vec![first, second], ID::new(2, 0).into()); + + let graph = dag_to_mermaid(&dag); + + assert!(graph.starts_with("flowchart RL")); + assert!(graph.contains("subgraph peer1")); + assert!(graph.contains("subgraph peer2")); + assert!(graph.contains("1-0(\"c1: [0, 1)\")")); + assert!(graph.contains("1-1(\"c1: [1, 2)\")")); + assert!(graph.contains("2-0(\"c2: [0, 1)\")")); + assert!(graph.contains("2-0 --> 1-1")); + } + + #[test] + fn to_str_renders_all_links_without_requiring_hash_map_order() { + let output = Output { + clients: FxHashMap::from_iter([ + (1, vec![IdSpan::new(1, 0, 1), IdSpan::new(1, 1, 2)]), + (2, vec![IdSpan::new(2, 0, 1)]), + ]), + links: FxHashMap::from_iter([(ID::new(2, 0), vec![ID::new(1, 0), ID::new(1, 1)])]), + }; + + let graph = to_str(output); + + assert!(graph.contains("subgraph peer1")); + assert!(graph.contains("1-1(\"c1: [1, 2)\") --> 1-0(\"c1: [0, 1)\")")); + assert!(graph.contains("2-0 --> 1-0")); + assert!(graph.contains("2-0 --> 1-1")); + } +} diff --git a/vendor/loro-internal/src/dag/test.excalidraw b/vendor/loro-internal/src/dag/test.excalidraw new file mode 100644 index 00000000..eb90d196 --- /dev/null +++ b/vendor/loro-internal/src/dag/test.excalidraw @@ -0,0 +1,4334 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://marketplace.visualstudio.com/items?itemName=pomdtr.excalidraw-editor", + "elements": [ + { + "type": "text", + "version": 101, + "versionNonce": 883063978, + "isDeleted": false, + "id": "lhg64zlt-7Bh50s_bwPDh", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 263.48630115928825, + "y": -113.63915404458658, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 247, + "height": 25, + "seed": 127706155, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Common Ancestors Tests", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Common Ancestors Tests" + }, + { + "type": "rectangle", + "version": 315, + "versionNonce": 978352886, + "isDeleted": false, + "id": "UFtnx7sfhV0uKBPc8bkjR", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 234.87301990928825, + "y": 593.3803772054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 56.71875, + "height": 53.90625, + "seed": 771228869, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "DvFA3B6P4HJIrrYLNfwWC", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 308, + "versionNonce": 759256938, + "isDeleted": false, + "id": "ndstLpUvl8l8gieSJBPWM", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 236.31442615928825, + "y": 492.0014709554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 51.8828125, + "height": 48.4921875, + "seed": 411378725, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "DvFA3B6P4HJIrrYLNfwWC", + "type": "arrow" + }, + { + "id": "lN2WY1kuFxjMAVkpFsGwW", + "type": "arrow" + }, + { + "id": "RU8Wyhtc02QcaaG6pQTMJ", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 532, + "versionNonce": 1135717430, + "isDeleted": false, + "id": "DvFA3B6P4HJIrrYLNfwWC", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 262.53317615928825, + "y": 547.0131897054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 1, + "height": 39.765625, + "seed": 653744005, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "ndstLpUvl8l8gieSJBPWM", + "focus": -0.03958466448687474, + "gap": 6.51953125 + }, + "endBinding": { + "elementId": "UFtnx7sfhV0uKBPc8bkjR", + "focus": -0.08757842935147449, + "gap": 6.6015625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -1, + 39.765625 + ] + ] + }, + { + "type": "rectangle", + "version": 245, + "versionNonce": 1253497386, + "isDeleted": false, + "id": "URqECeM4WGZqAFI0abbR6", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 237.63473865928825, + "y": 382.3725647054134, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 52.140625, + "height": 43.83203125, + "seed": 1133256421, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "lN2WY1kuFxjMAVkpFsGwW", + "type": "arrow" + }, + { + "id": "rsU49QVI8HgyJBtjILKZT", + "type": "arrow" + }, + { + "id": "4Xhxr95Fo3KM0W5_NblnS", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 551, + "versionNonce": 73053558, + "isDeleted": false, + "id": "lN2WY1kuFxjMAVkpFsGwW", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "x": 266.18161365928825, + "y": 434.3569397054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 3.11328125, + "height": 51.01953125, + "seed": 886110789, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "URqECeM4WGZqAFI0abbR6", + "focus": -0.15730547947414078, + "gap": 8.15234375 + }, + "endBinding": { + "elementId": "ndstLpUvl8l8gieSJBPWM", + "focus": -0.03906854855773698, + "gap": 6.625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3.11328125, + 51.01953125 + ] + ] + }, + { + "type": "rectangle", + "version": 229, + "versionNonce": 200148202, + "isDeleted": false, + "id": "WtyUQLscbyCgdCnYK_jNr", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 333.02536365928825, + "y": 382.2006897054134, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 46.50390625, + "height": 47.48046875, + "seed": 329476517, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "RU8Wyhtc02QcaaG6pQTMJ", + "type": "arrow" + }, + { + "id": "ceD-EHtPmAHrld2ReqW1d", + "type": "arrow" + }, + { + "id": "gi633X0Z9XWBOZyYzVt0P", + "type": "arrow" + }, + { + "id": "aGZg71ALGFz6MuRal-TTz", + "type": "arrow" + }, + { + "id": "EE1SzT8racuV0SEacrH5H", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 214, + "versionNonce": 1972410038, + "isDeleted": false, + "id": "lSqDDXX9cdXC4LCluuQmS", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 419.02926990928825, + "y": 382.3374084554134, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 50.58203125, + "height": 43.453125, + "seed": 19983621, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "KpyIcLVhnWpIY6J9Q8M3s", + "type": "arrow" + }, + { + "id": "Th-vLn9JQqbE80X8Pl1-U", + "type": "arrow" + }, + { + "id": "OzG5v9FzTWSKCdywOoGQL", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 456, + "versionNonce": 1517410218, + "isDeleted": false, + "id": "RU8Wyhtc02QcaaG6pQTMJ", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 325.62692615928825, + "y": 435.4702209554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 47.83203125, + "height": 47.4453125, + "seed": 485950565, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.01865831404761865, + "gap": 7.3984375 + }, + "endBinding": { + "elementId": "ndstLpUvl8l8gieSJBPWM", + "focus": -0.358531455976611, + "gap": 9.0859375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -47.83203125, + 47.4453125 + ] + ] + }, + { + "type": "arrow", + "version": 420, + "versionNonce": 536367094, + "isDeleted": false, + "id": "ceD-EHtPmAHrld2ReqW1d", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 345.79098865928825, + "y": 438.3452209554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 20.67578125, + "height": 99.421875, + "seed": 985870277, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.6110590696052777, + "gap": 8.6640625 + }, + "endBinding": { + "elementId": "1Swh32YEcsSztjU6TVYMO", + "focus": 0.1403757111100622, + "gap": 6.3203125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 20.67578125, + 99.421875 + ] + ] + }, + { + "type": "rectangle", + "version": 191, + "versionNonce": 913134186, + "isDeleted": false, + "id": "1Swh32YEcsSztjU6TVYMO", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 347.37692615928825, + "y": 544.0874084554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 43.8828125, + "height": 51.65625, + "seed": 1488121637, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "ceD-EHtPmAHrld2ReqW1d", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 181, + "versionNonce": 798417206, + "isDeleted": false, + "id": "7rpEPw0XEWSpd4zpGIL-i", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 456.69723865928825, + "y": 555.2280334554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.53125, + "height": 48.55078125, + "seed": 1429455493, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "gi633X0Z9XWBOZyYzVt0P", + "type": "arrow" + }, + { + "id": "KpyIcLVhnWpIY6J9Q8M3s", + "type": "arrow" + } + ], + "updated": 1666058888070, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 441, + "versionNonce": 1655403818, + "isDeleted": false, + "id": "gi633X0Z9XWBOZyYzVt0P", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 371.72848865928825, + "y": 442.6420959554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 115.453125, + "height": 106.890625, + "seed": 1694706149, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888070, + "link": null, + "locked": false, + "startBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.49474380369903664, + "gap": 12.9609375 + }, + "endBinding": { + "elementId": "7rpEPw0XEWSpd4zpGIL-i", + "focus": 1.0023527144670679, + "gap": 5.6953125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 115.453125, + 106.890625 + ] + ] + }, + { + "type": "arrow", + "version": 435, + "versionNonce": 131156598, + "isDeleted": false, + "id": "KpyIcLVhnWpIY6J9Q8M3s", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 444.05661365928825, + "y": 433.2905334554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.18359375, + "height": 109.609375, + "seed": 1227793733, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "lSqDDXX9cdXC4LCluuQmS", + "focus": 0.30532272017379974, + "gap": 7.5 + }, + "endBinding": { + "elementId": "7rpEPw0XEWSpd4zpGIL-i", + "focus": 0.6606224867711618, + "gap": 12.328125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 36.18359375, + 109.609375 + ] + ] + }, + { + "type": "rectangle", + "version": 442, + "versionNonce": 278347754, + "isDeleted": false, + "id": "CFSGkbwHAPWwmUPvyrNSU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 246.38864490928825, + "y": 194.93506470541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 41.6796875, + "height": 41.82421874999999, + "seed": 1158919333, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "aGZg71ALGFz6MuRal-TTz", + "type": "arrow" + }, + { + "id": "WCsuKPIiyyv46Ci9BE954", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 201, + "versionNonce": 1466731446, + "isDeleted": false, + "id": "PX8R3GWFK-y4FAPfGXDWQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 241.00192615928825, + "y": -23.361810294586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 1415567365, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "vYg0q3zocHglNFtQ-Rzfb", + "type": "text" + }, + { + "id": "vYg0q3zocHglNFtQ-Rzfb", + "type": "text" + }, + { + "id": "g1I2bmNZBEIbE5bPbc9PO", + "type": "arrow" + }, + { + "type": "text", + "id": "vYg0q3zocHglNFtQ-Rzfb" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 56, + "versionNonce": 205238954, + "isDeleted": false, + "id": "vYg0q3zocHglNFtQ-Rzfb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 246.00192615928825, + "y": -13.908685294586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 554733413, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "PX8R3GWFK-y4FAPfGXDWQ", + "originalText": "A" + }, + { + "type": "rectangle", + "version": 259, + "versionNonce": 747480310, + "isDeleted": false, + "id": "wkdtJfpB-XRjEMes0NHIG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 338.02926990928825, + "y": -23.916497794586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 963861189, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "B0jO-HvbxLWHssI5X1XjY", + "type": "text" + }, + { + "id": "X8GhhoUwWv39DpajqYRWn", + "type": "arrow" + }, + { + "type": "text", + "id": "B0jO-HvbxLWHssI5X1XjY" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 117, + "versionNonce": 1749273962, + "isDeleted": false, + "id": "B0jO-HvbxLWHssI5X1XjY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 343.02926990928825, + "y": -13.463372794586576, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 103135781, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "wkdtJfpB-XRjEMes0NHIG", + "originalText": "B" + }, + { + "type": "arrow", + "version": 305, + "versionNonce": 146380342, + "isDeleted": false, + "id": "aGZg71ALGFz6MuRal-TTz", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 267.8641585018744, + "y": 249.68115845541348, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 88.60298159582703, + "height": 125.39453124999994, + "seed": 1387909, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "CFSGkbwHAPWwmUPvyrNSU", + "focus": 0.6533881812518985, + "gap": 12.921875000000057 + }, + "endBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.5496088514101133, + "gap": 7.125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 88.60298159582703, + 125.39453124999994 + ] + ] + }, + { + "type": "arrow", + "version": 364, + "versionNonce": 704286762, + "isDeleted": false, + "id": "Th-vLn9JQqbE80X8Pl1-U", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 285.62692615928825, + "y": 148.7884978967083, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 155.87566467544525, + "height": 226.02937930870513, + "seed": 645870821, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": -0.3630659075617793, + "gap": 5.17578125 + }, + "endBinding": { + "elementId": "lSqDDXX9cdXC4LCluuQmS", + "focus": 0.4308256364516943, + "gap": 7.51953125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 155.87566467544525, + 226.02937930870513 + ] + ] + }, + { + "type": "rectangle", + "version": 231, + "versionNonce": 1960140662, + "isDeleted": false, + "id": "hWhmUfN-whJHDhqK7HkDw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 240.19333240928825, + "y": 108.20459595541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.2578125, + "height": 45.4921875, + "seed": 888229957, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "WCsuKPIiyyv46Ci9BE954", + "type": "arrow" + }, + { + "id": "Th-vLn9JQqbE80X8Pl1-U", + "type": "arrow" + }, + { + "id": "g1I2bmNZBEIbE5bPbc9PO", + "type": "arrow" + }, + { + "id": "SJqqF6rk1YusFfgDKDZIX", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 452, + "versionNonce": 1967848170, + "isDeleted": false, + "id": "WCsuKPIiyyv46Ci9BE954", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 268.16418593279104, + "y": 159.16553345541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.11351614941020216, + "height": 32.2109375, + "seed": 424214437, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": -0.38312080421234723, + "gap": 5.46875 + }, + "endBinding": { + "elementId": "CFSGkbwHAPWwmUPvyrNSU", + "focus": 0.05429266252944664, + "gap": 3.55859375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 0.11351614941020216, + 32.2109375 + ] + ] + }, + { + "type": "rectangle", + "version": 94, + "versionNonce": 354812086, + "isDeleted": false, + "id": "sHYBFDblul5QqJjxXun--", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 369.89255115928825, + "y": 81.69287720541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.76953125, + "height": 44.6484375, + "seed": 699646725, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "EE1SzT8racuV0SEacrH5H", + "type": "arrow" + }, + { + "id": "X8GhhoUwWv39DpajqYRWn", + "type": "arrow" + }, + { + "id": "T2wLMX7gk4q4Wlq65lvOL", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 71, + "versionNonce": 952110506, + "isDeleted": false, + "id": "MIEh2o7onmi2XVMBPNwtl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 486.58786365928825, + "y": 145.29443970541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 50.74609375, + "height": 53.9140625, + "seed": 447683173, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "rsU49QVI8HgyJBtjILKZT", + "type": "arrow" + }, + { + "id": "T2wLMX7gk4q4Wlq65lvOL", + "type": "arrow" + }, + { + "id": "EtBC7WdsIkjxflHPwhLE3", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 87, + "versionNonce": 1028684278, + "isDeleted": false, + "id": "h7rDKyrC7qd8W2VaDHOVo", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 589.7050511592882, + "y": 230.97412720541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 60.6328125, + "height": 52.44140625, + "seed": 889750981, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "OzG5v9FzTWSKCdywOoGQL", + "type": "arrow" + }, + { + "id": "EtBC7WdsIkjxflHPwhLE3", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 170, + "versionNonce": 1501874282, + "isDeleted": false, + "id": "OzG5v9FzTWSKCdywOoGQL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 601.5175511592882, + "y": 292.5288147054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 141.015625, + "height": 86.015625, + "seed": 1272808741, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "h7rDKyrC7qd8W2VaDHOVo", + "focus": -0.5378118988727681, + "gap": 9.11328125 + }, + "endBinding": { + "elementId": "lSqDDXX9cdXC4LCluuQmS", + "focus": -0.42120469532718297, + "gap": 3.79296875 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -141.015625, + 86.015625 + ] + ] + }, + { + "type": "arrow", + "version": 156, + "versionNonce": 72993590, + "isDeleted": false, + "id": "rsU49QVI8HgyJBtjILKZT", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 506.93161365928825, + "y": 207.66162720541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 236.51953125, + "height": 171.1328125, + "seed": 1922799749, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "MIEh2o7onmi2XVMBPNwtl", + "focus": -0.7011101066042149, + "gap": 8.453125 + }, + "endBinding": { + "elementId": "URqECeM4WGZqAFI0abbR6", + "focus": -0.5061734503271241, + "gap": 3.578125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -236.51953125, + 171.1328125 + ] + ] + }, + { + "type": "arrow", + "version": 203, + "versionNonce": 1600400170, + "isDeleted": false, + "id": "EE1SzT8racuV0SEacrH5H", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 394.04880115928825, + "y": 132.08350220541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29.9609375, + "height": 246.046875, + "seed": 1929396197, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "sHYBFDblul5QqJjxXun--", + "focus": -0.3111741555994148, + "gap": 5.7421875 + }, + "endBinding": { + "elementId": "WtyUQLscbyCgdCnYK_jNr", + "focus": 0.1692275109296219, + "gap": 4.0703125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -29.9609375, + 246.046875 + ] + ] + }, + { + "type": "arrow", + "version": 152, + "versionNonce": 1487336566, + "isDeleted": false, + "id": "X8GhhoUwWv39DpajqYRWn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 357.83786365928825, + "y": 26.669439705413424, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 28.1484375, + "height": 42.72265625, + "seed": 51968837, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "wkdtJfpB-XRjEMes0NHIG", + "focus": 0.5463571651085232, + "gap": 6.6796875 + }, + "endBinding": { + "elementId": "sHYBFDblul5QqJjxXun--", + "focus": 0.5277962544041825, + "gap": 12.30078125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 28.1484375, + 42.72265625 + ] + ] + }, + { + "type": "arrow", + "version": 145, + "versionNonce": 550057450, + "isDeleted": false, + "id": "T2wLMX7gk4q4Wlq65lvOL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 420.33786365928825, + "y": 109.02100220541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 88.5234375, + "height": 26.375, + "seed": 994265765, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "sHYBFDblul5QqJjxXun--", + "focus": -0.13918168745815823, + "gap": 9.67578125 + }, + "endBinding": { + "elementId": "MIEh2o7onmi2XVMBPNwtl", + "focus": 1.041000043955478, + "gap": 9.8984375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 88.5234375, + 26.375 + ] + ] + }, + { + "type": "arrow", + "version": 170, + "versionNonce": 335345078, + "isDeleted": false, + "id": "EtBC7WdsIkjxflHPwhLE3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 548.1972386592882, + "y": 171.43506470541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 72.3203125, + "height": 55.29296875, + "seed": 1277996549, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "MIEh2o7onmi2XVMBPNwtl", + "focus": -0.6152607989398555, + "gap": 10.86328125 + }, + "endBinding": { + "elementId": "h7rDKyrC7qd8W2VaDHOVo", + "focus": 0.6244234688285908, + "gap": 4.24609375 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 72.3203125, + 55.29296875 + ] + ] + }, + { + "type": "arrow", + "version": 342, + "versionNonce": 1858005162, + "isDeleted": false, + "id": "g1I2bmNZBEIbE5bPbc9PO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 256.4971091669645, + "y": 29.618658455413424, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 4.3475249654532035, + "height": 65.71875, + "seed": 1959961957, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "PX8R3GWFK-y4FAPfGXDWQ", + "focus": 0.2891038770717789, + "gap": 9.07421875 + }, + "endBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": 0.13304920019960997, + "gap": 12.8671875 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 4.3475249654532035, + 65.71875 + ] + ] + }, + { + "type": "rectangle", + "version": 144, + "versionNonce": 260099830, + "isDeleted": false, + "id": "anui-58DIlGAbjV4UqtEB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 186.06833240928825, + "y": 259.5795959554134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 35.375, + "height": 48.05859375, + "seed": 948089029, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "SJqqF6rk1YusFfgDKDZIX", + "type": "arrow" + }, + { + "id": "4Xhxr95Fo3KM0W5_NblnS", + "type": "arrow" + } + ], + "updated": 1666058888071, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 175, + "versionNonce": 1089487722, + "isDeleted": false, + "id": "SJqqF6rk1YusFfgDKDZIX", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 229.78317615928825, + "y": 140.61475220541342, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 26.859375, + "height": 109.1171875, + "seed": 1423593509, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888071, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hWhmUfN-whJHDhqK7HkDw", + "focus": 1.0945410835846927, + "gap": 10.41015625 + }, + "endBinding": { + "elementId": "anui-58DIlGAbjV4UqtEB", + "focus": -0.38855870595300823, + "gap": 9.84765625 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -26.859375, + 109.1171875 + ] + ] + }, + { + "type": "arrow", + "version": 180, + "versionNonce": 934772790, + "isDeleted": false, + "id": "4Xhxr95Fo3KM0W5_NblnS", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 209.10348865928825, + "y": 316.2319397054134, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.80859375, + "height": 60.609375, + "seed": 2009531269, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "elementId": "anui-58DIlGAbjV4UqtEB", + "focus": 0.4480882059017431, + "gap": 8.59375 + }, + "endBinding": { + "elementId": "URqECeM4WGZqAFI0abbR6", + "focus": -0.028542558229941315, + "gap": 5.53125 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 36.80859375, + 60.609375 + ] + ] + }, + { + "type": "text", + "version": 131, + "versionNonce": 1638417962, + "isDeleted": false, + "id": "YlP9Bsi-02bMH0FiCjimk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1338.8156749765176, + "y": -115.61648283257028, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 95, + "height": 25, + "seed": 1642740997, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Find Path", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Find Path" + }, + { + "type": "rectangle", + "version": 461, + "versionNonce": 545655158, + "isDeleted": false, + "id": "ar7IzDq-viaTUF4A87b_3", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1350.2562583548645, + "y": 297.740958380131, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 37.91699739062982, + "height": 94.91248590244665, + "seed": 118309546, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "OLV3s1miKo9A3zrGxFock", + "type": "arrow" + }, + { + "id": "0vxegaqxPquMPgd_r72TA", + "type": "arrow" + }, + { + "id": "unCTd-_GGU2aGYrHtH0Xs", + "type": "arrow" + }, + { + "id": "XWSRH57gG_wU8RgnjMLPE", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 334, + "versionNonce": 433189098, + "isDeleted": false, + "id": "VjZnajfB7LKe5DMAoo0Do", + "fillStyle": "cross-hatch", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1427.2168589704975, + "y": 345.30969428257765, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 50.58203125, + "height": 43.453125, + "seed": 845372778, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "4EwKyIvTMtoyxqVHUbeGd", + "type": "arrow" + }, + { + "id": "E2mIJ2Sl4cVsW3XGconh3", + "type": "arrow" + }, + { + "id": "TggNAReZxluYM_0WrlwOr", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 1123, + "versionNonce": 422668982, + "isDeleted": false, + "id": "0vxegaqxPquMPgd_r72TA", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1364.6693370711946, + "y": 401.4742259036285, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 72.00486815355634, + "height": 146.2109087738151, + "seed": 698884138, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.3965191378313845, + "gap": 8.159100701385455, + "elementId": "PDhm6OLJ3n51SVKIISduG" + }, + "endBinding": { + "focus": -0.13518700591321753, + "gap": 5.03237207597158, + "elementId": "MDw8O0rja4i4fRNIEVJ3V" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -72.00486815355634, + 146.2109087738151 + ] + ] + }, + { + "type": "rectangle", + "version": 307, + "versionNonce": 1559774122, + "isDeleted": false, + "id": "MDw8O0rja4i4fRNIEVJ3V", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1260.210836693423, + "y": 552.7175067534151, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 43.8828125, + "height": 51.65625, + "seed": 1220971242, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "0vxegaqxPquMPgd_r72TA", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 222, + "versionNonce": 1079944182, + "isDeleted": false, + "id": "i1kjJhOQKEyXNdqTQ4aN7", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1497.8783853089833, + "y": 548.2801914995628, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 36.53125, + "height": 48.55078125, + "seed": 910794154, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "unCTd-_GGU2aGYrHtH0Xs", + "type": "arrow" + }, + { + "id": "4EwKyIvTMtoyxqVHUbeGd", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 944, + "versionNonce": 1967537770, + "isDeleted": false, + "id": "unCTd-_GGU2aGYrHtH0Xs", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1392.5188356116166, + "y": 402.286327945094, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 137.47301625611294, + "height": 141.3501859235961, + "seed": 257710186, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.49474380369903687, + "gap": 10.567710822520771, + "elementId": "ar7IzDq-viaTUF4A87b_3" + }, + "endBinding": { + "focus": 1.0023527144670679, + "gap": 4.643677630872599, + "elementId": "i1kjJhOQKEyXNdqTQ4aN7" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 137.47301625611294, + 141.3501859235961 + ] + ] + }, + { + "type": "arrow", + "version": 718, + "versionNonce": 403879222, + "isDeleted": false, + "id": "4EwKyIvTMtoyxqVHUbeGd", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1455.0832738563568, + "y": 396.26281928257765, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.74251748189454, + "height": 139.68924721698505, + "seed": 331987754, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.3053227201737998, + "gap": 7.5, + "elementId": "VjZnajfB7LKe5DMAoo0Do" + }, + "endBinding": { + "focus": 0.6606224867711619, + "gap": 12.328125, + "elementId": "i1kjJhOQKEyXNdqTQ4aN7" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 63.74251748189454, + 139.68924721698505 + ] + ] + }, + { + "type": "rectangle", + "version": 593, + "versionNonce": 797368618, + "isDeleted": false, + "id": "RTqCReH94hrfFOfKvX3Xb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1237.3418614357154, + "y": 239.7958129496575, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 41.6796875, + "height": 41.82421874999999, + "seed": 1375520234, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Gm1LhBSST9OO99UkA6y8X", + "type": "arrow" + }, + { + "id": "PIZ_GY_wTp8sNyDtJZcnP", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 242, + "versionNonce": 431136374, + "isDeleted": false, + "id": "v3ufybsWQdzjQGQnJYauJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1282.1830728089833, + "y": -30.30965225043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 1392539818, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "cVT4Yqnzt5w3srqK1yI08", + "type": "text" + }, + { + "id": "cVT4Yqnzt5w3srqK1yI08", + "type": "text" + }, + { + "id": "ScvEbRxtwuaacH97zK60u", + "type": "arrow" + }, + { + "type": "text", + "id": "cVT4Yqnzt5w3srqK1yI08" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 97, + "versionNonce": 119876586, + "isDeleted": false, + "id": "cVT4Yqnzt5w3srqK1yI08", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1287.1830728089833, + "y": -20.85652725043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 1182782314, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "v3ufybsWQdzjQGQnJYauJ", + "originalText": "A" + }, + { + "type": "rectangle", + "version": 300, + "versionNonce": 1241735094, + "isDeleted": false, + "id": "EliJKYDq1GH48V4c34Lk1", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1379.2104165589833, + "y": -30.86433975043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 43.90625, + "seed": 784717354, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "HgWFwyAcCrdOtSBQ3AMZC", + "type": "text" + }, + { + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "type": "arrow" + }, + { + "type": "text", + "id": "HgWFwyAcCrdOtSBQ3AMZC" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 158, + "versionNonce": 1378623146, + "isDeleted": false, + "id": "HgWFwyAcCrdOtSBQ3AMZC", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1384.2104165589833, + "y": -20.41121475043724, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 29, + "height": 23, + "seed": 213745898, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "EliJKYDq1GH48V4c34Lk1", + "originalText": "B" + }, + { + "type": "arrow", + "version": 657, + "versionNonce": 1294596342, + "isDeleted": false, + "id": "E2mIJ2Sl4cVsW3XGconh3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1326.6992629818317, + "y": 143.9103142656606, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 123.98105485129145, + "height": 193.77103893976562, + "seed": 200397738, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.36231278326472255, + "gap": 5.17578125, + "elementId": "kKu5Ydyyyeh2KCFqMwJS4" + }, + "endBinding": { + "focus": 0.4326012077832137, + "gap": 7.628341077151447, + "elementId": "VjZnajfB7LKe5DMAoo0Do" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 123.98105485129145, + 193.77103893976562 + ] + ] + }, + { + "type": "rectangle", + "version": 276, + "versionNonce": 1833351530, + "isDeleted": false, + "id": "kKu5Ydyyyeh2KCFqMwJS4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1281.2656692318317, + "y": 101.25675399956276, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 40.2578125, + "height": 45.4921875, + "seed": 781045354, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Gm1LhBSST9OO99UkA6y8X", + "type": "arrow" + }, + { + "id": "E2mIJ2Sl4cVsW3XGconh3", + "type": "arrow" + }, + { + "id": "ScvEbRxtwuaacH97zK60u", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 848, + "versionNonce": 2058054198, + "isDeleted": false, + "id": "Gm1LhBSST9OO99UkA6y8X", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1301.5745444918389, + "y": 152.10888167241131, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 33.471215097623144, + "height": 84.01952770009473, + "seed": 2036101418, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.3812012413933706, + "gap": 5.359940172848553, + "elementId": "kKu5Ydyyyeh2KCFqMwJS4" + }, + "endBinding": { + "focus": 0.05283911803202242, + "gap": 3.667403577151447, + "elementId": "RTqCReH94hrfFOfKvX3Xb" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -33.471215097623144, + 84.01952770009473 + ] + ] + }, + { + "type": "rectangle", + "version": 144, + "versionNonce": 1740635178, + "isDeleted": false, + "id": "61dxkeeBxIqh92cwqJ1lM", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1411.0736978089833, + "y": 74.74503524956276, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 40.76953125, + "height": 44.6484375, + "seed": 533249002, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "type": "arrow" + }, + { + "id": "RPqJsAAGLOdXHUSB03TtV", + "type": "arrow" + }, + { + "id": "XWSRH57gG_wU8RgnjMLPE", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 116, + "versionNonce": 1902949238, + "isDeleted": false, + "id": "dLAlZSp2rgCOTpB7iMpyy", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1527.7690103089833, + "y": 138.34659774956276, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 50.74609375, + "height": 53.9140625, + "seed": 803057322, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "RPqJsAAGLOdXHUSB03TtV", + "type": "arrow" + }, + { + "id": "TggNAReZxluYM_0WrlwOr", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 517, + "versionNonce": 797107946, + "isDeleted": false, + "id": "TggNAReZxluYM_0WrlwOr", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1544.1200151918972, + "y": 204.47533658893383, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 69.09497933411285, + "height": 137.03534395324658, + "seed": 1044793706, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.27534455299140825, + "gap": 12.214676339371067, + "elementId": "dLAlZSp2rgCOTpB7iMpyy" + }, + "endBinding": { + "focus": 0.26615030765156217, + "gap": 3.7990137403972426, + "elementId": "VjZnajfB7LKe5DMAoo0Do" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -69.09497933411285, + 137.03534395324658 + ] + ] + }, + { + "type": "arrow", + "version": 889, + "versionNonce": 733140150, + "isDeleted": false, + "id": "XWSRH57gG_wU8RgnjMLPE", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1427.4290168688203, + "y": 128.0884665163548, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 36.397392064383894, + "height": 161.61331552909485, + "seed": 307072042, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.12673028883550302, + "gap": 8.692142102217417, + "elementId": "61dxkeeBxIqh92cwqJ1lM" + }, + "endBinding": { + "focus": 0.31432596950919534, + "gap": 8.532211308753404, + "elementId": "ar7IzDq-viaTUF4A87b_3" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -36.397392064383894, + 161.61331552909485 + ] + ] + }, + { + "type": "arrow", + "version": 302, + "versionNonce": 1025145258, + "isDeleted": false, + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1398.6789464539738, + "y": 19.830407576714318, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 27.97996373105775, + "height": 42.725507914574635, + "seed": 1373533930, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.5580789543163621, + "gap": 6.7884973271515605, + "elementId": "EliJKYDq1GH48V4c34Lk1" + }, + "endBinding": { + "focus": 0.5164516591849347, + "gap": 12.19197142284844, + "elementId": "61dxkeeBxIqh92cwqJ1lM" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 27.97996373105775, + 42.725507914574635 + ] + ] + }, + { + "type": "arrow", + "version": 292, + "versionNonce": 321048054, + "isDeleted": false, + "id": "RPqJsAAGLOdXHUSB03TtV", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1461.0276308555176, + "y": 102.16607065982501, + "strokeColor": "#364fc7", + "backgroundColor": "transparent", + "width": 88.74199015486556, + "height": 26.39089941688931, + "seed": 763627946, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": -0.1330668246148966, + "gap": 9.458161595696765, + "elementId": "61dxkeeBxIqh92cwqJ1lM" + }, + "endBinding": { + "focus": 1.0359692074076057, + "gap": 9.78962767284844, + "elementId": "dLAlZSp2rgCOTpB7iMpyy" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 88.74199015486556, + 26.39089941688931 + ] + ] + }, + { + "type": "arrow", + "version": 474, + "versionNonce": 1068610666, + "isDeleted": false, + "id": "ScvEbRxtwuaacH97zK60u", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1297.6782558166597, + "y": 22.562006672411314, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 4.274858880633019, + "height": 65.71875, + "seed": 208225386, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.28876032726731193, + "gap": 8.965408922848553, + "elementId": "v3ufybsWQdzjQGQnJYauJ" + }, + "endBinding": { + "focus": 0.13338192970579654, + "gap": 12.975997327151447, + "elementId": "kKu5Ydyyyeh2KCFqMwJS4" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 4.274858880633019, + 65.71875 + ] + ] + }, + { + "type": "rectangle", + "version": 172, + "versionNonce": 348233526, + "isDeleted": false, + "id": "PDhm6OLJ3n51SVKIISduG", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1350.6704030660512, + "y": 344.5622776479856, + "strokeColor": "#e67700", + "backgroundColor": "transparent", + "width": 36.1762450326662, + "height": 48.752847554257414, + "seed": 945366826, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "PIZ_GY_wTp8sNyDtJZcnP", + "type": "arrow" + }, + { + "id": "0vxegaqxPquMPgd_r72TA", + "type": "arrow" + } + ], + "updated": 1666058888072, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 575, + "versionNonce": 1226020650, + "isDeleted": false, + "id": "PIZ_GY_wTp8sNyDtJZcnP", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1272.6633717593447, + "y": 291.80621417637605, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 70.51754950478039, + "height": 53.81745925866585, + "seed": 163492330, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888072, + "link": null, + "locked": false, + "startBinding": { + "focus": 0.5414720442395691, + "gap": 10.186182476718557, + "elementId": "RTqCReH94hrfFOfKvX3Xb" + }, + "endBinding": { + "focus": 0.09938892036401328, + "gap": 7.48948180192599, + "elementId": "PDhm6OLJ3n51SVKIISduG" + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 70.51754950478039, + 53.81745925866585 + ] + ] + }, + { + "type": "arrow", + "version": 278, + "versionNonce": 888424566, + "isDeleted": false, + "id": "p4uOM1CtA_v2Pgf6ijK9W", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1816.8655741450823, + "y": 804.5549918935991, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 4.425399877032078, + "height": 863.6896840333961, + "seed": 639832037, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 4.425399877032078, + -863.6896840333961 + ] + ] + }, + { + "type": "text", + "version": 155, + "versionNonce": 2041258474, + "isDeleted": false, + "id": "j8MvrQTIbxG8XlEdrUsHO", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1710.063519604042, + "y": -31.350809576339884, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 80, + "height": 25, + "seed": 786904971, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Lamport", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Lamport" + }, + { + "type": "rectangle", + "version": 243, + "versionNonce": 595577270, + "isDeleted": false, + "id": "PNo3TYUfJ10IJi6267aAh", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1350.7224316028319, + "y": 295.70182854499143, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 37.41238356134112, + "height": 46.67318451720979, + "seed": 329573546, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "Y38QXVN9QX9BPQY-2ARsJ", + "type": "arrow" + }, + { + "id": "RPqJsAAGLOdXHUSB03TtV", + "type": "arrow" + }, + { + "id": "XWSRH57gG_wU8RgnjMLPE", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 163, + "versionNonce": 746804394, + "isDeleted": false, + "id": "2ZLvwZY7jTj3Y7A867PUw", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 907.8436350772699, + "y": 2.945826594468457, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 200, + "height": 25, + "seed": 1748790533, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Need to find all the", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Need to find all the" + }, + { + "type": "rectangle", + "version": 620, + "versionNonce": 1334387446, + "isDeleted": false, + "id": "awuQ8K6kOHnRv5HDrvJWm", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 926.507571138557, + "y": 50.93006817318019, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 41.6796875, + "height": 41.82421874999999, + "seed": 489627941, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "7o4qiZeXIC5XlAhjbYObN", + "type": "arrow" + }, + { + "id": "D7U5Ron34tY9c82vBqDaB", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 226, + "versionNonce": 591867754, + "isDeleted": false, + "id": "E3RI5q1yc3GeaorRS1w0w", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1046.9959782392652, + "y": 46.34815941558787, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 40.76953125, + "height": 44.6484375, + "seed": 1561043499, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "strWHmShIdrbcdVwMei4L", + "type": "arrow" + }, + { + "id": "gouvuPNNZV_QH-D9GO0TA", + "type": "arrow" + }, + { + "id": "JZQIc_KcE8CBpiAjy1v9j", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 39, + "versionNonce": 2073383990, + "isDeleted": false, + "id": "jhZwZVYHq9DVCoTsVfoNA", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 920.1084297636246, + "y": 103.86982908128641, + "strokeColor": "#d9480f", + "backgroundColor": "transparent", + "width": 55, + "height": 20, + "seed": 1102244805, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "From A", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "From A" + }, + { + "type": "text", + "version": 74, + "versionNonce": 843166250, + "isDeleted": false, + "id": "nwkFN4UJbc5Jg4yZ0eNgs", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1041.1473326329826, + "y": 101.57709076329934, + "strokeColor": "#1864ab", + "backgroundColor": "transparent", + "width": 56, + "height": 20, + "seed": 1781306661, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "From B", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "From B" + }, + { + "type": "text", + "version": 81, + "versionNonce": 1367956854, + "isDeleted": false, + "id": "kIrTAwchInsTnq6LCV63N", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 895.0964797797128, + "y": 397.1606789154389, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 214, + "height": 20, + "seed": 167668933, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "KLyXLrCImVTQP_jU2EX9O", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "This case is a little tricky", + "baseline": 14, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "This case is a little tricky" + }, + { + "type": "arrow", + "version": 142, + "versionNonce": 1818090730, + "isDeleted": false, + "id": "KLyXLrCImVTQP_jU2EX9O", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "dashed", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 1119.079507605672, + "y": 405.26356192336186, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 205.11324147046594, + "height": 45.80272521494817, + "seed": 459973771, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "kIrTAwchInsTnq6LCV63N", + "focus": 0.714758414751021, + "gap": 9.983027825959198 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 205.11324147046594, + -45.80272521494817 + ] + ] + }, + { + "type": "text", + "version": 188, + "versionNonce": 1990868662, + "isDeleted": false, + "id": "SztschmDzOAjFuvTM-OuV", + "fillStyle": "hachure", + "strokeWidth": 4, + "strokeStyle": "dashed", + "roughness": 2, + "opacity": 100, + "angle": 0, + "x": 893.4248140652206, + "y": 439.769275660658, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 276, + "height": 40, + "seed": 332043691, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 16, + "fontFamily": 1, + "text": "we need to slice it after we found\nthe nearest common ancestors", + "baseline": 34, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "we need to slice it after we found\nthe nearest common ancestors" + }, + { + "type": "rectangle", + "version": 108, + "versionNonce": 1327935402, + "isDeleted": false, + "id": "Y9vqkX1na6vJDz7l4PspG", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 207.46439441645236, + "y": 845.6472361162546, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 62.180771530968855, + "height": 64.0268369575906, + "seed": 1456447841, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "EJMwY_7pCh7jyDuzdJg6V" + }, + { + "id": "iBGHqUq7evqeBkhBGjpIY", + "type": "arrow" + }, + { + "id": "1FA4XZdv1alN1lHogHTl9", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 144, + "versionNonce": 530380790, + "isDeleted": false, + "id": "t51UcEh3ESlAROMlQggkd", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 150.87847590478498, + "y": 750.1418377874459, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 609.5544837384108, + "height": 29.249255457697245, + "seed": 699369007, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 23.399404366157817, + "fontFamily": 1, + "text": "What is the nearest common ancestors in this case?", + "baseline": 21.249255457697245, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "What is the nearest common ancestors in this case?" + }, + { + "type": "rectangle", + "version": 178, + "versionNonce": 1858546282, + "isDeleted": false, + "id": "OcZsyOg63jPy2vGYoS8dH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 197.62028593945422, + "y": 994.0831720435232, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 55, + "height": 67.44040550860723, + "seed": 1580756303, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "BW_IPp1nyObkgC7Vu3OYp" + }, + { + "id": "iBGHqUq7evqeBkhBGjpIY", + "type": "arrow" + }, + { + "id": "WLEzhxAeQRhzi52ESFg-q", + "type": "arrow" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 14, + "versionNonce": 610699574, + "isDeleted": false, + "id": "EJMwY_7pCh7jyDuzdJg6V", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 212.46439441645236, + "y": 865.1606545950499, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 52, + "height": 23, + "seed": 546698977, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Y9vqkX1na6vJDz7l4PspG", + "originalText": "A" + }, + { + "type": "text", + "version": 49, + "versionNonce": 571100458, + "isDeleted": false, + "id": "BW_IPp1nyObkgC7Vu3OYp", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 202.62028593945422, + "y": 1015.3033747978269, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 45, + "height": 23, + "seed": 1644625505, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "OcZsyOg63jPy2vGYoS8dH", + "originalText": "B" + }, + { + "type": "rectangle", + "version": 136, + "versionNonce": 342618742, + "isDeleted": false, + "id": "gCZ4PjyxtYNIC3EGFAKxA", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 356.58493261190574, + "y": 974.517711358304, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 54, + "height": 58, + "seed": 1573171983, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "1FA4XZdv1alN1lHogHTl9", + "type": "arrow" + }, + { + "id": "fOuTMdWLfJmuB2ov11q7v", + "type": "arrow" + }, + { + "type": "text", + "id": "86HkQ09dZUZF0MFiJVz63" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 185, + "versionNonce": 777721834, + "isDeleted": false, + "id": "hLRIE0OQJr_P647NEGYUb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 339.35656242842015, + "y": 1187.155175856424, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.38000584903011, + "height": 74.49416956838468, + "seed": 1700659777, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "WLEzhxAeQRhzi52ESFg-q", + "type": "arrow" + }, + { + "id": "fOuTMdWLfJmuB2ov11q7v", + "type": "arrow" + }, + { + "type": "text", + "id": "uc0dpPTx3z-Oih__yVi7r" + } + ], + "updated": 1666058888073, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 52, + "versionNonce": 1248998326, + "isDeleted": false, + "id": "iBGHqUq7evqeBkhBGjpIY", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 238.39425275353517, + "y": 916.6475734194214, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 3.4230113409226988, + "height": 64.07877230207362, + "seed": 355303279, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Y9vqkX1na6vJDz7l4PspG", + "focus": -0.05859992602942486, + "gap": 6.973500345576213 + }, + "endBinding": { + "elementId": "OcZsyOg63jPy2vGYoS8dH", + "focus": 0.2503697342260085, + "gap": 13.356826322028098 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -3.4230113409226988, + 64.07877230207362 + ] + ] + }, + { + "type": "arrow", + "version": 221, + "versionNonce": 1942260394, + "isDeleted": false, + "id": "1FA4XZdv1alN1lHogHTl9", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 280.2730259866174, + "y": 874.3827042892433, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 108.9112565402437, + "height": 90.60251065896, + "seed": 958569537, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Y9vqkX1na6vJDz7l4PspG", + "gap": 10.627860039196209, + "focus": -0.6558398277536983 + }, + "endBinding": { + "elementId": "gCZ4PjyxtYNIC3EGFAKxA", + "gap": 9.53249641010075, + "focus": 0.8392853510122925 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 108.9112565402437, + 90.60251065896 + ] + ] + }, + { + "type": "arrow", + "version": 352, + "versionNonce": 10374390, + "isDeleted": false, + "id": "WLEzhxAeQRhzi52ESFg-q", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 225.846817638945, + "y": 1073.1712589011736, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 130.36469464909064, + "height": 104.34282846123074, + "seed": 974853505, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "OcZsyOg63jPy2vGYoS8dH", + "focus": 0.8039769929584992, + "gap": 11.647681349043125 + }, + "endBinding": { + "elementId": "hLRIE0OQJr_P647NEGYUb", + "focus": 0.5592314137305141, + "gap": 9.641088494019641 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 130.36469464909064, + 104.34282846123074 + ] + ] + }, + { + "type": "arrow", + "version": 447, + "versionNonce": 418489706, + "isDeleted": false, + "id": "fOuTMdWLfJmuB2ov11q7v", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 383.2859330385243, + "y": 1039.3495698552904, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 0.8019442573741458, + "height": 139.71785644664988, + "seed": 1093453345, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "startBinding": { + "elementId": "gCZ4PjyxtYNIC3EGFAKxA", + "focus": 0.0034356344242915454, + "gap": 6.831858496986342 + }, + "endBinding": { + "elementId": "hLRIE0OQJr_P647NEGYUb", + "focus": 0.35034121969360427, + "gap": 8.087749554483707 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -0.8019442573741458, + 139.71785644664988 + ] + ] + }, + { + "type": "text", + "version": 86, + "versionNonce": 1685405238, + "isDeleted": false, + "id": "86HkQ09dZUZF0MFiJVz63", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 361.58493261190574, + "y": 992.017711358304, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 44, + "height": 23, + "seed": 1827546465, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "1", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "gCZ4PjyxtYNIC3EGFAKxA", + "originalText": "1" + }, + { + "type": "text", + "version": 97, + "versionNonce": 925722666, + "isDeleted": false, + "id": "uc0dpPTx3z-Oih__yVi7r", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 344.35656242842015, + "y": 1211.9022606406163, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 53, + "height": 23, + "seed": 1503038895, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "2", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "hLRIE0OQJr_P647NEGYUb", + "originalText": "2" + }, + { + "type": "text", + "version": 253, + "versionNonce": 719062902, + "isDeleted": false, + "id": "kI_tZyDto7LyRVi7fZo1b", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 458.5859491764503, + "y": 836.3733952089806, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 268, + "height": 25, + "seed": 381074991, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Is it just B? Or B and 2?", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Is it just B? Or B and 2?" + }, + { + "type": "text", + "version": 172, + "versionNonce": 107139818, + "isDeleted": false, + "id": "Ucbb53X_Dwn5QAY3EB0D8", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 459.9740392926316, + "y": 901.9771780808721, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 254, + "height": 25, + "seed": 343812495, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "Should be both B and 2. ", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Should be both B and 2. " + }, + { + "type": "text", + "version": 59, + "versionNonce": 605530294, + "isDeleted": false, + "id": "cOQa-Eb9wSAJWzgq9aI7x", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 159.49974308895753, + "y": 1311.600682807928, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 424, + "height": 25, + "seed": 1981785473, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888073, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "How to tackle this case when finding path?", + "baseline": 18, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "How to tackle this case when finding path?" + }, + { + "type": "rectangle", + "version": 52, + "versionNonce": 152888746, + "isDeleted": false, + "id": "hwOHW2_5-VmaE0Y21RH6I", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 174.03219675428193, + "y": 1394.1222450159953, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 49.409398183112216, + "height": 51.5151403321488, + "seed": 578415457, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "o2Bs8-53MFFZjUFarRyRN" + }, + { + "id": "qCbjrqfXeEu-zEq2Ffb3T", + "type": "arrow" + }, + { + "id": "R6d9DcWsOcD8YWu9N7JxL", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 14, + "versionNonce": 1338843638, + "isDeleted": false, + "id": "o2Bs8-53MFFZjUFarRyRN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 179.03219675428193, + "y": 1407.3798151820697, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 39, + "height": 23, + "seed": 1451199681, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "A", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "hwOHW2_5-VmaE0Y21RH6I", + "originalText": "A" + }, + { + "type": "rectangle", + "version": 48, + "versionNonce": 948346986, + "isDeleted": false, + "id": "D36mTqaq_mm5bD6sonefu", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 341.06570740140523, + "y": 1402.8757112588514, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 52, + "height": 54.08830058153217, + "seed": 1613404719, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "type": "text", + "id": "PnMoL-isuYyODBEw1k2wf" + }, + { + "id": "NyiyA39uyFO2fv9N82bF4", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "text", + "version": 14, + "versionNonce": 1895920438, + "isDeleted": false, + "id": "PnMoL-isuYyODBEw1k2wf", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 346.06570740140523, + "y": 1417.4198615496175, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 42, + "height": 23, + "seed": 467983471, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "fontSize": 20, + "fontFamily": 1, + "text": "B", + "baseline": 17, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "D36mTqaq_mm5bD6sonefu", + "originalText": "B" + }, + { + "type": "arrow", + "version": 83, + "versionNonce": 812319530, + "isDeleted": false, + "id": "qCbjrqfXeEu-zEq2Ffb3T", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 199.85822714717483, + "y": 1452.9508261303365, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.22628499953339, + "height": 184.9087119391695, + "seed": 105061935, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hwOHW2_5-VmaE0Y21RH6I", + "focus": 0.20038022884771262, + "gap": 7.3134407821924015 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + 40.22628499953339, + 184.9087119391695 + ] + ] + }, + { + "type": "arrow", + "version": 50, + "versionNonce": 572103798, + "isDeleted": false, + "id": "NyiyA39uyFO2fv9N82bF4", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 370.7066249163197, + "y": 1466.4917868555315, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 113.29459329211295, + "height": 165.0694103466767, + "seed": 366695329, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "D36mTqaq_mm5bD6sonefu", + "focus": -0.6449918259980243, + "gap": 9.527775015147881 + }, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -113.29459329211295, + 165.0694103466767 + ] + ] + }, + { + "type": "rectangle", + "version": 74, + "versionNonce": 1530203626, + "isDeleted": false, + "id": "s54CZ5KG2aYjaNsp6G4VZ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 217.9317270272469, + "y": 1640.4751908734663, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 63.21475702567511, + "height": 68.62547564180954, + "seed": 638536961, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "aw37qj_R25zNHnnWOQGPF", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 59, + "versionNonce": 1759157686, + "isDeleted": false, + "id": "R6d9DcWsOcD8YWu9N7JxL", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 179.45235816032232, + "y": 1451.817691341617, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 111.97732410022672, + "height": 94.94253110981413, + "seed": 1780700449, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "hwOHW2_5-VmaE0Y21RH6I", + "focus": -0.3337409231098588, + "gap": 6.1803059934729845 + }, + "endBinding": { + "elementId": "Uu6icBmJsdOeGkGZzL7sU", + "focus": -0.6678530131421979, + "gap": 9.065078309754199 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -111.97732410022672, + 94.94253110981413 + ] + ] + }, + { + "type": "rectangle", + "version": 72, + "versionNonce": 502251690, + "isDeleted": false, + "id": "Uu6icBmJsdOeGkGZzL7sU", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 36.14857854696129, + "y": 1555.8253007611854, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 54.08830058153217, + "height": 59.489576407760524, + "seed": 1867485153, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "2-6tyGZ_ZmFG9B-mW7C21", + "type": "arrow" + }, + { + "id": "R6d9DcWsOcD8YWu9N7JxL", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 299, + "versionNonce": 1248425718, + "isDeleted": false, + "id": "2-6tyGZ_ZmFG9B-mW7C21", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 69.80268177192295, + "y": 1626.4762548378305, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.55206125129018, + "height": 95.98595939442657, + "seed": 803727777, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "Uu6icBmJsdOeGkGZzL7sU", + "focus": -0.6031691429467954, + "gap": 11.161377668884597 + }, + "endBinding": { + "elementId": "kMzfHdkQJvza3Y7oH-Stk", + "focus": -0.780062894083163, + "gap": 7.828072832068642 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -40.55206125129018, + 95.98595939442657 + ] + ] + }, + { + "type": "rectangle", + "version": 117, + "versionNonce": 71703402, + "isDeleted": false, + "id": "kMzfHdkQJvza3Y7oH-Stk", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 5.5633820414475395, + "y": 1730.2902870643259, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 49.86265209859994, + "height": 320.63937404792387, + "seed": 1045724623, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "2-6tyGZ_ZmFG9B-mW7C21", + "type": "arrow" + }, + { + "id": "StiyyzlA2m4GVtTLr2uzG", + "type": "arrow" + }, + { + "id": "3HbM7P7WEUHkMZ9blj5wa", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 125, + "versionNonce": 1598761014, + "isDeleted": false, + "id": "xG6Ryld1g4RsmlYfeyRyl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 4.930715117745535, + "y": 1894.3540402860144, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 49.206378200133486, + "height": 156.04210319654658, + "seed": 441826383, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "29BHND6z19Wd0fVvFoxsH", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 40, + "versionNonce": 1820532266, + "isDeleted": false, + "id": "7avwsrGLRQ9zXDSv0w1np", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 166.8651192156325, + "y": 1781.2294172122108, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 24.806209083045587, + "height": 32.90812282238835, + "seed": 1685597185, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "aw37qj_R25zNHnnWOQGPF", + "type": "arrow" + }, + { + "id": "9h36VDCVJrRu8C1CMjJtB", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "rectangle", + "version": 52, + "versionNonce": 1734652278, + "isDeleted": false, + "id": "0uds6kbQkH4ArKucMh0Vg", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 115.45857096740258, + "y": 1831.7105720496534, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 33.93738692214151, + "height": 30.297191413380915, + "seed": 1820056143, + "groupIds": [], + "strokeSharpness": "sharp", + "boundElements": [ + { + "id": "9h36VDCVJrRu8C1CMjJtB", + "type": "arrow" + }, + { + "id": "29BHND6z19Wd0fVvFoxsH", + "type": "arrow" + } + ], + "updated": 1666058888074, + "link": null, + "locked": false + }, + { + "type": "arrow", + "version": 47, + "versionNonce": 865192170, + "isDeleted": false, + "id": "aw37qj_R25zNHnnWOQGPF", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 241.67090085091502, + "y": 1720.7483478643207, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 40.703145889786015, + "height": 49.39051260330052, + "seed": 1142455201, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "s54CZ5KG2aYjaNsp6G4VZ", + "focus": -0.5010982988927215, + "gap": 11.647681349044774 + }, + "endBinding": { + "elementId": "7avwsrGLRQ9zXDSv0w1np", + "focus": -0.03852497283277824, + "gap": 11.090556744589435 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -40.703145889786015, + 49.39051260330052 + ] + ] + }, + { + "type": "arrow", + "version": 36, + "versionNonce": 1576735414, + "isDeleted": false, + "id": "9h36VDCVJrRu8C1CMjJtB", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 164.44776499969817, + "y": 1807.2348606133176, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 17.138663679378624, + "height": 16.893151141822955, + "seed": 285622561, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "7avwsrGLRQ9zXDSv0w1np", + "focus": 0.17632017043480894, + "gap": 2.417354215934324 + }, + "endBinding": { + "elementId": "0uds6kbQkH4ArKucMh0Vg", + "focus": -0.2529475107653309, + "gap": 7.582560294512859 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -17.138663679378624, + 16.893151141822955 + ] + ] + }, + { + "type": "arrow", + "version": 53, + "versionNonce": 1627200426, + "isDeleted": false, + "id": "29BHND6z19Wd0fVvFoxsH", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 111.12433040055146, + "y": 1855.3930891338857, + "strokeColor": "#000000", + "backgroundColor": "transparent", + "width": 51.59068265139683, + "height": 34.503954316500995, + "seed": 668671791, + "groupIds": [], + "strokeSharpness": "round", + "boundElements": [], + "updated": 1666058888074, + "link": null, + "locked": false, + "startBinding": { + "elementId": "0uds6kbQkH4ArKucMh0Vg", + "focus": 0.2156267298677806, + "gap": 4.334240566851122 + }, + "endBinding": { + "elementId": "xG6Ryld1g4RsmlYfeyRyl", + "focus": -0.6606370592162203, + "gap": 5.396554431275604 + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "points": [ + [ + 0, + 0 + ], + [ + -51.59068265139683, + 34.503954316500995 + ] + ] + } + ], + "appState": { + "gridSize": null, + "viewBackgroundColor": "#ffffff" + }, + "files": {} +} \ No newline at end of file diff --git a/vendor/loro-internal/src/dag/test.rs b/vendor/loro-internal/src/dag/test.rs new file mode 100644 index 00000000..dc3afba8 --- /dev/null +++ b/vendor/loro-internal/src/dag/test.rs @@ -0,0 +1,1467 @@ +#![allow(unused)] +use arbitrary::Arbitrary; +use im::HashSet; +use loro_common::HasCounter; +use std::{cmp::Ordering, sync::Arc}; + +use super::*; +use crate::{ + array_mut_ref, + change::Lamport, + id::{Counter, PeerID, ID}, + span::HasIdSpan, +}; + +#[derive(Debug, PartialEq, Eq, Clone)] +struct TestNode { + id: ID, + lamport: Lamport, + len: usize, + deps: Arc, +} + +impl TestNode { + fn new(id: ID, lamport: Lamport, deps: Frontiers, len: usize) -> Self { + Self { + id, + lamport, + deps: Arc::new(deps), + len, + } + } +} + +impl DagNode for TestNode { + fn deps(&self) -> &Frontiers { + &self.deps + } +} + +impl Sliceable for TestNode { + fn slice(&self, from: usize, to: usize) -> Self { + Self { + id: self.id.inc(from as Counter), + lamport: self.lamport + from as Lamport, + len: to - from, + deps: if from > 0 { + Arc::new(self.id.inc(from as Counter - 1).into()) + } else { + self.deps.clone() + }, + } + } +} + +impl HasLamport for TestNode { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl HasId for TestNode { + fn id_start(&self) -> ID { + self.id + } +} + +impl HasCounter for TestNode { + fn ctr_start(&self) -> Counter { + self.id.counter + } +} + +impl HasLength for TestNode { + fn content_len(&self) -> usize { + self.len + } +} + +#[derive(Debug, PartialEq, Eq)] +struct TestDag { + nodes: FxHashMap>, + frontier: Frontiers, + version_vec: VersionVector, + next_lamport: Lamport, + client_id: PeerID, +} + +impl Dag for TestDag { + type Node = TestNode; + + fn get(&self, id: ID) -> Option { + let arr = self.nodes.get(&id.peer)?; + arr.binary_search_by(|node| { + if node.id.counter > id.counter { + Ordering::Greater + } else if node.id.counter + node.len as i32 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + .map_or(None, |x| Some(arr[x].clone())) + } + + fn frontier(&self) -> &Frontiers { + &self.frontier + } + + fn vv(&self) -> &VersionVector { + &self.version_vec + } + + fn contains(&self, id: ID) -> bool { + self.version_vec.includes_id(id) + } +} + +impl TestDag { + pub fn new(client_id: PeerID) -> Self { + Self { + nodes: FxHashMap::default(), + frontier: Frontiers::new(), + version_vec: VersionVector::new(), + next_lamport: 0, + client_id, + } + } + + fn get_last_node(&mut self) -> &mut TestNode { + self.nodes + .get_mut(&self.client_id) + .unwrap() + .last_mut() + .unwrap() + } + + fn push(&mut self, len: usize) { + let client_id = self.client_id; + let counter = self.version_vec.entry(client_id).or_insert(0); + let id = ID::new(client_id, *counter); + *counter += len as Counter; + let deps = std::mem::replace(&mut self.frontier, id.inc(len as Counter - 1).into()); + self.nodes.entry(client_id).or_default().push(TestNode::new( + id, + self.next_lamport, + deps, + len, + )); + self.next_lamport += len as u32; + } + + fn merge(&mut self, other: &TestDag) { + let mut pending = Vec::new(); + for (_, nodes) in other.nodes.iter() { + for (i, node) in nodes.iter().enumerate() { + if self._try_push_node(node, &mut pending, i) { + break; + } + } + } + + let mut current = pending; + let mut pending = Vec::new(); + while !pending.is_empty() || !current.is_empty() { + if current.is_empty() { + std::mem::swap(&mut pending, &mut current); + } + + let (client_id, index) = current.pop().unwrap(); + let node_vec = other.nodes.get(&client_id).unwrap(); + #[allow(clippy::needless_range_loop)] + for i in index..node_vec.len() { + let node = &node_vec[i]; + if self._try_push_node(node, &mut pending, i) { + break; + } + } + } + } + + fn update_frontier(frontier: &mut Frontiers, new_node_id: ID, new_node_deps: &Frontiers) { + frontier.retain(|x| { + if x.peer == new_node_id.peer && x.counter <= new_node_id.counter { + return false; + } + + !new_node_deps + .iter() + .any(|y| y.peer == x.peer && y.counter >= x.counter) + }); + + // nodes from the same client with `counter < new_node_id.counter` + // are filtered out from frontier. + if frontier.iter().all(|x| x.peer != new_node_id.peer) { + frontier.push(new_node_id); + } + } + + fn _try_push_node( + &mut self, + node: &TestNode, + pending: &mut Vec<(PeerID, usize)>, + i: usize, + ) -> bool { + let client_id = node.id.peer; + if self.contains(node.id_last()) { + return false; + } + if node.deps.iter().any(|dep| !self.contains(dep)) { + pending.push((client_id, i)); + return true; + } + Self::update_frontier(&mut self.frontier, node.id_last(), &node.deps); + let contains_start = self.contains(node.id_start()); + let arr = self.nodes.entry(client_id).or_default(); + if contains_start { + arr.pop(); + arr.push(node.clone()); + } else { + arr.push(node.clone()); + } + self.version_vec.set_end(node.id_end()); + self.next_lamport = self.next_lamport.max(node.lamport + node.len as u32); + false + } +} + +/// ```mermaid /// flowchart RL +/// subgraph client0 +/// 0-1("c0: [1, 3)") --> 0-0("c0: [0, 1)") +/// end +/// +/// subgraph client1 +/// 1-0("c1: [0, 2)") +/// end +/// +/// 0-1 --> 1-0 +/// ``` +#[test] +fn test_dag() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + a.push(1); + assert_eq!(a.frontier().len(), 1); + assert_eq!(a.frontier().as_single().unwrap().counter, 0); + b.push(1); + a.merge(&b); + assert_eq!(a.frontier().len(), 2); + a.push(1); + assert_eq!(a.frontier().len(), 1); + // a: 0 --(merge)--- 1 + // ↑ + // | + // b: 0 ---- + assert_eq!( + a.frontier().iter().next().unwrap(), + ID { + peer: 0, + counter: 1 + } + ); + + // a: 0 --(merge)--- 1 --- 2 ------- + // ↑ | + // | ↓ + // b: 0 ------------1----------(merge) + a.push(1); + b.push(1); + b.merge(&a); + assert_eq!(b.next_lamport, 3); + assert_eq!(b.frontier().len(), 2); + // println!("{}", b.mermaid()); + assert_eq!( + b.find_common_ancestor(&[ID::new(0, 2)].into(), &[ID::new(1, 1)].into()) + .0 + .len(), + 0 + ); +} + +#[derive(Debug, Clone, Copy, Arbitrary)] +pub struct Interaction { + dag_idx: usize, + merge_with: Option, + len: usize, +} + +impl Interaction { + fn generate(rng: &mut impl rand::Rng, num: usize) -> Self { + if rng.gen_bool(0.5) { + let dag_idx = rng.gen_range(0..num); + let merge_with = (rng.gen_range(1..num - 1) + dag_idx) % num; + Self { + dag_idx, + merge_with: Some(merge_with), + len: rng.gen_range(1..10), + } + } else { + Self { + dag_idx: rng.gen_range(0..num), + merge_with: None, + len: rng.gen_range(1..10), + } + } + } + + fn apply(&self, dags: &mut [TestDag]) { + if let Some(merge_with) = self.merge_with { + if merge_with != self.dag_idx { + let (from, to) = array_mut_ref!(dags, [self.dag_idx, merge_with]); + from.merge(to); + } + } + + dags[self.dag_idx].push(self.len); + } +} + +// prop_compose! { +// fn gen_interaction(num: usize) ( +// dag_idx in 0..num, +// merge_with in 0..num, +// length in 1..10, +// should_merge in 0..2 +// ) -> Interaction { +// Interaction { +// dag_idx, +// merge_with: if should_merge == 1 && merge_with != dag_idx { Some(merge_with) } else { None }, +// len: length as usize, +// } +// } +// } + +fn preprocess(interactions: &mut [Interaction], num: i32) { + for interaction in interactions.iter_mut() { + interaction.dag_idx %= num as usize; + if let Some(ref mut merge_with) = interaction.merge_with { + *merge_with %= num as usize; + if *merge_with == interaction.dag_idx { + *merge_with = (*merge_with + 1) % num as usize; + } + } + + interaction.len = (interaction.len % 128).max(1); + } +} + +mod iter { + use super::*; + + #[test] + fn test() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + // 0-0 + a.push(1); + // 1-0 + b.push(1); + a.merge(&b); + // 0-1 + a.push(1); + b.merge(&a); + // 1-1 + b.push(1); + a.merge(&b); + // 0-2 + a.push(1); + + let mut count = 0; + for (node, vv) in a.iter_with_vv() { + count += 1; + if node.id == ID::new(0, 0) { + assert_eq!(vv, vec![ID::new(0, 0)].into()); + } else if node.id == ID::new(0, 2) { + assert_eq!(vv, vec![ID::new(0, 2), ID::new(1, 1)].into()); + } + } + + assert_eq!(count, 5); + } +} + +mod dfs { + #[allow(unused)] + #[allow(dead_code)] + use super::*; + use crate::{allocation::calc_critical_version_dfs, delta::DeltaValue}; + use rand::{rngs::StdRng, SeedableRng}; + + #[test] + fn test_dfs_small() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + let mut c = TestDag::new(2); + a.push(10); + b.merge(&a); + b.push(3); + c.merge(&b); + c.push(4); + a.push(3); + a.merge(&c); + a.push(2); + b.merge(&a); + assert_eq!( + calc_critical_version_dfs::( + &b, + &[ID { + peer: 0, + counter: 13, + }] + .into(), + &[ID { + peer: 0, + counter: 9, + }] + .into(), + ) + .length(), + 0 + ); + } + + #[test] + fn test_dfs_big() { + let num = 5; + let mut rng = StdRng::seed_from_u64(100); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..100 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + let start = dags[0].frontier(); + let ends = [ + ID { + peer: 3, + counter: 7, + }, + ID { + peer: 4, + counter: 6, + }, + ] + .into(); + assert_eq!( + calc_critical_version_dfs(&dags[0], start, &ends).length(), + 0 + ); + } +} + +mod bfs { + use arref::array_mut_ref; + use loro_common::ID; + use rand::{rngs::StdRng, SeedableRng}; + + use crate::{ + allocation::calc_critical_version, + dag::{ + test::{TestDag, TestNode}, + Dag, Interaction, + }, + delta::DeltaValue as _, + }; + + #[test] + fn test_bfs_small() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + let mut c = TestDag::new(2); + a.push(10); + b.merge(&a); + b.push(3); + c.merge(&b); + c.push(4); + a.push(3); + a.merge(&c); + a.push(2); + b.merge(&a); + assert_eq!( + calc_critical_version::( + &b, + &[ID { + peer: 0, + counter: 13, + }] + .into(), + ) + .length(), + 0 + ); + } + + #[test] + fn test_bfs_big() { + let num = 5; + let mut rng = StdRng::seed_from_u64(100); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..100 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + let start = dags[0].frontier(); + let ends = [ID { + peer: 3, + counter: 7, + }]; + assert_eq!(calc_critical_version(&dags[0], start).length(), 0); + } +} + +mod mermaid { + use super::*; + use rand::{rngs::StdRng, SeedableRng}; + + #[test] + fn simple() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + // 0-0 + a.push(1); + // 1-0 + b.push(1); + a.merge(&b); + // 0-1 + a.push(1); + b.merge(&a); + // 1-1 + b.push(1); + a.merge(&b); + // 0-2 + a.push(1); + + println!("{}", a.mermaid()); + } + + #[test] + fn three() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + let mut c = TestDag::new(2); + a.push(10); + b.merge(&a); + b.push(3); + c.merge(&b); + c.push(4); + a.push(3); + a.merge(&c); + a.push(2); + b.merge(&a); + println!("{}", a.mermaid()); + } + + #[test] + fn gen_graph() { + let num = 5; + let mut rng = StdRng::seed_from_u64(100); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..100 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + println!("{}", dags[0].mermaid()); + } +} + +mod get_version_vector { + #[allow(unused)] + #[allow(dead_code)] + use super::*; + + #[test] + fn vv() { + let mut a = TestDag::new(0); + let mut b = TestDag::new(1); + a.push(1); + b.push(1); + a.merge(&b); + a.push(1); + let actual = a.get_vv(ID::new(0, 0)); + assert_eq!(actual, vec![ID::new(0, 0)].into()); + let actual = a.get_vv(ID::new(0, 1)); + assert_eq!(actual, vec![ID::new(0, 1), ID::new(1, 0)].into()); + + let mut c = TestDag::new(2); + c.merge(&a); + b.push(1); + c.merge(&b); + c.push(1); + let actual = c.get_vv(ID::new(2, 0)); + assert_eq!( + actual, + vec![ID::new(0, 1), ID::new(1, 1), ID::new(2, 0)].into() + ); + } +} + +#[test] +fn test_alloc() { + use rand::thread_rng; + for _ in 1..10000 { + let num = 4; + let mut rng = thread_rng(); + let mut dags = (0..num).map(TestDag::new).collect::>(); + for _ in 0..20 { + Interaction::generate(&mut rng, num as usize).apply(&mut dags); + } + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i as usize]); + a.merge(other); + } + let a = &dags[0]; + let start = a.frontier(); + let end = crate::allocation::get_end_list(a, start); + let critical_version_actual: HashSet = + crate::allocation::calc_critical_version(a, start) + .into_iter() + .collect(); + let critical_version: HashSet = + crate::allocation::calc_critical_version_dfs(a, start, &end) + .into_iter() + .collect(); + assert!(critical_version_actual.eq(&critical_version)); + } +} + +pub fn fuzz_alloc_tree(num: usize, mut actions: Vec) { + preprocess(&mut actions, num as i32); + let mut dags = (0..num as u64).map(TestDag::new).collect::>(); + for action in actions { + action.apply(&mut dags); + } + + for i in 1..num { + let (a, other) = array_mut_ref!(&mut dags, [0, i]); + a.merge(other); + } + + let a = &dags[0]; + let start = a.frontier(); + let end = crate::allocation::get_end_list(a, start); + let actual: HashSet = crate::allocation::calc_critical_version(a, start) + .into_iter() + .collect(); + let expected: HashSet = crate::allocation::calc_critical_version_dfs(a, start, &end) + .into_iter() + .collect(); + assert!(actual.eq(&expected)); +} + +#[test] +fn failed_fuzz() { + fuzz_alloc_tree( + 10, + vec![Interaction { + dag_idx: 14685055086129564619, + merge_with: Some(10), + len: 777939436809428736, + }], + ) +} + +// #[cfg(test)] +// mod find_path { +// use crate::{fx_map, span::CounterSpan}; + +// use super::*; + +// #[test] +// fn retreat_to_beginning() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(1); +// b.push(1); +// a.merge(&b); +// let actual = a.find_path(&[ID::new(0, 0)], &[ID::new(1, 0)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(0 => CounterSpan { start: 0, end: 1 }), +// right: fx_map!(1 => CounterSpan { start: 0, end: 1 }) +// } +// ); +// } + +// #[test] +// fn one_path() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// // 0 - 0 +// a.push(1); +// b.merge(&a); +// // 1 - 0 +// b.push(1); +// // 0 - 1 +// a.push(1); +// a.merge(&b); +// let actual = a.find_path(&[ID::new(0, 1)], &[ID::new(1, 0)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(0 => CounterSpan { start: 1, end: 2 }), +// right: fx_map!(1 => CounterSpan { start: 0, end: 1 }) +// } +// ); +// } + +// #[test] +// fn middle() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(2); +// b.push(1); +// b.merge(&a); +// b.push(2); +// a.push(2); +// b.merge(&a); +// // println!("{}", b.mermaid()); +// let actual = b.find_path(&[ID::new(0, 3)], &[ID::new(1, 2)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(0 => CounterSpan { start: 2, end: 4 }), +// right: fx_map!(1 => CounterSpan { start: 0, end: 3 }) +// } +// ); +// let actual = b.find_path(&[ID::new(1, 1), ID::new(1, 2)], &[ID::new(0, 3)]); +// assert_eq!( +// actual, +// VersionVectorDiff { +// left: fx_map!(1 => CounterSpan { start: 0, end: 3 }), +// right: fx_map!(0 => CounterSpan { start: 2, end: 4 }) +// } +// ); +// } + +// fn test_find_path( +// dag_num: i32, +// mut interactions: Vec, +// ) -> Result<(), TestCaseError> { +// preprocess(&mut interactions, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for interaction in interactions.iter_mut() { +// interaction.apply(&mut dags); +// } + +// for i in 1..dag_num { +// let (a, b) = array_mut_ref!(&mut dags, [0, i as usize]); +// a.merge(b); +// } + +// let a = &dags[0]; +// let mut nodes = Vec::new(); +// for (node, vv) in a.iter_with_vv() { +// nodes.push((node, vv)); +// } + +// // println!("{}", a.mermaid()); +// let vec: Vec<_> = nodes.iter().enumerate().collect(); +// for &(i, (node, vv)) in vec.iter() { +// if i > 3 { +// break; +// } + +// for &(j, (other_node, other_vv)) in vec.iter() { +// if i >= j { +// continue; +// } + +// let actual = a.find_path(&[node.id], &[other_node.id]); +// let expected = vv.diff(other_vv); +// prop_assert_eq!( +// actual, +// expected, +// "\ni={} j={} node={} other={}", +// i, +// j, +// node.id, +// other_node.id +// ); + +// for iter in nodes[j + 1..].iter() { +// let mut vv = vv.clone(); +// vv.merge(other_vv); +// let actual = a.find_path(&[node.id, other_node.id], &[iter.0.id]); +// let expected = vv.diff(&iter.1); +// prop_assert_eq!(actual, expected); +// } +// } +// } + +// Ok(()) +// } + +// #[test] +// fn issue() { +// if let Err(err) = test_find_path( +// 5, +// vec![ +// Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 3, +// }, +// Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 3, +// }, +// Interaction { +// dag_idx: 4, +// merge_with: Some(1), +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 2, +// merge_with: Some(0), +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 2, +// merge_with: Some(0), +// len: 1, +// }, +// Interaction { +// dag_idx: 3, +// merge_with: Some(0), +// len: 1, +// }, +// ], +// ) { +// panic!("{}", err); +// } +// } + +// proptest! { +// #[test] +// fn proptest_path( +// interactions in prop::collection::vec(gen_interaction(5), 0..10 * PROPTEST_FACTOR_10), +// ) { +// test_find_path(5, interactions)?; +// } + +// #[test] +// fn proptest_path_large( +// interactions in prop::collection::vec(gen_interaction(10), 0..PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10 + 10), +// ) { +// test_find_path(10, interactions)?; +// } +// } +// } + +// mod find_common_ancestors { + +// use super::*; + +// #[test] +// fn siblings() { +// let mut a = TestDag::new(0); +// a.push(5); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 2)], &[ID::new(0, 4)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, Some(ID::new(0, 2))); +// assert_eq!( +// a.find_common_ancestor(&[ID::new(0, 2)], &[ID::new(0, 4)]).1, +// DiffMode::Linear +// ); +// } + +// #[test] +// fn no_common_ancestors() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(1); +// b.push(1); +// a.merge(&b); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 0)], &[ID::new(1, 0)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); + +// // interactions between b and c +// let mut c = TestDag::new(2); +// c.merge(&b); +// c.push(2); +// b.merge(&c); +// b.push(3); + +// // should no exist any common ancestor between a and b +// let actual = a +// .find_common_ancestor(&[ID::new(0, 0)], &[ID::new(1, 0)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); +// assert_eq!( +// a.find_common_ancestor(&[ID::new(0, 0)], &[ID::new(1, 0)]).1, +// DiffMode::Checkout +// ) +// } + +// #[test] +// fn no_common_ancestors_when_there_is_an_redundant_node() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(1); +// b.push(1); +// b.merge(&a); +// b.push(1); +// a.push(4); +// a.merge(&b); +// println!("{}", a.mermaid()); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 4)], &[ID::new(0, 1), ID::new(1, 1)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); +// let actual = a +// .find_common_ancestor(&[ID::new(0, 4)], &[ID::new(1, 1)]) +// .0 +// .first() +// .copied(); +// assert_eq!(actual, None); +// } + +// #[test] +// fn dep_in_middle() { +// let mut a = TestDag::new(0); +// let mut b = TestDag::new(1); +// a.push(3); +// b.merge(&a); +// b.push(9); +// a.push(2); +// b.merge(&a); +// println!("{}", b.mermaid()); +// assert_eq!( +// b.find_common_ancestor(&[ID::new(0, 3)], &[ID::new(1, 8)]) +// .0 +// .first() +// .copied(), +// Some(ID::new(0, 2)) +// ); +// } + +// /// ![](https://mermaid.ink/img/pako:eNqNkTFPwzAQhf_K6SYqOZJ9CYsHJroxwYgXY7skInEq1xFCVf87jg5XVQQSnk6fz_feO5_RzT6gxsM4f7repgzPTyZCOafl7T3ZYw9uHELMkqls2juDTmp4bQV0O4M7aJqHwqlyEtDecFW5EkA3XFYuBaiVs0CInotfXSimqunW16q87gTcX6cqdqe27hSrKVZr_6tGTImn0nYqcWbaZiZWI1ajP9WK2zqnClFd5jVn3SIKnEKa7ODLb53Xa4O5D1MwqEvpbfowaOKl9C1Hb3PY-yHPCfXBjqcg0C55fvmKDnVOS6hNj4Mtgaefrss3dp6HFg) +// #[test] +// fn large_lamport_with_longer_path() { +// let mut a0 = TestDag::new(0); +// let mut a1 = TestDag::new(1); +// let mut a2 = TestDag::new(2); + +// a0.push(3); +// a1.merge(&a0); +// a2.merge(&a0); +// a1.push(3); +// a2.push(2); +// a2.push(1); +// a1.merge(&a2); +// a2.push(1); +// a1.push(1); +// a1.merge(&a2); +// a1.push(1); +// a0.push(1); +// a1.merge(&a2); +// a1.merge(&a0); +// println!("{}", a1.mermaid()); +// assert_eq!( +// a1.find_common_ancestor(&[ID::new(0, 3)], &[ID::new(1, 4)]) +// .0 +// .first() +// .copied(), +// Some(ID::new(0, 2)) +// ); +// assert_eq!( +// a1.find_common_ancestor(&[ID::new(2, 3)], &[ID::new(1, 3)]) +// .0 +// .iter() +// .copied() +// .collect::>(), +// vec![ID::new(0, 2)] +// ); +// } +// } + +// mod find_common_ancestors_proptest { + +// use crate::{ +// array_mut_ref, +// span::HasIdSpan, +// tests::{PROPTEST_FACTOR_1, PROPTEST_FACTOR_10}, +// }; + +// use super::*; + +// proptest! { +// #[test] +// fn test_2dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(2), 0..100 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(2), 0..100 * PROPTEST_FACTOR_10) +// ) { +// test_single_common_ancestor(2, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_4dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(4), 0..50 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(4), 0..50 * PROPTEST_FACTOR_10) +// ) { +// test_single_common_ancestor(4, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_10dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10) +// ) { +// test_single_common_ancestor(10, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_mul_ancestors_5dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(10), 0..30 + PROPTEST_FACTOR_1 * 500), +// after_merged_insertions in prop::collection::vec(gen_interaction(10), 0..30 + PROPTEST_FACTOR_1 * 500) +// ) { +// test_mul_ancestors::<2>(5, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_mul_ancestors_10dags( +// before_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(10), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10) +// ) { +// test_mul_ancestors::<3>(10, before_merged_insertions, after_merged_insertions)?; +// } + +// #[test] +// fn test_mul_ancestors_15dags_2( +// before_merged_insertions in prop::collection::vec(gen_interaction(15), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10), +// after_merged_insertions in prop::collection::vec(gen_interaction(15), 0..10 * PROPTEST_FACTOR_10 * PROPTEST_FACTOR_10) +// ) { +// test_mul_ancestors::<5>(15, before_merged_insertions, after_merged_insertions)?; +// } +// } + +// #[test] +// fn issue_0() { +// if let Err(err) = test_mul_ancestors::<3>( +// 10, +// vec![ +// Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 8, +// merge_with: None, +// len: 2, +// }, +// ], +// vec![Interaction { +// dag_idx: 1, +// merge_with: None, +// len: 1, +// }], +// ) { +// println!("{}", err); +// panic!(); +// } +// } + +// #[test] +// fn issue_1() { +// test_single_common_ancestor( +// 2, +// vec![], +// vec![ +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// ], +// ) +// .unwrap(); +// } + +// fn test_single_common_ancestor( +// dag_num: i32, +// mut before_merge_insertion: Vec, +// mut after_merge_insertion: Vec, +// ) -> Result<(), TestCaseError> { +// preprocess(&mut before_merge_insertion, dag_num); +// preprocess(&mut after_merge_insertion, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for interaction in before_merge_insertion { +// apply(interaction, &mut dags); +// } + +// for dag_idx in 1..dags.len() { +// let (dag0, dag) = arref::array_mut_ref!(&mut dags, [0, dag_idx]); +// dag0.merge(dag); +// } + +// dags[0].push(1); +// let expected = dags[0].frontier()[0]; +// for dag_idx in 1..dags.len() { +// let (dag0, dag) = arref::array_mut_ref!(&mut dags, [0, dag_idx]); +// dag.merge(dag0); +// } +// for interaction in after_merge_insertion.iter_mut() { +// if let Some(merge) = interaction.merge_with { +// // odd dag merges with the odd +// // even dag merges with the even +// if merge % 2 != interaction.dag_idx % 2 { +// interaction.merge_with = None; +// } +// } + +// apply(*interaction, &mut dags); +// } + +// let (dag0, dag1) = array_mut_ref!(&mut dags, [0, 1]); +// dag1.push(1); +// dag0.merge(dag1); +// // println!("{}", dag0.mermaid()); +// let a = dags[0].nodes.get(&0).unwrap().last().unwrap().id_last(); +// let b = dags[1].nodes.get(&1).unwrap().last().unwrap().id_last(); +// let actual = dags[0].find_common_ancestor(&[a], &[b]); +// prop_assert_eq!(&**actual.0, &[expected]); +// Ok(()) +// } + +// fn apply(interaction: Interaction, dags: &mut [TestDag]) { +// let Interaction { +// dag_idx, +// len, +// merge_with, +// } = interaction; +// if let Some(merge_with) = merge_with { +// let (dag, merge_target): (&mut TestDag, &mut TestDag) = +// array_mut_ref!(dags, [dag_idx, merge_with]); +// dag.push(len); +// dag.merge(merge_target); +// } else { +// dags[dag_idx].push(len); +// } +// } + +// fn test_mul_ancestors( +// dag_num: i32, +// mut before_merge_insertion: Vec, +// mut after_merge_insertion: Vec, +// ) -> Result<(), TestCaseError> { +// assert!(dag_num - 2 >= N as i32); +// preprocess(&mut before_merge_insertion, dag_num); +// preprocess(&mut after_merge_insertion, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for mut interaction in before_merge_insertion { +// if interaction.dag_idx < N { +// // cannot act on first N nodes +// interaction.dag_idx = interaction.dag_idx % (dags.len() - N) + N; +// } +// if let Some(merge) = interaction.merge_with { +// if interaction.dag_idx == merge { +// let next_merge = (merge + 1) % dags.len(); +// interaction.merge_with = Some(next_merge); +// } +// } + +// apply(interaction, &mut dags); +// } + +// for target in 0..N { +// for i in N..dags.len() { +// let (target, dag): (&mut TestDag, &mut TestDag) = +// arref::array_mut_ref!(&mut dags, [target, i]); +// target.merge(dag); +// } +// } + +// let mut expected = Vec::with_capacity(N); +// for dag in dags[0..N].iter_mut() { +// dag.push(1); +// expected.push(dag.frontier[0]); +// } + +// for target in 0..N { +// for i in N..dags.len() { +// let (target, dag): (&mut TestDag, &mut TestDag) = +// arref::array_mut_ref!(&mut dags, [target, i]); +// dag.merge(target); +// } +// } + +// let mut merged_to_even = [false; N]; +// let mut merged_to_odd = [false; N]; + +// for interaction in after_merge_insertion.iter_mut() { +// if interaction.dag_idx < N { +// // cannot act on first N nodes +// interaction.dag_idx = interaction.dag_idx % (dags.len() - N) + N; +// } +// if let Some(mut merge) = interaction.merge_with { +// if interaction.dag_idx == merge { +// let next_merge = (merge + 1) % dags.len(); +// interaction.merge_with = Some(next_merge); +// merge = next_merge; +// } + +// // odd dag merges with the odd +// // even dag merges with the even +// if merge >= N && merge % 2 != interaction.dag_idx % 2 { +// interaction.merge_with = None; +// } +// if merge < N { +// if interaction.dag_idx % 2 == 0 { +// merged_to_even[merge] = true; +// } else { +// merged_to_odd[merge] = true; +// } +// } +// } + +// if dags[interaction.dag_idx].is_first() { +// // need to merge to one of the common ancestors first +// let target = interaction.dag_idx % N; +// let (dag, target) = arref::array_mut_ref!(&mut dags, [interaction.dag_idx, target]); +// dag.merge(target); +// } + +// apply(*interaction, &mut dags); +// } + +// // make common ancestor dags be merged to opposite side (even/odd) +// for i in 0..N { +// let (odd, even) = if N % 2 == 0 { (N + 1, N) } else { (N, N + 1) }; +// if !merged_to_even[i] && i % 2 != 0 { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [even, i]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// if !merged_to_odd[i] && i % 2 == 0 { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [odd, i]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// } + +// // merge with all odds or evens +// for i in dags.len() - 2..dags.len() { +// if i % 2 == 0 { +// for target in (0..dags.len() - 2).step_by(2) { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [i, target]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// } else { +// for target in (1..dags.len() - 2).step_by(2) { +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [i, target]); +// dag_a.merge(dag_b); +// dag_a.push(1); +// } +// } +// } + +// let len = dags.len(); +// let (dag_a, dag_b) = array_mut_ref!(&mut dags, [len - 2, len - 1]); +// dag_a.push(1); +// dag_b.push(1); +// dag_a.merge(dag_b); +// let a = dag_a.get_last_node().id; +// let b = dag_b.get_last_node().id; +// let mut actual = dag_a.find_common_ancestor(&[a], &[b]).0; +// actual.sort(); +// let actual = actual.iter().copied().collect::>(); +// if actual != expected { +// println!("{}", dag_a.mermaid()); +// } + +// prop_assert_eq!(actual, expected); +// Ok(()) +// } +// } + +// #[cfg(test)] +// mod dag_partial_iter { +// use loro_common::HasCounterSpan; + +// use crate::{dag::iter::IterReturn, tests::PROPTEST_FACTOR_10}; + +// use super::*; + +// fn test_partial_iter( +// dag_num: i32, +// mut interactions: Vec, +// ) -> Result<(), TestCaseError> { +// preprocess(&mut interactions, dag_num); +// let mut dags = Vec::new(); +// for i in 0..dag_num { +// dags.push(TestDag::new(i as PeerID)); +// } + +// for interaction in interactions.iter_mut() { +// interaction.apply(&mut dags); +// } + +// for i in 1..dag_num { +// let (a, b) = array_mut_ref!(&mut dags, [0, i as usize]); +// a.merge(b); +// } + +// let a = &dags[0]; +// let mut nodes = Vec::new(); +// for (node, vv) in a.iter_with_vv() { +// nodes.push((node, vv)); +// } + +// let mut map = FxHashMap::default(); +// for (node, vv) in nodes.iter() { +// map.insert(node.id, vv.clone()); +// } +// let vec: Vec<_> = nodes.iter().enumerate().collect(); +// { +// // println!("{}", a.mermaid()); +// } +// for &(i, (node, vv)) in vec.iter() { +// if i > 3 { +// break; +// } + +// for &(j, (_other_node, other_vv)) in vec.iter() { +// if i >= j { +// continue; +// } + +// let diff_spans = other_vv.diff(vv).left; +// { +// // println!("TARGET IS TO GO FROM {} TO {}", node.id, other_node.id); +// // dbg!(&other_vv, &vv, &diff_spans); +// } +// let mut target_vv = vv.clone(); +// target_vv.forward(&diff_spans); + +// for IterReturn { data, slice } in a.iter_causal(&[node.id], diff_spans.clone()) { +// let sliced = data.slice(slice.start as usize, slice.end as usize); +// { +// // println!("-----------------------------------"); +// // dbg!(&sliced, &forward, &retreat, slice); +// } +// assert!(diff_spans +// .get(&data.id.peer) +// .unwrap() +// .contains(sliced.id.counter)); +// let mut data_vv = map.get(&data.id).unwrap().clone(); +// data_vv.extend_to_include(IdSpan::new( +// sliced.id.peer, +// sliced.id.counter, +// sliced.id.counter + 1, +// )); +// data_vv.shrink_to_exclude(IdSpan::new( +// sliced.id.peer, +// sliced.id.counter, +// sliced.ctr_end(), +// )); +// } +// } +// } + +// Ok(()) +// } + +// #[test] +// #[ignore] +// fn issue() { +// if let Err(err) = test_partial_iter( +// 5, +// vec![ +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 2, +// }, +// Interaction { +// dag_idx: 1, +// merge_with: Some(0), +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: None, +// len: 1, +// }, +// Interaction { +// dag_idx: 0, +// merge_with: Some(1), +// len: 1, +// }, +// ], +// ) { +// panic!("{}", err); +// } +// } + +// proptest! { +// #[test] +// #[ignore] +// fn proptest_iter_2( +// interactions in prop::collection::vec(gen_interaction(2), 0..40 * PROPTEST_FACTOR_10), +// ) { +// test_partial_iter(2, interactions)?; +// } + +// fn proptest_iter_3( +// interactions in prop::collection::vec(gen_interaction(3), 0..40 * PROPTEST_FACTOR_10), +// ) { +// test_partial_iter(3, interactions)?; +// } + +// fn proptest_iter_5( +// interactions in prop::collection::vec(gen_interaction(5), 0..40 * PROPTEST_FACTOR_10), +// ) { +// test_partial_iter(5, interactions)?; +// } +// } +// } diff --git a/vendor/loro-internal/src/delta.rs b/vendor/loro-internal/src/delta.rs new file mode 100644 index 00000000..efc72a0e --- /dev/null +++ b/vendor/loro-internal/src/delta.rs @@ -0,0 +1,14 @@ +mod movable_list; +pub(crate) use movable_list::{ElementDelta, MovableListInnerDelta}; +mod seq; +pub use seq::{Delta, DeltaItem, DeltaType, DeltaValue, Meta}; +mod map; +pub use map::{MapDiff, ValuePair}; +mod map_delta; +pub use map_delta::{MapDelta, MapValue, ResolvedMapDelta, ResolvedMapValue}; +mod text; +pub use text::{StyleMeta, StyleMetaItem}; +mod tree; +pub use tree::{ + TreeDelta, TreeDeltaItem, TreeDiff, TreeDiffItem, TreeExternalDiff, TreeInternalDiff, +}; diff --git a/vendor/loro-internal/src/delta/map.rs b/vendor/loro-internal/src/delta/map.rs new file mode 100644 index 00000000..9b650f72 --- /dev/null +++ b/vendor/loro-internal/src/delta/map.rs @@ -0,0 +1,203 @@ +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::Serialize; +use std::fmt::Debug; + +use crate::{InternalString, LoroValue}; + +#[derive(Clone, Debug, Serialize)] +pub struct ValuePair { + pub old: T, + pub new: T, +} + +impl From<(LoroValue, LoroValue)> for ValuePair { + fn from((old, new): (LoroValue, LoroValue)) -> Self { + ValuePair { old, new } + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct MapDiff { + pub added: FxHashMap, + pub updated: FxHashMap>, + pub deleted: FxHashMap, +} + +impl MapDiff { + pub fn new() -> Self { + Self::default() + } + + pub fn insert(mut self, key: InternalString, value: T) -> Self { + self.deleted.remove(&key); + self.added.insert(key, value); + self + } + + pub fn delete(mut self, key: &InternalString, value: T) -> Self { + self.added.remove(key); + self.deleted.insert(key.clone(), value); + self + } + + pub fn compose(mut self, other: Self) -> Self { + for (k, v) in other.added.into_iter() { + if let Some(dv) = self.deleted.remove(&k) { + self.updated.insert(k, ValuePair { old: dv, new: v }); + } else if let Some(ValuePair { old: _, new }) = self.updated.get_mut(&k) { + // should not happen in transaction + *new = v; + } else { + self.added.insert(k, v); + } + } + + for (k, v) in other.deleted.into_iter() { + if let Some(av) = self.added.remove(&k) { + self.deleted.insert(k, av); + } else if let Some(ValuePair { old, .. }) = self.updated.remove(&k) { + self.deleted.insert(k, old); + } else { + // delete old value + self.deleted.insert(k, v); + } + } + + for (k, ValuePair { old, new }) in other.updated.into_iter() { + if let Some(av) = self.added.get_mut(&k) { + *av = new; + } else if let Some(dv) = self.deleted.remove(&k) { + self.updated.insert(k, ValuePair { old: dv, new }); + } else if let Some(ValuePair { old: _, new: n }) = self.updated.get_mut(&k) { + *n = new + } else { + self.updated.insert(k, ValuePair { old, new }); + } + } + self + } +} + +#[derive(Clone, Debug, Serialize)] +#[allow(dead_code)] +pub struct MapDiffRaw { + pub(crate) added: FxHashMap, + pub(crate) deleted: FxHashSet, +} + +impl Default for MapDiff { + fn default() -> Self { + Self { + added: Default::default(), + updated: Default::default(), + deleted: Default::default(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{MapDiff, ValuePair}; + use crate::InternalString; + + fn key(value: &str) -> InternalString { + value.into() + } + + fn map_diff( + added: &[(&str, i32)], + updated: &[(&str, (i32, i32))], + deleted: &[(&str, i32)], + ) -> MapDiff { + let mut ans = MapDiff::new(); + for (k, v) in added { + ans.added.insert(key(k), *v); + } + for (k, (old, new)) in updated { + ans.updated.insert( + key(k), + ValuePair { + old: *old, + new: *new, + }, + ); + } + for (k, v) in deleted { + ans.deleted.insert(key(k), *v); + } + ans + } + + fn assert_map_diff_eq(actual: MapDiff, expected: MapDiff) { + assert_eq!(actual.added, expected.added); + assert_eq!(actual.deleted, expected.deleted); + + assert_eq!(actual.updated.len(), expected.updated.len()); + for (key, expected_pair) in expected.updated { + let actual_pair = actual.updated.get(&key).unwrap_or_else(|| { + panic!("missing updated entry for key {:?}", key); + }); + assert_eq!(actual_pair.old, expected_pair.old); + assert_eq!(actual_pair.new, expected_pair.new); + } + } + + #[test] + fn insert_replaces_previous_delete_marker() { + let diff = map_diff(&[], &[], &[("key", 1)]).insert(key("key"), 2); + assert_map_diff_eq(diff, map_diff(&[("key", 2)], &[], &[])); + } + + #[test] + fn delete_removes_pending_add_and_records_deleted_value() { + let diff = map_diff(&[("key", 1)], &[], &[]).delete(&key("key"), 1); + assert_map_diff_eq(diff, map_diff(&[], &[], &[("key", 1)])); + } + + #[test] + fn compose_add_then_update_keeps_latest_value_in_added() { + let lhs = map_diff(&[("key", 1)], &[], &[]); + let rhs = map_diff(&[], &[("key", (1, 2))], &[]); + + assert_map_diff_eq(lhs.compose(rhs), map_diff(&[("key", 2)], &[], &[])); + } + + #[test] + fn compose_delete_then_add_becomes_update_with_original_old_value() { + let lhs = map_diff(&[], &[], &[("key", 1)]); + let rhs = map_diff(&[("key", 2)], &[], &[]); + + assert_map_diff_eq(lhs.compose(rhs), map_diff(&[], &[("key", (1, 2))], &[])); + } + + #[test] + fn compose_update_then_delete_keeps_initial_old_value() { + let lhs = map_diff(&[], &[("key", (1, 2))], &[]); + let rhs = map_diff(&[], &[], &[("key", 2)]); + + assert_map_diff_eq(lhs.compose(rhs), map_diff(&[], &[], &[("key", 1)])); + } + + #[test] + fn compose_multiple_updates_preserves_first_old_and_latest_new() { + let lhs = map_diff(&[], &[("key", (1, 2))], &[]); + let rhs = map_diff(&[], &[("key", (2, 3))], &[]); + let third = map_diff(&[], &[("key", (3, 4))], &[]); + + assert_map_diff_eq( + lhs.compose(rhs).compose(third), + map_diff(&[], &[("key", (1, 4))], &[]), + ); + } + + #[test] + fn compose_is_key_local_and_preserves_other_entries() { + let lhs = map_diff(&[("added", 10)], &[("updated", (1, 2))], &[("deleted", 3)]); + let rhs = map_diff(&[("added", 11)], &[("updated", (2, 3))], &[("deleted", 4)]); + + assert_map_diff_eq( + lhs.compose(rhs), + map_diff(&[("added", 11)], &[("updated", (1, 3))], &[("deleted", 4)]), + ); + } +} diff --git a/vendor/loro-internal/src/delta/map_delta.rs b/vendor/loro-internal/src/delta/map_delta.rs new file mode 100644 index 00000000..b7f15493 --- /dev/null +++ b/vendor/loro-internal/src/delta/map_delta.rs @@ -0,0 +1,314 @@ +use std::hash::Hash; + +use loro_common::IdLp; +use rustc_hash::FxHashMap; +use serde::{ser::SerializeStruct, Serialize}; + +use crate::{ + change::Lamport, handler::ValueOrHandler, id::PeerID, span::HasLamport, InternalString, + LoroValue, +}; + +#[derive(Default, Debug, Clone, Serialize)] +pub struct MapDelta { + /// If the value is none, it's a uncreate op that should remove the entry from the + /// map container. + pub updated: FxHashMap>, +} + +#[derive(Debug, Clone)] +pub struct MapValue { + pub value: Option, + pub lamp: Lamport, + pub peer: PeerID, +} + +impl Ord for MapValue { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamp + .cmp(&other.lamp) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +impl PartialOrd for MapValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for MapValue { + fn eq(&self, other: &Self) -> bool { + self.lamp == other.lamp && self.peer == other.peer + } +} + +impl Eq for MapValue {} + +impl MapValue { + pub fn idlp(&self) -> IdLp { + IdLp::new(self.peer, self.lamp) + } +} + +#[derive(Default, Debug, Clone)] +pub struct ResolvedMapDelta { + pub updated: FxHashMap, +} + +#[derive(Debug, Clone)] +pub struct ResolvedMapValue { + pub value: Option, + pub idlp: IdLp, +} + +impl ResolvedMapValue { + /// This is used to indicate that the entry is unset. (caused by checkout to before the entry is created) + pub fn new_unset() -> Self { + ResolvedMapValue { + idlp: IdLp::new(PeerID::default(), Lamport::MAX), + value: None, + } + } +} + +impl MapDelta { + pub(crate) fn compose(mut self, x: MapDelta) -> MapDelta { + for (k, v) in x.updated.into_iter() { + if let Some(old) = self.updated.get_mut(&k) { + if &v > old { + *old = v; + } + } else { + self.updated.insert(k, v); + } + } + self + } + + #[inline] + pub fn new() -> Self { + MapDelta { + updated: FxHashMap::default(), + } + } + + #[inline] + pub fn with_entry(mut self, key: InternalString, map_value: MapValue) -> Self { + self.updated.insert(key, Some(map_value)); + self + } +} + +impl ResolvedMapDelta { + pub(crate) fn compose(&self, x: ResolvedMapDelta) -> ResolvedMapDelta { + let mut updated = self.updated.clone(); + for (k, v) in x.updated.into_iter() { + if let Some(old) = updated.get_mut(&k) { + if v.idlp > old.idlp { + *old = v; + } + } else { + updated.insert(k, v); + } + } + ResolvedMapDelta { updated } + } + + #[inline] + pub fn new() -> Self { + ResolvedMapDelta { + updated: FxHashMap::default(), + } + } + + #[inline] + pub fn with_entry(mut self, key: InternalString, map_value: ResolvedMapValue) -> Self { + self.updated.insert(key, map_value); + self + } + + pub(crate) fn transform(&mut self, b: &ResolvedMapDelta, left_prior: bool) { + for (k, _) in b.updated.iter() { + if !left_prior { + self.updated.remove(k); + } + } + } +} + +impl Hash for MapValue { + fn hash(&self, state: &mut H) { + // value is not being hashed + self.peer.hash(state); + self.lamp.hash(state); + } +} + +impl HasLamport for MapValue { + fn lamport(&self) -> Lamport { + self.lamp + } +} + +impl Serialize for MapValue { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut s = serializer.serialize_struct("MapValue", 2)?; + s.serialize_field("value", &self.value)?; + s.serialize_field("lamport", &self.lamp)?; + s.serialize_field("id", &self.idlp())?; + s.end() + } +} + +#[cfg(test)] +mod tests { + use std::hash::{Hash, Hasher}; + + use rustc_hash::FxHasher; + + use crate::{handler::ValueOrHandler, LoroValue}; + + use super::*; + + fn key(value: &str) -> InternalString { + value.into() + } + + fn map_value(lamp: Lamport, peer: PeerID, value: i64) -> MapValue { + MapValue { + value: Some(LoroValue::I64(value)), + lamp, + peer, + } + } + + fn resolved_value(lamp: Lamport, peer: PeerID, value: i64) -> ResolvedMapValue { + ResolvedMapValue { + value: Some(ValueOrHandler::Value(LoroValue::I64(value))), + idlp: IdLp::new(peer, lamp), + } + } + + fn hash_map_value(value: &MapValue) -> u64 { + let mut hasher = FxHasher::default(); + value.hash(&mut hasher); + hasher.finish() + } + + #[test] + fn map_value_ordering_identity_and_hash_are_timestamp_based() { + let lower_lamport = map_value(1, 9, 10); + let higher_lamport = map_value(2, 1, 20); + let lower_peer = map_value(2, 1, 30); + let higher_peer = map_value(2, 2, 40); + + assert!(higher_lamport > lower_lamport); + assert!(higher_peer > lower_peer); + assert_eq!(higher_peer.idlp(), IdLp::new(2, 2)); + assert_eq!(higher_peer.lamport(), 2); + + let same_id_different_value = MapValue { + value: Some(LoroValue::I64(999)), + ..higher_peer.clone() + }; + assert_eq!(higher_peer, same_id_different_value); + assert_eq!( + hash_map_value(&higher_peer), + hash_map_value(&same_id_different_value) + ); + } + + #[test] + fn map_delta_compose_keeps_latest_timestamp_per_key() { + let initial = MapDelta::new() + .with_entry(key("same"), map_value(2, 1, 10)) + .with_entry(key("left-only"), map_value(1, 1, 11)); + + let mut incoming = MapDelta::new() + .with_entry(key("same"), map_value(2, 2, 20)) + .with_entry(key("right-only"), map_value(1, 1, 21)); + incoming.updated.insert(key("unset"), None); + + let composed = initial.compose(incoming); + assert_eq!( + composed + .updated + .get(&key("same")) + .and_then(|value| value.as_ref()) + .and_then(|value| value.value.as_ref()), + Some(&LoroValue::I64(20)) + ); + assert!(composed.updated.contains_key(&key("left-only"))); + assert!(composed.updated.contains_key(&key("right-only"))); + assert_eq!(composed.updated.get(&key("unset")), Some(&None)); + } + + #[test] + fn map_delta_compose_does_not_let_older_entries_overwrite_newer_ones() { + let initial = MapDelta::new().with_entry(key("same"), map_value(5, 1, 50)); + let incoming = MapDelta::new().with_entry(key("same"), map_value(4, 9, 40)); + + let composed = initial.compose(incoming); + let value = composed + .updated + .get(&key("same")) + .and_then(|value| value.as_ref()) + .and_then(|value| value.value.as_ref()); + assert_eq!(value, Some(&LoroValue::I64(50))); + } + + #[test] + fn resolved_map_delta_compose_and_transform_apply_the_same_conflict_contract() { + let left = ResolvedMapDelta::new() + .with_entry(key("same"), resolved_value(1, 1, 10)) + .with_entry(key("left-only"), resolved_value(1, 1, 11)); + let right = ResolvedMapDelta::new() + .with_entry(key("same"), resolved_value(1, 2, 20)) + .with_entry(key("right-only"), resolved_value(1, 1, 21)); + + let composed = left.compose(right.clone()); + let value = composed.updated.get(&key("same")).unwrap(); + assert_eq!(value.idlp, IdLp::new(2, 1)); + assert_eq!(value.value.as_ref().unwrap().to_value(), LoroValue::I64(20)); + assert!(composed.updated.contains_key(&key("left-only"))); + assert!(composed.updated.contains_key(&key("right-only"))); + + let mut left_prior = composed.clone(); + left_prior.transform(&right, true); + assert!(left_prior.updated.contains_key(&key("same"))); + + let mut right_prior = composed; + right_prior.transform(&right, false); + assert!(!right_prior.updated.contains_key(&key("same"))); + assert!(!right_prior.updated.contains_key(&key("right-only"))); + assert!(right_prior.updated.contains_key(&key("left-only"))); + } + + #[test] + fn resolved_unset_uses_max_lamport_to_win_against_material_values() { + let unset = ResolvedMapValue::new_unset(); + let value = resolved_value(Lamport::MAX - 1, PeerID::MAX, 1); + + let composed = ResolvedMapDelta::new() + .with_entry(key("field"), value) + .compose(ResolvedMapDelta::new().with_entry(key("field"), unset)); + let entry = composed.updated.get(&key("field")).unwrap(); + assert_eq!(entry.idlp, IdLp::new(PeerID::default(), Lamport::MAX)); + assert!(entry.value.is_none()); + } + + #[test] + fn map_value_serialization_exposes_value_lamport_and_id() { + let serialized = serde_json::to_value(map_value(7, 8, 42)).unwrap(); + assert_eq!(serialized["value"], serde_json::json!(42)); + assert_eq!(serialized["lamport"], serde_json::json!(7)); + assert_eq!( + serialized["id"], + serde_json::json!({ "peer": 8, "lamport": 7 }) + ); + } +} diff --git a/vendor/loro-internal/src/delta/movable_list.rs b/vendor/loro-internal/src/delta/movable_list.rs new file mode 100644 index 00000000..6c2a1e42 --- /dev/null +++ b/vendor/loro-internal/src/delta/movable_list.rs @@ -0,0 +1,56 @@ +use loro_common::{CompactIdLp, IdFull, IdLp, LoroValue}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +use super::{Delta, DeltaValue}; + +#[derive(Clone, Debug)] +pub(crate) struct MovableListInnerDelta { + pub list: Delta, ()>, + pub elements: FxHashMap, +} + +impl DeltaValue for SmallVec<[IdFull; 1]> { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + for v in other { + self.push(v) + } + + Ok(()) + } + + fn take(&mut self, length: usize) -> Self { + self.drain(..length).collect() + } + + fn length(&self) -> usize { + self.len() + } +} + +impl MovableListInnerDelta { + pub(crate) fn is_empty(&self) -> bool { + self.list.is_empty() && self.elements.is_empty() + } +} + +#[derive(Clone, Debug)] +pub struct ElementDelta { + /// This must be Some if it's in checkout mode (not fast-forward mode) + pub pos: Option, + pub value: LoroValue, + pub value_updated: bool, + /// This must be Some if it's in checkout mode (not fast-forward mode) + pub value_id: Option, +} + +impl ElementDelta { + pub fn placeholder() -> Self { + Self { + pos: None, + value: LoroValue::Null, + value_updated: false, + value_id: None, + } + } +} diff --git a/vendor/loro-internal/src/delta/seq.rs b/vendor/loro-internal/src/delta/seq.rs new file mode 100644 index 00000000..eed69a55 --- /dev/null +++ b/vendor/loro-internal/src/delta/seq.rs @@ -0,0 +1,950 @@ +use enum_as_inner::EnumAsInner; +use serde::Serialize; +use std::fmt::Debug; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Delta { + pub(crate) vec: Vec>, +} + +impl Serialize for Delta { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.vec.iter()) + } +} + +#[derive(Debug, EnumAsInner, Clone, PartialEq, Eq, Serialize)] +pub enum DeltaItem { + Retain { retain: usize, attributes: Meta }, + Insert { insert: Value, attributes: Meta }, + Delete { delete: usize, attributes: Meta }, +} + +#[derive(PartialEq, Debug, Eq)] +pub enum DeltaType { + Retain, + Insert, + Delete, +} + +/// The metadata of a DeltaItem +/// If empty metadata is used to override the older one, we will remove the metadata into None +pub trait Meta: Debug + Clone + PartialEq + Default { + fn empty() -> Self { + Self::default() + } + fn is_empty(&self) -> bool; + + /// this is used when composing two [DeltaItem]s with the same length + fn compose(&mut self, other: &Self, type_pair: (DeltaType, DeltaType)); + #[allow(unused)] + fn take(&mut self, other: &Self) -> Self { + self.clone() + } + + fn is_mergeable(&self, other: &Self) -> bool; + /// This is used when we merge two [DeltaItem]s. + /// And it's guaranteed that [Meta::is_mergeable] is true + fn merge(&mut self, other: &Self); +} + +impl Meta for () { + fn empty() -> Self {} + fn is_empty(&self) -> bool { + true + } + + fn compose(&mut self, _other: &Self, _type_pair: (DeltaType, DeltaType)) {} + + fn is_mergeable(&self, _other: &Self) -> bool { + true + } + + fn merge(&mut self, _other: &Self) {} +} + +/// The value of [DeltaItem::Insert] +pub trait DeltaValue: Debug + Sized { + /// the other will be merged into self + fn value_extend(&mut self, other: Self) -> Result<(), Self>; + /// takes the first number of `length` elements + fn take(&mut self, length: usize) -> Self; + /// the length of the value + fn length(&self) -> usize; +} + +impl DeltaValue for DeltaItem { + fn value_extend(&mut self, _other: Self) -> Result<(), Self> { + unreachable!() + } + + fn take(&mut self, _length: usize) -> Self { + unreachable!() + } + + fn length(&self) -> usize { + match self { + DeltaItem::Retain { + retain: len, + attributes: _, + } => *len, + DeltaItem::Insert { + insert: value, + attributes: _, + } => value.length(), + DeltaItem::Delete { + delete: len, + attributes: _, + } => *len, + } + } +} + +impl DeltaItem { + pub fn meta(&self) -> &M { + match self { + DeltaItem::Insert { + attributes: meta, .. + } => meta, + DeltaItem::Retain { + attributes: meta, .. + } => meta, + DeltaItem::Delete { + delete: _, + attributes: meta, + } => meta, + } + } + + pub fn meta_mut(&mut self) -> &mut M { + match self { + DeltaItem::Insert { + attributes: meta, .. + } => meta, + DeltaItem::Retain { + attributes: meta, .. + } => meta, + DeltaItem::Delete { + delete: _, + attributes: meta, + } => meta, + } + } + + pub fn set_meta(&mut self, meta: M) { + match self { + DeltaItem::Insert { attributes: m, .. } => *m = meta, + DeltaItem::Retain { attributes: m, .. } => *m = meta, + DeltaItem::Delete { + delete: _, + attributes: m, + } => *m = meta, + } + } + + fn type_(&self) -> DeltaType { + match self { + DeltaItem::Insert { .. } => DeltaType::Insert, + DeltaItem::Retain { .. } => DeltaType::Retain, + DeltaItem::Delete { .. } => DeltaType::Delete, + } + } + + pub fn compose_meta(&mut self, other: &Self) { + let type_pair = (self.type_(), other.type_()); + let meta = self.meta_mut(); + let other_meta = other.meta(); + Meta::compose(meta, other_meta, type_pair); + } + + // change self-length to self.len()-length + // and return the taken one. + pub(crate) fn take(&mut self, length: usize) -> Self { + match self { + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + let v = value.take(length); + Self::Insert { + insert: v, + attributes: meta.clone(), + } + } + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + *len -= length; + Self::Retain { + retain: length, + attributes: meta.clone(), + } + } + DeltaItem::Delete { + delete: len, + attributes: _, + } => { + *len -= length; + Self::Delete { + delete: length, + // meta may store utf16 length, this take will invalidate it + attributes: M::empty(), + } + } + } + } + + pub(crate) fn take_with_meta_ref(&mut self, length: usize, other_meta: &Self) -> Self { + match self { + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + let v = value.take(length); + Self::Insert { + insert: v, + attributes: meta.take(other_meta.meta()), + } + } + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + *len -= length; + Self::Retain { + retain: length, + attributes: meta.take(other_meta.meta()), + } + } + DeltaItem::Delete { + delete: len, + attributes: meta, + } => { + *len -= length; + Self::Delete { + delete: length, + attributes: meta.take(other_meta.meta()), + } + } + } + } + + fn insert_inner(self) -> Value { + match self { + DeltaItem::Insert { insert: value, .. } => value, + _ => unreachable!(), + } + } +} + +#[derive(Debug)] +pub struct DeltaIterator { + // The reversed Vec uses pop() to simulate getting the first element each time + ops: Vec>, +} + +impl DeltaIterator { + fn new(ops: Vec>) -> Self { + Self { ops } + } + + #[inline(always)] + fn next>>(&mut self, len: L) -> DeltaItem { + self.next_impl(len.into()) + } + + fn next_impl(&mut self, len: Option) -> DeltaItem { + let length = len.unwrap_or(usize::MAX); + let next_op = self.peek_mut(); + if next_op.is_none() { + return DeltaItem::Retain { + retain: usize::MAX, + attributes: M::empty(), + }; + } + let op = next_op.unwrap(); + let op_length = op.length(); + if length < op_length { + // a part of the peek op + op.take(length) + } else { + self.take_peek().unwrap() + } + } + + fn next_with_ref(&mut self, len: usize, other: &DeltaItem) -> DeltaItem { + let next_op = self.peek_mut(); + if next_op.is_none() { + return DeltaItem::Retain { + retain: other.length(), + attributes: M::empty(), + }; + } + let op = next_op.unwrap(); + let op_length = op.length(); + if len < op_length { + // a part of the peek op + op.take_with_meta_ref(len, other) + } else { + self.take_peek().unwrap() + } + } + + fn next_pair(&mut self, other: &mut Self) -> (DeltaItem, DeltaItem) { + let self_len = self.peek_length(); + let other_len = other.peek_length(); + if self_len > other_len { + let length = other_len; + let other_op = other.next(None); + debug_assert_eq!(other_op.length(), length); + let this_op = self.next_with_ref(length, &other_op); + (this_op, other_op) + } else { + let length = self_len; + let this_op = self.next(None); + debug_assert_eq!(this_op.length(), length); + let other_op = other.next_with_ref(length, &this_op); + (this_op, other_op) + } + } + + fn peek_mut(&mut self) -> Option<&mut DeltaItem> { + self.ops.last_mut() + } + + fn take_peek(&mut self) -> Option> { + self.ops.pop() + } + + fn rest(mut self) -> Vec> { + self.ops.reverse(); + self.ops + } + + fn has_next(&self) -> bool { + !self.ops.is_empty() + } + + fn peek(&self) -> Option<&DeltaItem> { + self.ops.last() + } + + fn peek_length(&self) -> usize { + if let Some(op) = self.peek() { + op.length() + } else { + usize::MAX + } + } + + fn peek_is_insert(&self) -> bool { + if let Some(op) = self.peek() { + op.is_insert() + } else { + false + } + } + + fn peek_is_delete(&self) -> bool { + if let Some(op) = self.peek() { + op.is_delete() + } else { + false + } + } +} + +impl Delta { + pub fn new() -> Self { + Self { vec: Vec::new() } + } + + pub fn items(&self) -> &[DeltaItem] { + &self.vec + } + + pub fn inner(self) -> Vec> { + self.vec + } + + pub fn retain_with_meta(mut self, len: usize, meta: M) -> Self { + if len == 0 { + // currently no meta for retain if len == 0 + return self; + } + + self.push(DeltaItem::Retain { + retain: len, + attributes: meta, + }); + self + } + + pub fn insert_with_meta>(mut self, value: V, meta: M) -> Self { + self.push(DeltaItem::Insert { + insert: value.into(), + attributes: meta, + }); + self + } + + pub fn delete_with_meta(mut self, len: usize, meta: M) -> Self { + self.push(DeltaItem::Delete { + delete: len, + attributes: meta, + }); + self + } + + pub fn delete(mut self, len: usize) -> Self { + if len == 0 { + return self; + } + self.push(DeltaItem::Delete { + delete: len, + attributes: M::empty(), + }); + self + } + + pub fn retain(mut self, len: usize) -> Self { + if len == 0 { + return self; + } + + self.push(DeltaItem::Retain { + retain: len, + attributes: M::empty(), + }); + self + } + + pub fn insert>(mut self, value: V) -> Self { + self.push(DeltaItem::Insert { + insert: value.into(), + attributes: M::empty(), + }); + self + } + + // If the new_op is merged, return true + pub fn push(&mut self, new_op: DeltaItem) -> bool { + let mut index = self.vec.len(); + let last_op = self.vec.pop(); + if let Some(mut last_op) = last_op { + if new_op.is_delete() && last_op.is_delete() { + *last_op.as_delete_mut().unwrap().0 += new_op.length(); + self.vec.push(last_op); + return true; + } + // Since it does not matter if we insert before or after deleting at the same index, + // always prefer to insert first + if last_op.is_delete() && new_op.is_insert() { + index -= 1; + let _last_op = self.vec.pop(); + self.vec.push(last_op); + if let Some(_last_op_inner) = _last_op { + last_op = _last_op_inner; + } else { + self.vec.insert(0, new_op); + return true; + }; + } + if last_op.meta().is_mergeable(new_op.meta()) { + if new_op.is_insert() && last_op.is_insert() { + let value = last_op.as_insert_mut().unwrap().0; + let meta = new_op.meta().clone(); + match value.value_extend(new_op.insert_inner()) { + Ok(_) => { + last_op.meta_mut().merge(&meta); + self.vec.insert(index - 1, last_op); + return true; + } + Err(inner) => { + self.vec.insert(index - 1, last_op); + self.vec.insert( + index, + DeltaItem::Insert { + insert: inner, + attributes: meta, + }, + ); + return false; + } + } + } else if new_op.is_retain() && last_op.is_retain() { + *last_op.as_retain_mut().unwrap().0 += new_op.length(); + last_op.meta_mut().merge(new_op.meta()); + // self.vec.push(last_op); + self.vec.insert(index - 1, last_op); + return true; + } + } + + // self.vec.push(last_op); + self.vec.insert(index - 1, last_op); + } + if index == self.vec.len() { + self.vec.push(new_op); + false + } else { + self.vec.insert(index, new_op); + true + } + } + + pub fn iter(&self) -> impl Iterator> { + self.vec.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator> { + self.vec.iter_mut() + } + + pub fn into_op_iter(self) -> DeltaIterator { + DeltaIterator::new(self.vec.into_iter().rev().collect()) + } + + pub fn len(&self) -> usize { + self.vec.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Reference: [Quill Delta](https://github.com/quilljs/delta) + // TODO: PERF use &mut self and &other + pub fn compose(self, other: Delta) -> Delta { + let mut this_iter = self.into_op_iter(); + let mut other_iter = other.into_op_iter(); + let mut ops = vec![]; + let first_other = other_iter.peek(); + if let Some(first_other) = first_other { + // if other.delta starts with retain, we insert corresponding number of inserts from self.delta + if first_other.is_retain() && first_other.meta().is_empty() { + let mut first_left = first_other.length(); + let mut first_this = this_iter.peek(); + while let Some(first_this_inner) = first_this { + if first_this_inner.is_insert() && first_this_inner.length() <= first_left { + first_left -= first_this_inner.length(); + let mut op = this_iter.next(None); + op.compose_meta(first_other); + ops.push(op); + first_this = this_iter.peek(); + } else { + break; + } + } + if first_other.length() - first_left > 0 { + other_iter.next(first_other.length() - first_left); + } + } + } + let mut delta = Delta { vec: ops }; + while this_iter.has_next() || other_iter.has_next() { + if other_iter.peek_is_insert() { + // nothing to compose here + delta.push(other_iter.next(None)); + } else if this_iter.peek_is_delete() { + // nothing to compose here + delta.push(this_iter.next(None)); + } else { + // possible cases: + // 1. this: insert, other: retain + // 2. this: retain, other: retain + // 3. this: retain, other: delete + // 4. this: insert, other: delete + + let (mut this_op, mut other_op) = this_iter.next_pair(&mut other_iter); + if other_op.is_retain() { + // 1. this: insert, other: retain + // 2. this: retain, other: retain + this_op.compose_meta(&other_op); + let merged = delta.push(this_op); + let concat_rest = !other_iter.has_next() && !merged; + if concat_rest { + let vec = this_iter.rest(); + if vec.is_empty() { + break; + } + + let rest = Delta { vec }; + delta = delta.concat(rest); + break; + } + } else if other_op.is_delete() && this_op.is_retain() { + // 3. this: retain, other: delete + other_op.compose_meta(&this_op); + // other deletes the retained text + delta.push(other_op); + } else { + // 4. this: insert, other: delete + // nothing to do here, because insert and delete have the same length + } + } + } + + delta.chop() + } + + pub(crate) fn concat(mut self, mut other: Self) -> Self { + if !other.vec.is_empty() { + let other_first = other.vec.remove(0); + self.push(other_first); + self.vec.extend(other.vec); + } + self + } + + pub fn chop(mut self) -> Self { + loop { + match self.vec.last() { + Some(last_op) if last_op.is_retain() && last_op.meta().is_empty() => { + self.vec.pop(); + } + _ => break, + } + } + + self + } + + pub fn insert_len(&self) -> usize { + self.vec + .iter() + .map(|x| match x { + DeltaItem::Insert { insert, .. } => insert.length(), + _ => 0, + }) + .sum() + } +} + +impl IntoIterator for Delta { + type Item = DeltaItem; + + type IntoIter = std::vec::IntoIter>; + fn into_iter(self) -> Self::IntoIter { + self.vec.into_iter() + } +} + +impl Default for Delta { + fn default() -> Self { + Self::new() + } +} + +impl DeltaValue for Vec { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + self.extend(other); + Ok(()) + } + + fn take(&mut self, length: usize) -> Self { + let mut new = self.split_off(length); + std::mem::swap(self, &mut new); + new + } + + fn length(&self) -> usize { + self.len() + } +} + +impl DeltaValue for String { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + self.push_str(&other); + Ok(()) + } + + fn take(&mut self, length: usize) -> Self { + let mut new = self.split_off(length); + std::mem::swap(self, &mut new); + new + } + fn length(&self) -> usize { + self.len() + } +} + +#[cfg(test)] +mod test { + use super::{Delta, DeltaItem, DeltaType, Meta}; + + #[derive(Debug, PartialEq, Clone, Default)] + struct TestMeta { + bold: Option>, + color: Option>, + } + + impl Meta for TestMeta { + fn empty() -> Self { + TestMeta { + bold: None, + color: None, + } + } + fn is_empty(&self) -> bool { + self.bold.is_none() && self.color.is_none() + } + + fn compose( + &mut self, + other: &Self, + (this_type, other_type): (super::DeltaType, super::DeltaType), + ) { + if this_type != DeltaType::Delete && other_type != DeltaType::Delete { + if let Some(other_bold) = other.bold { + self.bold = Some(other_bold); + } + if let Some(other_color) = other.color { + self.color = Some(other_color); + } + } + } + + fn is_mergeable(&self, other: &Self) -> bool { + self == other + } + + fn merge(&mut self, _other: &Self) {} + } + + type TestDelta = Delta; + const BOLD_META: TestMeta = TestMeta { + bold: Some(Some(true)), + color: None, + }; + const COLOR_META: TestMeta = TestMeta { + bold: None, + color: Some(Some(())), + }; + const EMPTY_META: TestMeta = TestMeta { + bold: Some(None), + color: Some(None), + }; + const BOTH_META: TestMeta = TestMeta { + bold: Some(Some(true)), + color: Some(Some(())), + }; + + #[test] + fn delta_push() { + let mut a: Delta = Delta::new().insert("a".to_string()); + a.push(DeltaItem::Insert { + insert: "b".to_string(), + attributes: (), + }); + assert_eq!(a, Delta::new().insert("ab".to_string())); + } + + #[test] + fn delta_compose() { + let a: Delta = Delta::new().retain(3).insert("abcde".to_string()); + let b = Delta::new().retain(5).delete(6); + assert_eq!( + a.compose(b), + Delta::new().retain(3).insert("ab".to_string()).delete(3) + ); + + let a: Delta = Delta::new().insert("123".to_string()); + let b = Delta::new().retain(1).insert("123".to_string()); + assert_eq!(a.compose(b), Delta::new().insert("112323".to_string())); + } + + #[test] + fn delete_failed() { + let a: Delta = Delta::new() + .retain(2) + .insert("[31354]") + .retain(1) + .insert("[31354]") + .retain(12) + .insert("[31354]"); + let b: Delta = Delta::new().retain(27).delete(3); + assert_eq!( + a.compose(b), + Delta::new() + .retain(2) + .insert("[31354]") + .retain(1) + .insert("[31354]") + .retain(10) + .insert("31354]") + .delete(2) + ); + } + + #[test] + fn insert_insert() { + let a = TestDelta::new().insert("a"); + let b = TestDelta::new().insert("b"); + assert_eq!(a.compose(b), TestDelta::new().insert("b").insert("a")); + } + + #[test] + fn insert_retain() { + let a = TestDelta::new().insert("a"); + let b = TestDelta::new().retain_with_meta(1, BOLD_META); + assert_eq!( + a.compose(b), + TestDelta::new().insert_with_meta("a", BOLD_META) + ); + } + + #[test] + fn insert_delete() { + let a = TestDelta::new().insert("a"); + let b = TestDelta::new().delete(1); + assert_eq!(a.compose(b), TestDelta::new()); + } + + #[test] + fn delete_insert() { + let a = TestDelta::new().delete(1); + let b = TestDelta::new().insert("b"); + assert_eq!(a.compose(b), TestDelta::new().insert("b").delete(1)); + } + + #[test] + fn delete_retain() { + let a = TestDelta::new().delete(1); + let b = TestDelta::new().retain_with_meta(1, BOLD_META); + assert_eq!( + a.compose(b), + TestDelta::new().delete(1).retain_with_meta(1, BOLD_META) + ); + } + + #[test] + fn delete_delete() { + let a = TestDelta::new().delete(1); + let b = TestDelta::new().delete(1); + assert_eq!(a.compose(b), TestDelta::new().delete(2)); + } + + #[test] + fn retain_insert() { + let a = TestDelta::new().retain_with_meta(1, BOLD_META); + let b = TestDelta::new().insert("b"); + assert_eq!( + a.compose(b), + TestDelta::new().insert("b").retain_with_meta(1, BOLD_META) + ); + } + + #[test] + fn retain_retain() { + let a = TestDelta::new().retain_with_meta(1, BOLD_META); + let b = TestDelta::new().retain_with_meta(1, COLOR_META); + assert_eq!( + a.compose(b), + TestDelta::new().retain_with_meta(1, BOTH_META) + ); + } + + #[test] + fn retain_delete() { + let a = TestDelta::new().retain_with_meta(1, BOLD_META); + let b = TestDelta::new().delete(1); + assert_eq!(a.compose(b), TestDelta::new().delete(1)); + } + + #[test] + fn delete_entire() { + let a = TestDelta::new().retain(4).insert("abcde"); + let b = TestDelta::new().delete(9); + assert_eq!(a.compose(b), TestDelta::new().delete(4)); + } + + #[test] + fn retain_more() { + let a = TestDelta::new().insert("abcde"); + let b = TestDelta::new().retain(10); + assert_eq!(a.compose(b), TestDelta::new().insert("abcde")); + } + + #[test] + fn remove_meta() { + let a = TestDelta::new().insert_with_meta("a", BOLD_META); + let b = TestDelta::new().retain_with_meta(1, EMPTY_META); + assert_eq!( + a.compose(b), + TestDelta::new().insert_with_meta("a", EMPTY_META) + ); + } + + #[test] + fn retain_start_opt() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .delete(1); + let b = TestDelta::new().retain(3).insert("d"); + let expect = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .insert("d") + .delete(1); + assert_eq!(a.compose(b), expect); + } + + #[test] + fn retain_start_opt_split() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .retain(5) + .delete(1); + let b = TestDelta::new().retain(4).insert("d"); + let expect = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .retain(1) + .insert("d") + .retain(4) + .delete(1); + assert_eq!(a.compose(b), expect); + } + + #[test] + fn retain_end_opt() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META); + let b = TestDelta::new().delete(1); + let expect = TestDelta::new() + .insert("b") + .insert_with_meta("c", BOLD_META); + assert_eq!(a.compose(b), expect); + } + + #[test] + fn retain_end_opt_join() { + let a = TestDelta::new() + .insert_with_meta("a", BOLD_META) + .insert("b") + .insert_with_meta("c", BOLD_META) + .insert("d") + .insert_with_meta("e", BOLD_META) + .insert("f"); + let b = TestDelta::new().retain(1).delete(1); + let expect = TestDelta::new() + .insert_with_meta("ac", BOLD_META) + .insert("d") + .insert_with_meta("e", BOLD_META) + .insert("f"); + assert_eq!(a.compose(b), expect); + } +} diff --git a/vendor/loro-internal/src/delta/text.rs b/vendor/loro-internal/src/delta/text.rs new file mode 100644 index 00000000..7d4051aa --- /dev/null +++ b/vendor/loro-internal/src/delta/text.rs @@ -0,0 +1,332 @@ +use loro_common::{InternalString, LoroValue, PeerID}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; + +use crate::change::Lamport; +use crate::container::richtext::{Style, Styles}; +use crate::event::TextMeta; +use crate::ToJson; + +use super::Meta; + +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StyleMeta { + map: FxHashMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StyleMetaItem { + // We need lamport and peer to compose the event + pub lamport: Lamport, + pub peer: PeerID, + pub value: LoroValue, +} + +impl StyleMetaItem { + pub fn try_replace(&mut self, other: &StyleMetaItem) { + if (self.lamport, self.peer) < (other.lamport, other.peer) { + self.lamport = other.lamport; + self.peer = other.peer; + self.value = other.value.clone(); + } + } +} + +impl From<&Styles> for StyleMeta { + fn from(styles: &Styles) -> Self { + let mut map = FxHashMap::with_capacity_and_hasher(styles.len(), Default::default()); + for (key, value) in styles.iter() { + if let Some(value) = value.get() { + map.insert( + key.key().clone(), + StyleMetaItem { + value: value.to_value(), + lamport: value.lamport, + peer: value.peer, + }, + ); + } + } + Self { map } + } +} + +impl From for StyleMeta { + fn from(styles: Styles) -> Self { + let temp = &styles; + temp.into() + } +} + +impl Meta for StyleMeta { + fn is_empty(&self) -> bool { + self.map.is_empty() + } + + fn compose(&mut self, other: &Self, _type_pair: (super::DeltaType, super::DeltaType)) { + for (key, value) in other.map.iter() { + match self.map.get_mut(key) { + Some(old_value) => { + old_value.try_replace(value); + } + None => { + self.map.insert(key.clone(), value.clone()); + } + } + } + } + + fn is_mergeable(&self, other: &Self) -> bool { + self.map == other.map + } + + fn merge(&mut self, _: &Self) {} +} + +impl Meta for TextMeta { + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn compose(&mut self, other: &Self, _: (super::DeltaType, super::DeltaType)) { + for (key, value) in other.0.iter() { + self.0.insert(key.clone(), value.clone()); + } + } + + fn is_mergeable(&self, other: &Self) -> bool { + self.0 == other.0 + } + + fn merge(&mut self, _: &Self) {} +} + +impl StyleMeta { + pub(crate) fn iter(&self) -> impl Iterator + '_ { + self.map.iter().map(|(key, style)| { + ( + key.clone(), + Style { + key: key.clone(), + data: style.value.clone(), + }, + ) + }) + } + + pub(crate) fn insert(&mut self, key: InternalString, value: StyleMetaItem) { + self.map.insert(key, value); + } + + pub(crate) fn contains_key(&self, key: &InternalString) -> bool { + self.map.contains_key(key) + } + + pub(crate) fn to_value(&self) -> LoroValue { + LoroValue::Map(self.to_map_without_null_value().into()) + } + + /// §act-eleven A11.10 (flowstate vendor patch): the null-filtered attribute + /// map exactly as the delta value's `attributes` entry carries it — an + /// unmarked style (key → null) is INVISIBLE. Public so streaming-span + /// consumers (`TextHandler::for_each_richtext_span`) can build delta + /// segments identical to `get_richtext_value`'s. + pub fn to_visible_attributes(&self) -> FxHashMap { + self.to_map_without_null_value() + } + + /// §act-eleven A11.10 (flowstate vendor patch): equality on the VISIBLE + /// (null-filtered) attribute values — the exact merge criterion + /// `get_richtext_value` applies between adjacent spans (it compares the + /// null-filtered `to_value()` maps). Allocation-free, so streaming + /// consumers can use it per span. + pub fn visible_eq(&self, other: &Self) -> bool { + let visible_len = |meta: &Self| { + meta.map + .values() + .filter(|item| !item.value.is_null()) + .count() + }; + visible_len(self) == visible_len(other) + && self + .map + .iter() + .filter(|(_, item)| !item.value.is_null()) + .all(|(key, item)| { + other + .map + .get(key) + .is_some_and(|o| !o.value.is_null() && o.value == item.value) + }) + } + + fn to_map_without_null_value(&self) -> FxHashMap { + self.map + .iter() + .filter_map(|(key, value)| { + if value.value.is_null() { + None + } else { + Some((key.to_string(), value.value.clone())) + } + }) + .collect() + } + + pub(crate) fn to_map(&self) -> FxHashMap { + self.map + .iter() + .map(|(key, value)| (key.to_string(), value.value.clone())) + .collect() + } + + pub(crate) fn to_option_map(&self) -> Option> { + if self.is_empty() { + return None; + } + + Some(self.to_map()) + } + + pub(crate) fn to_option_map_without_null_value(&self) -> Option> { + let map = self.to_map_without_null_value(); + if map.is_empty() { + None + } else { + Some(map) + } + } +} + +impl ToJson for TextMeta { + fn to_json_value(&self) -> serde_json::Value { + let mut map = serde_json::Map::new(); + for (key, value) in self.0.iter() { + let value = serde_json::to_value(value).unwrap(); + map.insert(key.to_string(), value); + } + + serde_json::Value::Object(map) + } + + fn from_json(s: &str) -> Self { + let map: FxHashMap = serde_json::from_str(s).unwrap(); + TextMeta(map) + } +} + +#[cfg(test)] +mod tests { + use rustc_hash::FxHashMap; + + use super::*; + use crate::delta::{DeltaType, Meta}; + + fn key(value: &str) -> InternalString { + value.into() + } + + fn style_item(lamport: Lamport, peer: PeerID, value: LoroValue) -> StyleMetaItem { + StyleMetaItem { + lamport, + peer, + value, + } + } + + #[test] + fn style_meta_item_replacement_is_ordered_by_lamport_then_peer() { + let mut item = style_item(1, 9, LoroValue::String("old".into())); + item.try_replace(&style_item(1, 8, LoroValue::String("ignored".into()))); + assert_eq!(item.value, LoroValue::String("old".into())); + + item.try_replace(&style_item(1, 10, LoroValue::String("peer-wins".into()))); + assert_eq!(item.value, LoroValue::String("peer-wins".into())); + + item.try_replace(&style_item(2, 0, LoroValue::String("lamport-wins".into()))); + assert_eq!(item.value, LoroValue::String("lamport-wins".into())); + } + + #[test] + fn style_meta_compose_keeps_latest_value_per_key_and_preserves_new_keys() { + let mut left = StyleMeta::default(); + left.insert(key("bold"), style_item(2, 1, LoroValue::Bool(true))); + left.insert( + key("color"), + style_item(1, 1, LoroValue::String("red".into())), + ); + + let mut right = StyleMeta::default(); + right.insert(key("bold"), style_item(1, 99, LoroValue::Bool(false))); + right.insert( + key("color"), + style_item(3, 0, LoroValue::String("blue".into())), + ); + right.insert( + key("link"), + style_item(1, 1, LoroValue::String("docs".into())), + ); + + left.compose(&right, (DeltaType::Retain, DeltaType::Retain)); + let values = left.to_map(); + assert_eq!(values.get("bold"), Some(&LoroValue::Bool(true))); + assert_eq!(values.get("color"), Some(&LoroValue::String("blue".into()))); + assert_eq!(values.get("link"), Some(&LoroValue::String("docs".into()))); + } + + #[test] + fn style_meta_value_views_distinguish_null_from_absent() { + let mut meta = StyleMeta::default(); + assert!(meta.is_empty()); + assert_eq!(meta.to_option_map(), None); + + meta.insert(key("bold"), style_item(1, 1, LoroValue::Bool(true))); + meta.insert(key("deleted"), style_item(1, 2, LoroValue::Null)); + + assert!(meta.contains_key(&key("deleted"))); + assert_eq!( + meta.to_map().get("deleted"), + Some(&LoroValue::Null), + "to_map keeps explicit null style values" + ); + assert_eq!( + meta.to_map_without_null_value().get("deleted"), + None, + "serialized style values omit null entries" + ); + assert_eq!( + meta.to_value(), + LoroValue::Map( + FxHashMap::from_iter([(String::from("bold"), LoroValue::Bool(true))]).into() + ) + ); + + let styles: FxHashMap<_, _> = meta.iter().map(|style| (style.0, style.1.data)).collect(); + assert_eq!(styles.get(&key("bold")), Some(&LoroValue::Bool(true))); + assert_eq!(styles.get(&key("deleted")), Some(&LoroValue::Null)); + } + + #[test] + fn text_meta_compose_and_json_roundtrip_are_map_like() { + let mut left = TextMeta(FxHashMap::from_iter([( + String::from("lang"), + LoroValue::String("en".into()), + )])); + let right = TextMeta(FxHashMap::from_iter([ + (String::from("lang"), LoroValue::String("fr".into())), + (String::from("author"), LoroValue::String("loro".into())), + ])); + + left.compose(&right, (DeltaType::Retain, DeltaType::Retain)); + assert_eq!(left.0.get("lang"), Some(&LoroValue::String("fr".into()))); + assert_eq!( + left.0.get("author"), + Some(&LoroValue::String("loro".into())) + ); + assert!(left.is_mergeable(&left.clone())); + + let json = left.to_json_value(); + let decoded = TextMeta::from_json(&json.to_string()); + assert_eq!(decoded.0, left.0); + } +} diff --git a/vendor/loro-internal/src/delta/tree.rs b/vendor/loro-internal/src/delta/tree.rs new file mode 100644 index 00000000..8f368e59 --- /dev/null +++ b/vendor/loro-internal/src/delta/tree.rs @@ -0,0 +1,209 @@ +use fractional_index::FractionalIndex; +use itertools::Itertools; +use loro_common::{IdFull, TreeID}; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; + +use crate::state::TreeParentId; + +#[derive(Debug, Clone, Default)] +pub struct TreeDiff { + pub diff: Vec, +} + +#[derive(Debug, Clone)] +pub struct TreeDiffItem { + pub target: TreeID, + pub action: TreeExternalDiff, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum TreeExternalDiff { + Create { + parent: TreeParentId, + index: usize, + position: FractionalIndex, + }, + Move { + parent: TreeParentId, + index: usize, + position: FractionalIndex, + old_parent: TreeParentId, + old_index: usize, + }, + Delete { + old_parent: TreeParentId, + old_index: usize, + }, +} + +impl TreeDiff { + pub(crate) fn compose(mut self, other: Self) -> Self { + self.diff.extend(other.diff); + // self = compose_tree_diff(&self); + self + } + + pub(crate) fn extend>(mut self, other: I) -> Self { + self.diff.extend(other); + self + } + + fn to_hash_map_mut(&mut self) -> FxHashMap { + let mut ans = FxHashSet::default(); + for index in (0..self.diff.len()).rev() { + let target = self.diff[index].target; + if ans.contains(&target) { + self.diff.remove(index); + continue; + } + ans.insert(target); + } + self.iter() + .map(|x| x.target) + .enumerate() + .map(|(i, x)| (x, i)) + .collect() + } + + pub(crate) fn transform(&mut self, b: &TreeDiff, left_prior: bool) { + // println!("\ntransform prior {:?} {:?} \nb {:?}", left_prior, self, b); + if b.is_empty() || self.is_empty() { + return; + } + if !left_prior { + let mut self_update = self.to_hash_map_mut(); + for i in b + .iter() + .map(|x| x.target) + .filter_map(|x| self_update.remove(&x)) + .sorted() + .rev() + { + self.remove(i); + } + } + } +} + +/// Representation of differences in movable tree. It's an ordered list of [`TreeDiff`]. +#[derive(Clone, Default)] +pub struct TreeDelta { + pub(crate) diff: Vec, +} + +impl Debug for TreeDelta { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("TreeDelta{ diff: [\n")?; + for item in self.diff.iter() { + f.write_fmt(format_args!("\t{:?}, \n", item))?; + } + f.write_str("]}") + } +} + +/// The semantic action in movable tree. +#[derive(Debug, Clone)] +pub struct TreeDeltaItem { + pub target: TreeID, + pub action: TreeInternalDiff, + pub last_effective_move_op_id: IdFull, +} + +/// The action of [`TreeDiff`]. It's the same as [`crate::container::tree::tree_op::TreeOp`], but semantic. +#[derive(Debug, Clone)] +pub enum TreeInternalDiff { + /// First create the node, have not seen it before + Create { + parent: TreeParentId, + position: FractionalIndex, + }, + /// For retreating, if the node is only created, not move it to `DELETED_ROOT` but delete it directly + UnCreate, + /// Move the node to the parent, the node exists + Move { + parent: TreeParentId, + position: FractionalIndex, + }, + /// move under a parent that is deleted + Delete { + parent: TreeParentId, + position: Option, + }, + /// old parent is deleted, new parent is deleted too + MoveInDelete { + parent: TreeParentId, + position: Option, + }, +} + +impl TreeDeltaItem { + /// * `is_new_parent_deleted` and `is_old_parent_deleted`: we need to infer whether it's a `creation`. + /// It's a creation if the old_parent is deleted but the new parent isn't. + /// If it is a creation, we need to emit the `Create` event so that downstream event handler can + /// handle the new containers easier. + pub(crate) fn new( + target: TreeID, + parent: TreeParentId, + old_parent: TreeParentId, + op_id: IdFull, + is_new_parent_deleted: bool, + is_old_parent_deleted: bool, + position: Option, + ) -> Self { + let action = if matches!(parent, TreeParentId::Unexist) { + TreeInternalDiff::UnCreate + } else { + match ( + is_new_parent_deleted, + is_old_parent_deleted || old_parent == TreeParentId::Unexist, + ) { + (true, true) => TreeInternalDiff::MoveInDelete { parent, position }, + (true, false) => TreeInternalDiff::Delete { parent, position }, + (false, true) => TreeInternalDiff::Create { + parent, + position: position.unwrap(), + }, + (false, false) => TreeInternalDiff::Move { + parent, + position: position.unwrap(), + }, + } + }; + + TreeDeltaItem { + target, + action, + last_effective_move_op_id: op_id, + } + } +} + +impl Deref for TreeDelta { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.diff + } +} + +impl TreeDelta { + // TODO: cannot handle this for now + pub(crate) fn compose(mut self, x: TreeDelta) -> TreeDelta { + self.diff.extend(x.diff); + self + } +} + +impl Deref for TreeDiff { + type Target = Vec; + fn deref(&self) -> &Self::Target { + &self.diff + } +} + +impl DerefMut for TreeDiff { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.diff + } +} diff --git a/vendor/loro-internal/src/diff/LICENSE b/vendor/loro-internal/src/diff/LICENSE new file mode 100644 index 00000000..f49a4e16 --- /dev/null +++ b/vendor/loro-internal/src/diff/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/loro-internal/src/diff/diff_impl.rs b/vendor/loro-internal/src/diff/diff_impl.rs new file mode 100644 index 00000000..aef44cd5 --- /dev/null +++ b/vendor/loro-internal/src/diff/diff_impl.rs @@ -0,0 +1,857 @@ +//! This file was copied from the similar crate at +//! https://github.com/mitsuhiko/similar/blob/2b31f65445df9093ba007ca5a5ae6a71b899d491/src/algorithms/myers.rs +//! The original license is in the LICENSE file in the same directory as this file +//! +//! This file has been optimized for constant performance, removed some unnecessary cases +//! and simplified the use of DiffHandler. +//! Myers' diff algorithm. +//! +//! * time: `O((N+M)D)` +//! * space `O(N+M)` +//! +//! See [the original article by Eugene W. Myers](http://www.xmailserver.org/diff2.pdf) +//! describing it. +//! +//! The implementation of this algorithm is based on the implementation by +//! Brandon Williams. +use crate::change::get_sys_timestamp; +use rustc_hash::FxHashMap; +use std::cmp::Ordering; +use std::collections::BinaryHeap; +use std::iter::zip; +use std::ops::{Index, IndexMut}; + +/// Options for controlling the text update behavior. +/// +/// - `timeout_ms`: Optional timeout in milliseconds for the diff computation +/// - `use_refined_diff`: Whether to use a more refined but slower diff algorithm. Defaults to true. +#[derive(Clone, Debug)] +pub struct UpdateOptions { + pub timeout_ms: Option, + pub use_refined_diff: bool, +} + +impl Default for UpdateOptions { + fn default() -> Self { + Self { + timeout_ms: None, + use_refined_diff: true, + } + } +} + +#[derive(thiserror::Error, Debug, PartialEq)] +pub enum UpdateTimeoutError { + #[error("Timeout")] + Timeout, +} + +/// Utility function to check if a range is empty that works on older rust versions +#[inline(always)] +fn is_empty_range(start: usize, end: usize) -> bool { + start >= end +} + +#[inline(always)] +fn is_not_empty_range(start: usize, end: usize) -> bool { + start < end +} + +fn common_prefix(xs: &[u32], ys: &[u32]) -> usize { + let chunk_size = 4; + let off = zip(xs.chunks_exact(chunk_size), ys.chunks_exact(chunk_size)) + .take_while(|(xs_chunk, ys_chunk)| xs_chunk == ys_chunk) + .count() + * chunk_size; + off + zip(&xs[off..], &ys[off..]) + .take_while(|(x, y)| x == y) + .count() +} + +fn common_suffix_len(old: &[u32], new: &[u32]) -> usize { + let chunk_size = 4; + let old_len = old.len(); + let new_len = new.len(); + + let off = zip(old.rchunks_exact(chunk_size), new.rchunks_exact(chunk_size)) + .take_while(|(old_chunk, new_chunk)| old_chunk == new_chunk) + .count() + * chunk_size; + + off + zip( + old[..old_len - off].iter().rev(), + new[..new_len - off].iter().rev(), + ) + .take_while(|(o, n)| o == n) + .count() +} + +pub(crate) trait DiffHandler { + fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize); + fn delete(&mut self, old_index: usize, old_len: usize); +} + +#[derive(Debug)] +pub(crate) struct OperateProxy { + handler: D, +} + +impl OperateProxy { + pub fn new(handler: D) -> Self { + Self { handler } + } + + pub fn delete(&mut self, old_index: usize, old_len: usize) { + self.handler.delete(old_index, old_len); + } + + pub fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize) { + self.handler.insert(old_index, new_index, new_len); + } + + #[allow(unused)] + fn unwrap(self) -> D { + self.handler + } +} + +pub(crate) fn diff( + proxy: &mut OperateProxy, + options: UpdateOptions, + old: &[u32], + new: &[u32], +) -> Result<(), UpdateTimeoutError> { + let max_d = (old.len() + new.len()).div_ceil(2) + 1; + let mut vb = OffsetVec::new(max_d); + let mut vf = OffsetVec::new(max_d); + let start_time = if options.timeout_ms.is_some() { + get_sys_timestamp() + } else { + 0. + }; + + conquer( + proxy, + options.use_refined_diff, + options.timeout_ms, + start_time, + old, + 0, + old.len(), + new, + 0, + new.len(), + &mut vf, + &mut vb, + ) +} + +struct OffsetVec(isize, Vec); + +impl OffsetVec { + fn new(max_d: usize) -> Self { + Self(max_d as isize, vec![0; max_d << 1]) + } + + fn len(&self) -> usize { + self.1.len() + } +} + +impl Index for OffsetVec { + type Output = usize; + fn index(&self, index: isize) -> &Self::Output { + &self.1[(index + self.0) as usize] + } +} + +impl IndexMut for OffsetVec { + fn index_mut(&mut self, index: isize) -> &mut Self::Output { + &mut self.1[(index + self.0) as usize] + } +} + +struct MiddleSnakeResult { + #[allow(unused)] + d: usize, + x_start: usize, + y_start: usize, +} + +#[allow(clippy::too_many_arguments)] +fn find_middle_snake( + timeout_ms: Option, + start_time: f64, + old: &[u32], + old_start: usize, + old_end: usize, + new: &[u32], + new_start: usize, + new_end: usize, + vf: &mut OffsetVec, + vb: &mut OffsetVec, +) -> Result, UpdateTimeoutError> { + let n = old_end - old_start; + let m = new_end - new_start; + let delta = n as isize - m as isize; + let odd = delta & 1 != 0; + vf[1] = 0; + vb[1] = 0; + let d_max = (n + m).div_ceil(2) + 1; + assert!(vf.len() >= d_max); + assert!(vb.len() >= d_max); + + for d in 0..d_max as isize { + for k in (-d..=d).rev().step_by(2) { + let mut x = if k == -d || (k != d && vf[k - 1] < vf[k + 1]) { + vf[k + 1] + } else { + vf[k - 1] + 1 + }; + let y = (x as isize - k) as usize; + let (x0, y0) = (x, y); + if x < n && y < m { + let advance = + common_prefix(&old[old_start + x..old_end], &new[new_start + y..new_end]); + x += advance; + } + vf[k] = x; + if odd && (k - delta).abs() <= (d - 1) && vf[k] + vb[delta - k] >= n { + return Ok(Some(MiddleSnakeResult { + d: d as usize, + x_start: x0 + old_start, + y_start: y0 + new_start, + })); + } + + if let Some(timeout_ms) = timeout_ms { + if get_sys_timestamp() - start_time > timeout_ms { + return Err(UpdateTimeoutError::Timeout); + } + } + } + + for k in (-d..=d).rev().step_by(2) { + let mut x = if k == -d || (k != d && vb[k - 1] < vb[k + 1]) { + vb[k + 1] + } else { + vb[k - 1] + 1 + }; + let mut y = (x as isize - k) as usize; + if x < n && y < m { + let advance = common_suffix_len( + &old[old_start..old_start + n - x], + &new[new_start..new_start + m - y], + ); + x += advance; + y += advance; + } + vb[k] = x; + if !odd && (k - delta).abs() <= d && vb[k] + vf[delta - k] >= n { + return Ok(Some(MiddleSnakeResult { + d: d as usize, + x_start: n - x + old_start, + y_start: m - y + new_start, + })); + } + + if let Some(timeout_ms) = timeout_ms { + if get_sys_timestamp() - start_time > timeout_ms { + return Err(UpdateTimeoutError::Timeout); + } + } + } + } + + Ok(None) +} + +#[allow(clippy::too_many_arguments)] +fn conquer( + proxy: &mut OperateProxy, + should_use_dj: bool, + timeout_ms: Option, + start_time: f64, + old: &[u32], + mut old_start: usize, + mut old_end: usize, + new: &[u32], + mut new_start: usize, + mut new_end: usize, + vf: &mut OffsetVec, + vb: &mut OffsetVec, +) -> Result<(), UpdateTimeoutError> { + let common_prefix_len = common_prefix(&old[old_start..old_end], &new[new_start..new_end]); + if common_prefix_len > 0 { + old_start += common_prefix_len; + new_start += common_prefix_len; + } + + let common_suffix_len = common_suffix_len(&old[old_start..old_end], &new[new_start..new_end]); + old_end -= common_suffix_len; + new_end -= common_suffix_len; + + if is_not_empty_range(old_start, old_end) || is_not_empty_range(new_start, new_end) { + let len_old = old_end - old_start; + let len_new = new_end - new_start; + if should_use_dj && (len_old.max(1) * len_new.max(1) < 128 * 128) { + let ok = dj_diff( + proxy, + &old[old_start..old_end], + &new[new_start..new_end], + old_start, + new_start, + 10_000, + ); + if ok { + return Ok(()); + } + } + + if is_empty_range(new_start, new_end) { + proxy.delete(old_start, old_end - old_start); + } else if is_empty_range(old_start, old_end) { + proxy.insert(old_start, new_start, new_end - new_start); + } else if let Some(MiddleSnakeResult { + d: _, + x_start, + y_start, + }) = find_middle_snake( + timeout_ms, start_time, old, old_start, old_end, new, new_start, new_end, vf, vb, + )? { + conquer( + proxy, + should_use_dj, + timeout_ms, + start_time, + old, + old_start, + x_start, + new, + new_start, + y_start, + vf, + vb, + )?; + conquer( + proxy, + should_use_dj, + timeout_ms, + start_time, + old, + x_start, + old_end, + new, + y_start, + new_end, + vf, + vb, + )?; + } else { + proxy.delete(old_start, old_end - old_start); + proxy.insert(old_start, new_start, new_end - new_start); + } + } + + Ok(()) +} + +/// Return false if this method gives up early +#[must_use] +pub(crate) fn dj_diff( + proxy: &mut OperateProxy, + old: &[u32], + new: &[u32], + old_offset: usize, + new_offset: usize, + max_try_count: usize, +) -> bool { + let common_prefix_len = common_prefix(old, new); + let common_suffix_len = common_suffix_len(&old[common_prefix_len..], &new[common_prefix_len..]); + let old = &old[common_prefix_len..old.len() - common_suffix_len]; + let new = &new[common_prefix_len..new.len() - common_suffix_len]; + if old.len() >= u16::MAX as usize || new.len() >= u16::MAX as usize { + return false; + } + + if old.is_empty() { + if new.is_empty() { + return true; + } + + proxy.insert( + old_offset + common_prefix_len, + new_offset + common_prefix_len, + new.len(), + ); + return true; + } + + if new.is_empty() { + proxy.delete(old_offset + common_prefix_len, old.len()); + return true; + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + struct Point { + x: u16, + y: u16, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Direction { + Up, + Left, + UpLeft, + } + + struct QueueItem { + point: Point, + cost: u32, + from: Direction, + } + + impl PartialEq for QueueItem { + fn eq(&self, other: &Self) -> bool { + self.cost == other.cost + } + } + + impl PartialOrd for QueueItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for QueueItem { + fn cmp(&self, other: &Self) -> Ordering { + other.cost.cmp(&self.cost) + } + } + + impl Eq for QueueItem {} + + let mut visited: FxHashMap = FxHashMap::default(); + let mut q: BinaryHeap = BinaryHeap::new(); + q.push(QueueItem { + point: Point { x: 0, y: 0 }, + cost: 0, + from: Direction::UpLeft, + }); + + while let Some(QueueItem { point, cost, from }) = q.pop() { + if visited.contains_key(&point) { + continue; + } + + visited.insert(point, from); + if point.x == old.len() as u16 && point.y == new.len() as u16 { + break; + } + + if visited.len() + q.len() > max_try_count { + // println!("give up on: visited len: {}", visited.len()); + // println!("queue len: {}", q.len()); + return false; + } + + if point.x < old.len() as u16 { + let next_point = Point { + x: point.x + 1, + y: point.y, + }; + + if !visited.contains_key(&next_point) { + let mut next_cost = cost + 1; + if from != Direction::Left { + next_cost += 8; + } + + q.push(QueueItem { + point: next_point, + cost: next_cost, + from: Direction::Left, + }); + } + } + + if point.y < new.len() as u16 { + let next_point = Point { + x: point.x, + y: point.y + 1, + }; + + if !visited.contains_key(&next_point) { + let direction = Direction::Up; + let mut next_cost = cost + 1; + if from != direction { + next_cost += 8; + } + + q.push(QueueItem { + point: next_point, + cost: next_cost, + from: Direction::Up, + }); + } + } + + if point.x < old.len() as u16 + && point.y < new.len() as u16 + && old[point.x as usize] == new[point.y as usize] + { + let next_point = Point { + x: point.x + 1, + y: point.y + 1, + }; + + if !visited.contains_key(&next_point) { + let next_cost = if from == Direction::UpLeft { + cost + } else { + cost + 1 + }; + + q.push(QueueItem { + point: next_point, + cost: next_cost, + from: Direction::UpLeft, + }); + } + } + } + + // println!("visited len: {}", visited.len()); + // println!("queue len: {}", q.len()); + + // Backtrack from end point to construct diff operations + let mut current = Point { + x: old.len() as u16, + y: new.len() as u16, + }; + + let mut path: Vec<(Direction, usize)> = Vec::new(); + while current.x > 0 || current.y > 0 { + let direction = visited.get(¤t).unwrap(); + if let Some((last_dir, count)) = path.last_mut() { + if last_dir == direction { + *count += 1; + } else { + path.push((*direction, 1)); + } + } else { + path.push((*direction, 1)); + } + match direction { + Direction::Left => { + current.x -= 1; + } + Direction::Up => { + current.y -= 1; + } + Direction::UpLeft => { + current.x -= 1; + current.y -= 1; + } + } + } + + path.reverse(); + + let mut old_index = common_prefix_len; + let mut new_index = common_prefix_len; + + for (direction, count) in path { + match direction { + Direction::Left => { + proxy.delete(old_offset + old_index, count); + old_index += count; + } + Direction::Up => { + proxy.insert(old_offset + old_index, new_index + new_offset, count); + new_index += count; + } + Direction::UpLeft => { + old_index += count; + new_index += count; + } + } + } + + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, Default)] + struct RecordingDiffHandler { + ops: Vec, + } + + #[derive(Debug, PartialEq)] + enum DiffOperation { + Insert { + old_index: usize, + new_index: usize, + length: usize, + }, + Delete { + old_index: usize, + length: usize, + }, + } + + impl DiffHandler for RecordingDiffHandler { + fn insert(&mut self, pos: usize, start: usize, len: usize) { + self.ops.push(DiffOperation::Insert { + old_index: pos, + new_index: start, + length: len, + }); + } + + fn delete(&mut self, pos: usize, len: usize) { + self.ops.push(DiffOperation::Delete { + old_index: pos, + length: len, + }); + } + } + + #[test] + fn test_recording_diff_handler() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3]; + let new = vec![1, 4, 3]; + + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Delete { + old_index: 1, + length: 1 + }, + DiffOperation::Insert { + old_index: 2, + new_index: 1, + length: 1 + }, + ] + ); + } + + #[test] + fn test_dj_diff_same() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3]; + let new = vec![1, 2, 3]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!(handler.ops, vec![]); + } + + #[test] + fn test_dj_diff_1() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1]; + let new = vec![0, 1, 2]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 1 + }, + DiffOperation::Insert { + old_index: 1, + new_index: 2, + length: 1 + }, + ] + ); + } + + #[test] + fn test_diff_may_scatter() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 3, 4, 5, 98, 97, 96, 3, 95, 4, 93, 92, 5, 91]; + diff(&mut proxy, UpdateOptions::default(), &old, &new).unwrap(); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 1 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 6, + length: 10 + }, + ] + ); + } + + #[test] + fn test_diff_may_scatter_1() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 98, 97, 96, 3, 95, 4, 93, 92, 5, 1, 2, 3, 4, 5, 91]; + diff(&mut proxy, UpdateOptions::default(), &old, &new).unwrap(); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 12 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 17, + length: 1 + }, + ] + ); + } + + #[test] + fn test_dj_diff_may_scatter() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 3, 4, 5, 98, 97, 96, 3, 95, 4, 93, 92, 5, 91]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 1 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 6, + length: 10 + }, + ] + ); + } + + #[test] + fn test_dj_diff_may_scatter_1() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1, 2, 3, 4, 5]; + let new = vec![99, 1, 2, 98, 97, 96, 3, 95, 4, 93, 92, 5, 1, 2, 3, 4, 5, 91]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 12 + }, + DiffOperation::Insert { + old_index: 5, + new_index: 17, + length: 1 + }, + ] + ); + } + + #[test] + fn test_dj_diff_100_differences() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 100]; + let new = vec![2; 100]; + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Delete { + old_index: 0, + length: 100 + }, + DiffOperation::Insert { + old_index: 100, + new_index: 0, + length: 100 + }, + ] + ); + } + + #[test] + fn test_dj_diff_insert() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 100]; + let mut new = old.clone(); + new.splice(50..50, [1, 2, 3, 4, 1, 2, 3, 4]); + new.splice(0..0, [0, 1, 2, 3, 4, 5, 6, 7]); + let _ = dj_diff(&mut proxy, &old, &new, 0, 0, 100_000); + let handler = proxy.unwrap(); + assert_eq!( + handler.ops, + vec![ + DiffOperation::Insert { + old_index: 0, + new_index: 0, + length: 9 + }, + DiffOperation::Insert { + old_index: 50, + new_index: 59, + length: 7 + } + ] + ); + } + + #[test] + fn test_diff() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 100]; + let new = vec![2; 100]; + diff(&mut proxy, Default::default(), &old, &new).unwrap(); + } + + #[test] + fn test_timeout() { + let handler = RecordingDiffHandler::default(); + let mut proxy = OperateProxy::new(handler); + let old = vec![1; 10000]; + let new = vec![2; 10000]; + let options = UpdateOptions { + timeout_ms: Some(0.1), + ..Default::default() + }; + let result = diff(&mut proxy, options, &old, &new); + assert!(result.is_err()); + } +} diff --git a/vendor/loro-internal/src/diff/mod.rs b/vendor/loro-internal/src/diff/mod.rs new file mode 100644 index 00000000..5d9d5cb7 --- /dev/null +++ b/vendor/loro-internal/src/diff/mod.rs @@ -0,0 +1,4 @@ +pub mod diff_impl; +pub(crate) use diff_impl::diff; +pub(crate) use diff_impl::DiffHandler; +pub(crate) use diff_impl::OperateProxy; diff --git a/vendor/loro-internal/src/diff_calc.rs b/vendor/loro-internal/src/diff_calc.rs new file mode 100644 index 00000000..e589a0b6 --- /dev/null +++ b/vendor/loro-internal/src/diff_calc.rs @@ -0,0 +1,2126 @@ +use std::{num::NonZeroU16, sync::Arc}; + +#[cfg(feature = "counter")] +mod counter; +#[cfg(feature = "counter")] +pub(crate) use counter::CounterDiffCalculator; +pub(super) mod tree; +mod unknown; +use either::Either; +use generic_btree::rle::HasLength as _; +use itertools::Itertools; + +use enum_dispatch::enum_dispatch; +use loro_common::{ + CompactIdLp, ContainerID, Counter, HasCounterSpan, IdFull, IdLp, IdSpan, LoroValue, PeerID, ID, +}; +use loro_delta::DeltaRope; +use rustc_hash::{FxHashMap, FxHashSet}; +use smallvec::SmallVec; +use tracing::{info_span, instrument}; + +use crate::{ + change::Lamport, + container::{ + idx::ContainerIdx, + list::list_op::InnerListOp, + richtext::{ + richtext_state::{unicode_to_utf8_index, RichtextStateChunk, TextChunk}, + AnchorType, CrdtRopeDelta, RichtextChunk, RichtextChunkValue, RichtextTracker, StyleOp, + }, + }, + cursor::AbsolutePosition, + delta::{ + Delta, DeltaItem, DeltaValue, ElementDelta, MapDelta, MapValue, MovableListInnerDelta, + }, + event::{DiffVariant, InternalDiff}, + op::{InnerContent, RichOp, SliceRange, SliceWithId}, + span::{HasId, HasLamport}, + version::Frontiers, + InternalString, VersionVector, +}; + +use self::tree::TreeDiffCalculator; + +use self::unknown::UnknownDiffCalculator; + +use super::{event::InternalContainerDiff, oplog::OpLog}; + +/// Calculate the diff between two versions. given [OpLog][super::oplog::OpLog] +/// and [AppState][super::state::AppState]. +/// +/// TODO: persist diffCalculator and skip processed version +#[derive(Debug)] +pub struct DiffCalculator { + /// ContainerIdx -> (depth, calculator) + /// + /// if depth is None, we need to calculate it again + calculators: FxHashMap, ContainerDiffCalculator)>, + retain_mode: DiffCalculatorRetainMode, +} + +#[derive(Debug)] +enum DiffCalculatorRetainMode { + /// The diff calculator can only be used once. + Once { used: bool }, + /// The diff calculator will be persisted and can be reused after the diff calc is done. + Persist, +} + +/// This mode defines how the diff is calculated and how it should be applied on the state. +#[derive(Debug, Clone, PartialEq, Eq, Copy)] +pub(crate) enum DiffMode { + /// This is the most general mode of diff calculation. + /// + /// When applying `Checkout` diff, we already know the current state of the affected registers. + /// So there is no need to compare the lamport values. + /// + /// It can be used whenever a user want to switch to a different version. + /// But it is also the slowest mode. It relies on the `ContainerHistoryCache`, which is expensive to build and maintain in memory. + Checkout, + /// This mode is used when the user imports new updates. + /// + /// When applying `Import` diff, we may need to know the the current state. + /// For example, we may need to compare the current register's lamport with the update's lamport to decide + /// what's the new value. + /// + /// It has stricter requirements than `Checkout`: + /// + /// - The target version vector must be greater than the current version vector. + Import, + /// This mode is used when the user imports new updates and all the updates are guaranteed to greater than the current version. + /// + /// It has stricter requirements than `Import`. + /// - All the updates are greater than the current version. No update is concurrent to the current version. + /// - So LCA is always the `from` version + ImportGreaterUpdates, + /// This mode is used when we don't need to build CRDTs to calculate the difference. It is the fastest mode. + /// + /// It has stricter requirements than `ImportGreaterUpdates`. + /// - In `ImportGreaterUpdates`, all the updates are guaranteed to be greater than the current version. + /// - In `Linear`, all the updates are ordered, no concurrent update exists. + Linear, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct DiffCalcVersionInfo<'a> { + from_vv: &'a VersionVector, + to_vv: &'a VersionVector, + from_frontiers: &'a Frontiers, + to_frontiers: &'a Frontiers, + /// §flowstate vendor patch (oom-leads #4): the diff mode INFERRED from the + /// version transition, BEFORE the retained-calculator `Persist` override + /// forces `Checkout`. The override is an apply-law decision; per-container + /// calculators may still pick the cheap forward-content path when the + /// transition itself is provably forward (`Linear`/`ImportGreaterUpdates`). + origin_mode: DiffMode, +} + +impl DiffCalculator { + /// Create a new diff calculator. + /// + /// If `persist` is true, the diff calculator will be persisted after the diff calc is done. + /// This is useful when we need to cache the diff calculator for future use. But it is slower + /// for importing updates and requires more memory. + pub fn new(persist: bool) -> Self { + Self { + calculators: Default::default(), + retain_mode: if persist { + DiffCalculatorRetainMode::Persist + } else { + DiffCalculatorRetainMode::Once { used: false } + }, + } + } + + #[allow(unused)] + pub(crate) fn get_calc(&self, container: ContainerIdx) -> Option<&ContainerDiffCalculator> { + self.calculators.get(&container).map(|(_, c)| c) + } + + + /// Calculate the diff between two versions. + /// + /// Return the diff and the origin diff mode (it's not the diff mode used by the diff calculator. + /// It's the expected diff mode inferred from the two version, which can reflect the direction of the + /// change). + /// + /// §act-eleven A11.5 (flowstate vendor patch): `caller` is a static tag + /// naming the CALL SITE, printed by the `FLOWSTATE_DIFF_SPAN_PROBE` line so + /// duplicate same-calculator spans can be attributed to the code path that + /// requested them. + pub(crate) fn calc_diff_internal_tagged( + &mut self, + oplog: &super::oplog::OpLog, + before: &crate::VersionVector, + before_frontiers: &Frontiers, + after: &crate::VersionVector, + after_frontiers: &Frontiers, + container_filter: Option<&dyn Fn(ContainerIdx) -> bool>, + caller: &'static str, + ) -> (Vec, DiffMode) { + if before == after { + return (Vec::new(), DiffMode::Linear); + } + + let s = tracing::span!(tracing::Level::INFO, "DiffCalc", ?before, ?after,); + let _e = s.enter(); + + let mut merged = before.clone(); + merged.merge(after); + // §patch #24b: the meet decides both the clamp (below) and the main + // walk's enumeration FLOOR — computed before the iterator so the + // dominator-to-meet prefix is never even enumerated (it was ~64.5M + // op-units of pure skip-checking per dirty-history checkout). The + // meet-clamp kill switch disables the floor too. + static MEET_CLAMP_DISABLED_EARLY: std::sync::OnceLock = std::sync::OnceLock::new(); + let meet_clamp_early = matches!(self.retain_mode, DiffCalculatorRetainMode::Persist) + && !*MEET_CLAMP_DISABLED_EARLY + .get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_MEET_CLAMP").is_some()); + let floor_vv = if meet_clamp_early { + let m = before.intersection(after); + if m.is_empty() { None } else { Some(m) } + } else { + None + }; + let (lca, origin_diff_mode, iter) = oplog.iter_from_lca_causally_floored( + before, + before_frontiers, + after, + after_frontiers, + None, + floor_vv.as_ref(), + ); + let mut diff_mode = origin_diff_mode; + match &mut self.retain_mode { + DiffCalculatorRetainMode::Once { used } => { + if *used { + panic!("DiffCalculator with retain_mode Once can only be used once"); + } + } + DiffCalculatorRetainMode::Persist => { + diff_mode = DiffMode::Checkout; + } + } + + // §flowstate vendor patch #21 (meet-clamped diff feed — oom-leads #9 + // P1.C): ops at-or-below `meet = before ∧ after` (pointwise-min + // version vector) are common to BOTH diffed versions — they can never + // appear in the diff, and a calculator only needs them as positional + // CONTEXT, which it already has when its tracker covers the meet + // (`meet_clamp_safe`). The dominator LCA anchoring this walk sits + // arbitrarily deep under persistent concurrency (a peer holding + // unexchanged local ops pins it at the last full exchange), so the + // unclamped walk re-iterated the ENTIRE history on EVERY import + // (measured 64.5M op-units per bare-boundary import — ~0.7-1.3s/round + // of pure skip-checking) and mis-marked thousands of untouched + // containers as affected. Containers whose calculator cannot prove + // the clamp safe (typically a tracker that legitimately rebuilt at + // the dominator) are re-fed unclamped in a second, container-filtered + // pass — exactly the old behavior, but only for those containers. + // `Once` calculators are always fresh (every container would need the + // full pass): they keep the single unclamped walk. + // Kill switch: `FLOWSTATE_DISABLE_MEET_CLAMP=1` restores the original + // single unclamped full-span walk (also the A/B lever for the guards). + static MEET_CLAMP_DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + let meet_clamp = matches!(self.retain_mode, DiffCalculatorRetainMode::Persist) + && !*MEET_CLAMP_DISABLED + .get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_MEET_CLAMP").is_some()); + let meet = if meet_clamp { + before.intersection(after) + } else { + crate::VersionVector::new() + }; + // §patch #21 phase-0 (gap pre-feed): retained trackers that will + // REUSE at this dominator but are missing common ops below the meet + // (they are never fed local commits) get exactly their gap + // `(floor, meet]` BEFORE the main walk — the main walk then feeds + // their ∆ inline instead of deferring the whole ∆ to the full-span + // fallback (which double-walks the DAG: ~2× on big-divergence + // imports). Ordering is sound: every gap op is common to both + // versions, so no gap op can causally descend from a ∆ op; feeding + // gap-then-∆ preserves every dependency. + let mut probe_gap_ops: u64 = 0; + let mut probe_gap_floors: u64 = 0; + if meet_clamp && !meet.is_empty() { + let mut gap_floors: FxHashMap = FxHashMap::default(); + for (idx, (_, calc)) in self.calculators.iter() { + if let Some(floor) = calc.pre_gap_floor(&lca, &meet) { + gap_floors.insert(*idx, floor); + } + } + probe_gap_floors = gap_floors.len() as u64; + if !gap_floors.is_empty() { + let (_l0, _m0, gap_iter) = oplog.iter_from_lca_causally( + before, + before_frontiers, + after, + after_frontiers, + Some(&meet), + ); + for (change, (start_counter, end_counter), vv) in gap_iter { + let peer = change.peer(); + if gap_floors + .values() + .all(|floor| end_counter <= floor.get(&peer).copied().unwrap_or(0)) + { + continue; + } + let iter_start = change + .ops + .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) + .unwrap_or_else(|e| e); + let mut visited = FxHashSet::default(); + for mut op in &change.ops.vec()[iter_start..] { + if op.counter >= end_counter { + break; + } + let Some(floor) = gap_floors.get(&op.container) else { + continue; + }; + if op.ctr_end() <= floor.get(&peer).copied().unwrap_or(0) { + continue; + } + let stack_sliced_op; + if op.ctr_last() < start_counter { + continue; + } + if op.counter < start_counter || op.ctr_end() > end_counter { + stack_sliced_op = Some(op.slice( + (start_counter as usize).saturating_sub(op.counter as usize), + op.atom_len().min((end_counter - op.counter) as usize), + )); + op = stack_sliced_op.as_ref().unwrap(); + } + let vv = &mut vv.borrow_mut(); + vv.extend_to_include_end_id(ID::new(change.peer(), op.counter)); + let container = op.container; + let depth = oplog.arena.get_depth(container); + let (old_depth, calculator) = self.get_or_create_calc(container, depth); + if *old_depth != depth { + *old_depth = depth; + } + probe_gap_ops += op.atom_len() as u64; + if visited.contains(&container) { + calculator.apply_change(oplog, RichOp::new_by_change(&change, op), None); + } else { + calculator.apply_change( + oplog, + RichOp::new_by_change(&change, op), + Some(vv), + ); + visited.insert(container); + } + } + } + // Every gap container is now hole-free up to the meet — + // whether ops were fed or the walk found none to feed. + for idx in gap_floors.keys() { + if let Some((_, calc)) = self.calculators.get_mut(idx) { + calc.note_covered(&meet); + } + } + } + } + let affected_set = { + loro_common::debug!("LCA: {:?} mode={:?}", &lca, diff_mode); + // §act-five P1.0 (flowstate vendor patch): quantify the import diff's + // LCA→after op span. `DiffMode::Checkout` + a LARGE span = a genuine + // concurrent retreat (needs the incremental-checkout fix, P1.C); a + // SMALL span that is still slow = the fixed per-import tracker rebuild + // (needs the retained-calculator fix, P1.B). Gated behind a cached env + // flag; the accumulation is one add/op and the read is one atomic load. + let mut probe_span_ops: u64 = 0; + let mut probe_clamped_ops: u64 = 0; + let mut started_set = FxHashSet::default(); + // Container → the floor ABOVE which it must be re-fed unclamped + // (its tracker's coverage gap; empty floor = full re-feed). + let mut needs_full: FxHashMap = FxHashMap::default(); + for (change, (start_counter, end_counter), vv) in iter { + probe_span_ops += u64::try_from(end_counter - start_counter).unwrap_or(0); + let mut start_counter = start_counter; + if meet_clamp { + let meet_end = meet.get(&change.peer()).copied().unwrap_or(0); + if end_counter <= meet_end { + probe_clamped_ops += u64::try_from(end_counter - start_counter).unwrap_or(0); + continue; + } + if start_counter < meet_end { + probe_clamped_ops += u64::try_from(meet_end - start_counter).unwrap_or(0); + start_counter = meet_end; + } + } + let iter_start = change + .ops + .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) + .unwrap_or_else(|e| e); + let mut visited = FxHashSet::default(); + for mut op in &change.ops.vec()[iter_start..] { + if op.counter >= end_counter { + break; + } + + let idx = op.container; + if let Some(filter) = container_filter { + if !filter(idx) { + continue; + } + } + if needs_full.contains_key(&idx) { + // Fed from its coverage floor in the second pass below. + continue; + } + + // slice the op if needed + // PERF: we can skip the slice by using the RichOp::new_slice + let stack_sliced_op; + if op.ctr_last() < start_counter { + continue; + } + + if op.counter < start_counter || op.ctr_end() > end_counter { + stack_sliced_op = Some(op.slice( + (start_counter as usize).saturating_sub(op.counter as usize), + op.atom_len().min((end_counter - op.counter) as usize), + )); + op = stack_sliced_op.as_ref().unwrap(); + } + + let vv = &mut vv.borrow_mut(); + vv.extend_to_include_end_id(ID::new(change.peer(), op.counter)); + let container = op.container; + let depth = oplog.arena.get_depth(container); + let (old_depth, calculator) = self.get_or_create_calc(container, depth); + // checkout use the same diff_calculator, the depth of calculator is not updated + // That may cause the container to be considered deleted + if *old_depth != depth { + *old_depth = depth; + } + + if !started_set.contains(&op.container) { + started_set.insert(container); + calculator.start_tracking(oplog, &lca, diff_mode); + if meet_clamp { + if let Some(floor) = calculator.meet_clamp_floor(&meet) { + needs_full.insert(container, floor); + continue; + } + } + } + + if visited.contains(&op.container) { + // don't checkout if we have already checked out this container in this round + calculator.apply_change(oplog, RichOp::new_by_change(&change, op), None); + } else { + calculator.apply_change( + oplog, + RichOp::new_by_change(&change, op), + Some(vv), + ); + visited.insert(container); + } + } + } + + // §patch #21 second pass: the containers that could not take the + // clamped feed are re-fed from their OWN coverage floors — the + // retained import calculator's gap is "local ops since the last + // import", not the whole span from the dominator; only a genuinely + // rebuilt tracker (empty floor) pays the full walk. Whole changes + // below EVERY floor are skipped without touching their ops. + // `start_tracking` already ran in the first pass; this pass only + // feeds. + let mut probe_phase2_changes: u64 = 0; + let mut probe_phase2_ops: u64 = 0; + if !needs_full.is_empty() { + let (_lca2, _mode2, full_iter) = + oplog.iter_from_lca_causally(before, before_frontiers, after, after_frontiers, None); + for (change, (start_counter, end_counter), vv) in full_iter { + // A change at-or-below every needed floor has nothing to + // feed — skip it without touching its ops. + let peer = change.peer(); + if needs_full + .values() + .all(|floor| end_counter <= floor.get(&peer).copied().unwrap_or(0)) + { + continue; + } + probe_phase2_changes += 1; + let iter_start = change + .ops + .binary_search_by(|op| op.ctr_last().cmp(&start_counter)) + .unwrap_or_else(|e| e); + let mut visited = FxHashSet::default(); + for mut op in &change.ops.vec()[iter_start..] { + if op.counter >= end_counter { + break; + } + let Some(floor) = needs_full.get(&op.container) else { + continue; + }; + // Fully below this container's floor ⇒ already in its + // tracker; partial overlaps feed whole (the tracker's + // skip-applied logic handles the known prefix). + if op.ctr_end() <= floor.get(&peer).copied().unwrap_or(0) { + continue; + } + + let stack_sliced_op; + if op.ctr_last() < start_counter { + continue; + } + if op.counter < start_counter || op.ctr_end() > end_counter { + stack_sliced_op = Some(op.slice( + (start_counter as usize).saturating_sub(op.counter as usize), + op.atom_len().min((end_counter - op.counter) as usize), + )); + op = stack_sliced_op.as_ref().unwrap(); + } + + let vv = &mut vv.borrow_mut(); + vv.extend_to_include_end_id(ID::new(change.peer(), op.counter)); + let container = op.container; + let depth = oplog.arena.get_depth(container); + let (old_depth, calculator) = self.get_or_create_calc(container, depth); + if *old_depth != depth { + *old_depth = depth; + } + probe_phase2_ops += op.atom_len() as u64; + if visited.contains(&container) { + calculator.apply_change(oplog, RichOp::new_by_change(&change, op), None); + } else { + calculator.apply_change( + oplog, + RichOp::new_by_change(&change, op), + Some(vv), + ); + visited.insert(container); + } + } + } + } + + static P1_DIFF_SPAN_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + if *P1_DIFF_SPAN_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_DIFF_SPAN_PROBE").is_some()) { + // §act-ten A9.1: `calc` is this DiffCalculator's identity — the + // fuzz runs many peers in ONE process, so identical back-to-back + // spans from DIFFERENT calc pointers are cross-peer mirrors of + // the same divergence (each peer legitimately reconciles once), + // while repeats from the SAME pointer are genuine recomputation + // (the dedup/memoization candidate). + // Compact (before, after) identity so REPEATED lines can be told + // apart from equal-sized mirrors (a checkout there-and-back has + // identical op counts but swapped endpoints). + let endpoint_hash = |frontiers: &Frontiers| -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::hash::DefaultHasher::new(); + for id in frontiers.iter() { + id.hash(&mut hasher); + } + hasher.finish() & 0xffff + }; + eprintln!( + "[flowstate-diff-span] calc={:p} caller={caller} ops={probe_span_ops} clamped={probe_clamped_ops} gap_floors={probe_gap_floors} gap_ops={probe_gap_ops} needs_full={} p2_changes={probe_phase2_changes} p2_ops={probe_phase2_ops} containers={} mode={diff_mode:?} before={:04x} after={:04x}", + self as *const _, + needs_full.len(), + started_set.len(), + endpoint_hash(before_frontiers), + endpoint_hash(after_frontiers), + ); + } + Some(started_set) + }; + + // Because we need to get correct `bring_back` value that indicates container is created during this round of diff calc, + // we need to iterate from parents to children. i.e. from smaller depth to larger depth. + let mut new_containers = FxHashSet::default(); + let mut container_id_to_depth = FxHashMap::default(); + let mut all: Vec<(Option, ContainerIdx)> = if let Some(set) = affected_set { + // only visit the affected containers + set.into_iter() + .map(|x| { + let (depth, _) = self.calculators.get_mut(&x).unwrap(); + (*depth, x) + }) + .collect() + } else { + self.calculators + .iter_mut() + .map(|(x, (depth, _))| (*depth, *x)) + .collect() + }; + let mut ans = FxHashMap::default(); + let info = DiffCalcVersionInfo { + from_vv: before, + to_vv: after, + from_frontiers: before_frontiers, + to_frontiers: after_frontiers, + origin_mode: origin_diff_mode, + }; + while !all.is_empty() { + // sort by depth and lamport, ensure we iterate from top to bottom + all.sort_by_key(|x| x.0); + for (_, container_idx) in std::mem::take(&mut all) { + if ans.contains_key(&container_idx) { + continue; + } + let (depth, calc) = self.calculators.get_mut(&container_idx).unwrap(); + // §patch #21: this round fed (or verified coverage of) every + // op of this container up to the merged version. + calc.note_covered(&merged); + if depth.is_none() { + let d = oplog.arena.get_depth(container_idx); + if d != *depth { + *depth = d; + all.push((*depth, container_idx)); + continue; + } + } + let id = oplog.arena.idx_to_id(container_idx).unwrap(); + let bring_back = new_containers.remove(&id); + + info_span!("CalcDiff", ?id).in_scope(|| { + let (diff, diff_mode) = calc.calculate_diff(container_idx, oplog, info, |c| { + new_containers.insert(c.clone()); + container_id_to_depth + .insert(c.clone(), depth.and_then(|d| d.checked_add(1))); + oplog.arena.register_container(c); + }); + calc.finish_this_round(); + if !diff.is_empty() || bring_back { + ans.insert( + container_idx, + ( + *depth, + InternalContainerDiff { + idx: container_idx, + bring_back, + diff: diff.into(), + diff_mode, + }, + ), + ); + } + }); + } + } + + while !new_containers.is_empty() { + for id in std::mem::take(&mut new_containers) { + // Registration can be lazy; ensure it is registered so we can proceed + let idx = oplog.arena.register_container(&id); + if ans.contains_key(&idx) { + continue; + } + let depth = container_id_to_depth.remove(&id).unwrap(); + ans.insert( + idx, + ( + depth, + InternalContainerDiff { + idx, + bring_back: true, + diff: DiffVariant::None, + diff_mode: DiffMode::Checkout, + }, + ), + ); + } + } + + ( + ans.into_values().map(|x| x.1).collect_vec(), + origin_diff_mode, + ) + } + + // TODO: we may remove depth info + pub(crate) fn get_or_create_calc( + &mut self, + idx: ContainerIdx, + depth: Option, + ) -> &mut (Option, ContainerDiffCalculator) { + self.calculators + .entry(idx) + .or_insert_with(|| match idx.get_type() { + crate::ContainerType::Text => ( + depth, + ContainerDiffCalculator::Richtext(RichtextDiffCalculator::new()), + ), + crate::ContainerType::Map => ( + depth, + ContainerDiffCalculator::Map(MapDiffCalculator::new(idx)), + ), + crate::ContainerType::List => ( + depth, + ContainerDiffCalculator::List(ListDiffCalculator::default()), + ), + crate::ContainerType::Tree => ( + depth, + ContainerDiffCalculator::Tree(TreeDiffCalculator::new(idx)), + ), + crate::ContainerType::Unknown(_) => ( + depth, + ContainerDiffCalculator::Unknown(unknown::UnknownDiffCalculator), + ), + crate::ContainerType::MovableList => ( + depth, + ContainerDiffCalculator::MovableList(MovableListDiffCalculator::new(idx)), + ), + #[cfg(feature = "counter")] + crate::ContainerType::Counter => ( + depth, + ContainerDiffCalculator::Counter(CounterDiffCalculator::new(idx)), + ), + }) + } +} + +/// DiffCalculator should track the history first before it can calculate the difference. +/// +/// So we need it to first apply all the ops between the two versions. +/// +/// NOTE: not every op between two versions are included in a certain container. +/// So there may be some ops that cannot be seen by the container. +/// +#[enum_dispatch] +pub(crate) trait DiffCalculatorTrait { + fn start_tracking(&mut self, oplog: &OpLog, vv: &crate::VersionVector, mode: DiffMode); + /// §flowstate vendor patch #21 (meet-clamped diff feed): how much of the + /// common prefix (ops at-or-below `meet = before ∧ after`) this calculator + /// can safely SKIP being fed. Common ops can never appear IN the diff, + /// but a positional tracker still needs them PRESENT (holes corrupt + /// retain offsets). + /// + /// - `None`: fully clamp — feed only ops above the meet. Correct for + /// calculators with no positional tracker (map/unknown/counter — their + /// content strategies only need the ∆ ops) and for trackers that + /// provably contain everything up to the meet (`all_vv ⊇ meet`). + /// - `Some(floor)`: feed this container every op above `floor` (⊆ meet). + /// A retained tracker returns its coverage floor (`all_vv ∧ meet`) so + /// only its GAP is re-walked — e.g. the retained import calculator is + /// never fed local commits, so its gap is exactly "local ops since the + /// last import", not the whole history from the dominator. A REBUILT + /// tracker (empty `all_vv`) returns an empty floor = the full walk. + /// + /// Called AFTER `start_tracking` for the round, so the answer reflects + /// the tracker state the round will actually use. + fn meet_clamp_floor(&self, _meet: &crate::VersionVector) -> Option { + None + } + /// §flowstate vendor patch #21 (gap pre-feed): BEFORE the round's walk, a + /// retained tracker that (a) will REUSE at this round's `lca` and (b) is + /// missing common ops below the meet (it is never fed local commits) + /// returns its coverage floor — the caller then feeds it exactly + /// `(floor, meet]` up front, so the main walk can feed its ∆ inline + /// instead of deferring the whole ∆ to the full-span fallback (which + /// double-walks the DAG). `None` = no pre-feed needed or possible. + fn pre_gap_floor( + &self, + _lca: &crate::VersionVector, + _meet: &crate::VersionVector, + ) -> Option { + None + } + /// §flowstate vendor patch #21: advance the coverage watermark — the + /// caller guarantees every op of this container at-or-below `vv` has now + /// been fed (or was already covered). No-op for calculators without a + /// positional tracker. + fn note_covered(&mut self, _vv: &crate::VersionVector) {} + fn apply_change( + &mut self, + oplog: &OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ); + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode); + /// This round of diff calc is finished, we can clear the cache + fn finish_this_round(&mut self); +} + +#[enum_dispatch(DiffCalculatorTrait)] +#[derive(Debug)] +pub(crate) enum ContainerDiffCalculator { + Map(MapDiffCalculator), + List(ListDiffCalculator), + Richtext(RichtextDiffCalculator), + Tree(TreeDiffCalculator), + MovableList(MovableListDiffCalculator), + #[cfg(feature = "counter")] + Counter(counter::CounterDiffCalculator), + Unknown(UnknownDiffCalculator), +} + +#[derive(Debug)] +pub(crate) struct MapDiffCalculator { + container_idx: ContainerIdx, + changed: FxHashMap>, + current_mode: DiffMode, +} + +impl MapDiffCalculator { + pub(crate) fn new(container_idx: ContainerIdx) -> Self { + Self { + container_idx, + changed: Default::default(), + current_mode: DiffMode::Checkout, + } + } +} + +impl DiffCalculatorTrait for MapDiffCalculator { + fn start_tracking( + &mut self, + _oplog: &crate::OpLog, + _vv: &crate::VersionVector, + mode: DiffMode, + ) { + self.changed.clear(); + self.current_mode = mode; + } + + fn apply_change( + &mut self, + _oplog: &crate::OpLog, + op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + // §flowstate vendor patch (oom-leads #4): record in EVERY mode. The + // retained import calculator (patch #11) forces `Checkout`, which used + // to skip recording and push every registry-map diff onto the history + // cache — an O(all keys the container ever had) walk per import + // (28ms/remote-split on a 5k-paragraph doc, 147× a text import). + // `calculate_diff` only CONSUMES the recording when the ORIGIN mode + // proves the transition forward; retreats still use the history cache + // and ignore this (cleared per round by `start_tracking`). + let map = op.raw_op().content.as_map().unwrap(); + let new_value = MapValue { + value: map.value.clone(), + peer: op.peer, + lamp: op.lamport(), + }; + match self.changed.get(&map.key) { + Some(Some(old_value)) if old_value > &new_value => {} + _ => { + self.changed.insert(map.key.clone(), Some(new_value)); + } + } + } + + fn finish_this_round(&mut self) { + self.changed.clear(); + self.current_mode = DiffMode::Checkout; + } + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + oplog: &super::oplog::OpLog, + DiffCalcVersionInfo { + from_vv, + to_vv, + origin_mode, + .. + }: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + // §flowstate vendor patch (oom-leads #4): pick the CONTENT strategy by + // the ORIGIN mode. For `Linear`/`ImportGreaterUpdates` every span op is + // causally after the whole pre-state, so the span's per-key + // (lamport, peer)-max IS the CRDT winner and overwrite-apply (the + // forced `Checkout` law) is correct. `Import` (concurrent with the + // current version) and true `Checkout` retreats keep the history-cache + // walk. The RETURNED mode stays `self.current_mode` — the apply law is + // unchanged. + let use_recorded = matches!( + origin_mode, + DiffMode::Linear | DiffMode::ImportGreaterUpdates + ) && !matches!(self.current_mode, DiffMode::Linear | DiffMode::ImportGreaterUpdates); + if use_recorded { + let changed = std::mem::take(&mut self.changed); + let mode = self.current_mode; + self.current_mode = DiffMode::Checkout; + return ( + InternalDiff::Map(MapDelta { + updated: changed + .into_iter() + .map(|(key, value)| { + let value = value.map(|v| { + if let Some(LoroValue::Container(c)) = &v.value { + on_new_container(c); + } + v + }); + (key, value) + }) + .collect(), + }), + mode, + ); + } + // §flowstate vendor patch (oom-leads #4): `Import` origin guarantees + // `from ⊆ to`, so a key's winner can only change if the SPAN carries an + // op for it — restrict the from/to winner comparison to the recorded + // span keys via per-key prefix queries. Same computation as the + // whole-container walk, minus every untouched key (registry maps hold + // thousands). True `Checkout` retreats keep the full walk: the + // retreated branch's keys have no span ops. + if matches!(origin_mode, DiffMode::Import) + && matches!(self.current_mode, DiffMode::Checkout | DiffMode::Import) + { + let recorded = std::mem::take(&mut self.changed); + self.current_mode = DiffMode::Checkout; + return oplog.with_history_cache(|h| { + let checkout_index = &h.get_checkout_index().map; + let mut updated = + FxHashMap::with_capacity_and_hasher(recorded.len(), Default::default()); + for (key, _) in recorded { + let peek_from = checkout_index.get_container_key_latest_op_at_vv( + self.container_idx, + &key, + from_vv, + oplog, + ); + let peek_to = checkout_index.get_container_key_latest_op_at_vv( + self.container_idx, + &key, + to_vv, + oplog, + ); + let changed_value = match (&peek_from, &peek_to) { + (Some(a), Some(b)) if a.value == b.value => continue, + (None, None) => continue, + _ => peek_to, + }; + let value = changed_value.map(|v| { + let value = v.value.clone(); + if let Some(LoroValue::Container(c)) = &value { + on_new_container(c); + } + MapValue { + value, + lamp: v.lamport, + peer: v.peer, + } + }); + updated.insert(key, value); + } + (InternalDiff::Map(MapDelta { updated }), DiffMode::Checkout) + }); + } + match self.current_mode { + DiffMode::Checkout | DiffMode::Import => oplog.with_history_cache(|h| { + let checkout_index = &h.get_checkout_index().map; + let mut changed = Vec::new(); + let from_map = checkout_index.get_container_latest_op_at_vv( + self.container_idx, + from_vv, + Lamport::MAX, + oplog, + ); + let mut to_map = checkout_index.get_container_latest_op_at_vv( + self.container_idx, + to_vv, + Lamport::MAX, + oplog, + ); + + for (k, peek_from) in from_map.iter() { + let peek_to = to_map.remove(k); + match peek_to { + None => changed.push((k.clone(), None)), + Some(b) => { + if peek_from.value != b.value { + changed.push((k.clone(), Some(b))) + } + } + } + } + + for (k, peek_to) in to_map.into_iter() { + changed.push((k, Some(peek_to))); + } + + let mut updated = + FxHashMap::with_capacity_and_hasher(changed.len(), Default::default()); + for (key, value) in changed { + let value = value.map(|v| { + let value = v.value.clone(); + if let Some(LoroValue::Container(c)) = &value { + on_new_container(c); + } + + MapValue { + value, + lamp: v.lamport, + peer: v.peer, + } + }); + + updated.insert(key, value); + } + + (InternalDiff::Map(MapDelta { updated }), DiffMode::Checkout) + }), + DiffMode::ImportGreaterUpdates | DiffMode::Linear => { + let changed = std::mem::take(&mut self.changed); + let mode = self.current_mode; + // Reset this field to avoid we use `has_all` to cache the diff calc and use it next round + // (In the next round we need to use the checkout mode) + self.current_mode = DiffMode::Checkout; + (InternalDiff::Map(MapDelta { updated: changed }), mode) + } + } + } +} + +use rle::{HasLength as _, Sliceable}; + +#[derive(Default)] +pub(crate) struct ListDiffCalculator { + start_vv: VersionVector, + tracker: Box, + /// §flowstate vendor patch #21: per-container coverage watermark (see the + /// richtext calculator's `covered_vv`). + covered_vv: VersionVector, +} + +impl ListDiffCalculator { + pub(crate) fn get_id_latest_pos(&self, id: ID) -> Option { + self.tracker.get_target_id_latest_index_at_new_version(id) + } +} + +impl MovableListDiffCalculator { + pub(crate) fn get_id_latest_pos(&self, id: ID) -> Option { + self.list + .tracker + .get_target_id_latest_index_at_new_version(id) + } +} + +impl std::fmt::Debug for ListDiffCalculator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ListDiffCalculator") + // .field("tracker", &self.tracker) + .finish() + } +} + +impl DiffCalculatorTrait for ListDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, vv: &crate::VersionVector, _mode: DiffMode) { + if !vv.includes_vv(&self.start_vv) || !self.tracker.all_vv().includes_vv(vv) { + *self.tracker = RichtextTracker::new_with_unknown(); + self.start_vv = vv.clone(); + // §patch #21: everything at-or-below the rebuild point is frozen + // prefix — covered by construction. + self.covered_vv = vv.clone(); + } + + self.tracker.checkout(vv); + } + + fn meet_clamp_floor(&self, meet: &crate::VersionVector) -> Option { + // §patch #21: positional tracker — fully clamp only when every common + // op up to the meet is provably tracked (no retain-offset holes); + // otherwise re-feed exactly the coverage gap. + if self.covered_vv.includes_vv(meet) { + None + } else { + Some(self.covered_vv.intersection(meet)) + } + } + + fn pre_gap_floor( + &self, + lca: &crate::VersionVector, + meet: &crate::VersionVector, + ) -> Option { + // Pre-feed only when this round will REUSE the tracker (same law as + // `start_tracking`) and common ops below the meet are missing. + let all_vv = self.tracker.all_vv(); + if lca.includes_vv(&self.start_vv) && all_vv.includes_vv(lca) && !self.covered_vv.includes_vv(meet) + { + Some(self.covered_vv.intersection(meet)) + } else { + None + } + } + + fn note_covered(&mut self, vv: &crate::VersionVector) { + self.covered_vv.extend_to_include_vv(vv.iter()); + } + + fn apply_change( + &mut self, + _oplog: &OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ) { + if let Some(vv) = vv { + self.tracker.checkout(vv); + } + + match &op.op().content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { slice, pos } => { + self.tracker.insert( + op.id_full(), + *pos, + RichtextChunk::new_text(slice.0.clone()), + ); + } + InnerListOp::Delete(del) => { + self.tracker.delete( + op.id_start(), + del.id_start, + del.start() as usize, + del.atom_len(), + del.is_reversed(), + ); + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } + + fn finish_this_round(&mut self) {} + + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + let mut delta = Delta::new(); + for item in self.tracker.diff(info.from_vv, info.to_vv) { + match item { + CrdtRopeDelta::Retain(len) => { + delta = delta.retain(len); + } + CrdtRopeDelta::Insert { + chunk: value, + id, + lamport, + } => match value.value() { + RichtextChunkValue::Text(range) => { + for i in range.clone() { + let v = oplog.arena.get_value(i as usize); + if let Some(LoroValue::Container(c)) = &v { + on_new_container(c); + } + } + delta = delta.insert(SliceWithId { + values: Either::Left(SliceRange(range)), + id: IdFull::new(id.peer, id.counter, lamport.unwrap()), + elem_id: None, + }); + } + RichtextChunkValue::StyleAnchor { .. } => unreachable!(), + RichtextChunkValue::Unknown(len) => { + delta = handle_unknown(idx, id, oplog, len, &mut on_new_container, delta); + } + RichtextChunkValue::MoveAnchor => { + delta = handle_unknown(idx, id, oplog, 1, &mut on_new_container, delta); + } + }, + CrdtRopeDelta::Delete(len) => { + delta = delta.delete(len); + } + } + } + + /// Handle span with unknown content when calculating diff + /// + /// We can lookup the content of the span by the id in the oplog + fn handle_unknown( + idx: ContainerIdx, + mut id: ID, + oplog: &OpLog, + len: u32, + on_new_container: &mut dyn FnMut(&ContainerID), + mut delta: Delta, + ) -> Delta { + // assert not unknown id + assert_ne!(id.peer, PeerID::MAX); + let mut acc_len = 0; + let end = id.counter + len as Counter; + let shallow_root = oplog.shallow_since_vv().get(&id.peer).copied().unwrap_or(0); + if id.counter < shallow_root { + // need to find the content between id.counter ~ target_end in gc state + let target_end = shallow_root.min(end); + delta = oplog.with_history_cache(|h| { + let chunks = + h.find_list_chunks_in(idx, IdSpan::new(id.peer, id.counter, target_end)); + for c in chunks { + acc_len += c.length(); + match &c.values { + Either::Left(_) => unreachable!(), + Either::Right(r) => { + if let LoroValue::Container(c) = r { + on_new_container(c) + } + } + } + delta = delta.insert(c); + } + + delta + }); + id.counter = shallow_root; + } + + if id.counter < end { + for rich_op in oplog.iter_ops(IdSpan::new(id.peer, id.counter, end)) { + acc_len += rich_op.content_len(); + let op = rich_op.op(); + let lamport = rich_op.lamport(); + + if let InnerListOp::Insert { slice, pos: _ } = op.content.as_list().unwrap() { + let range = slice.clone(); + for i in slice.0.clone() { + let v = oplog.arena.get_value(i as usize); + if let Some(LoroValue::Container(c)) = &v { + (on_new_container)(c); + } + } + + delta = delta.insert(SliceWithId { + values: Either::Left(range), + id: IdFull::new(id.peer, op.counter, lamport), + elem_id: None, + }); + } else if let InnerListOp::Move { elem_id, .. } = op.content.as_list().unwrap() + { + delta = delta.insert(SliceWithId { + // We do NOT need an actual value range, + // movable list container will only use the id info + values: Either::Right(LoroValue::Null), + id: IdFull::new(id.peer, op.counter, lamport), + elem_id: Some(elem_id.compact()), + }); + } + } + } + + debug_assert_eq!(acc_len, len as usize); + delta + } + + (InternalDiff::ListRaw(delta), DiffMode::Checkout) + } +} + +#[derive(Debug)] +pub(crate) struct RichtextDiffCalculator { + mode: Box, +} + +#[derive(Debug)] +enum RichtextCalcMode { + Crdt { + tracker: Box, + /// (op, end_pos) + styles: Vec<(StyleOp, usize)>, + start_vv: VersionVector, + /// §flowstate vendor patch #21: coverage watermark — every op of THIS + /// container at-or-below `covered_vv` is in the tracker (no holes). + /// The tracker's own `all_vv` cannot serve this purpose: it is seeded + /// with the whole-doc rebuild vv and only ever extended by this + /// container's fed ops, so comparing it against a whole-doc meet + /// (which advances with OTHER containers' ops) is permanently false. + /// Advanced by `note_covered` when a round's feed completes. + covered_vv: VersionVector, + }, + Linear { + diff: DeltaRope, + last_style_start: Option<(Arc, u32)>, + }, +} + +impl RichtextDiffCalculator { + pub fn new() -> Self { + Self { + mode: Box::new(RichtextCalcMode::Crdt { + tracker: Box::new(RichtextTracker::new_with_unknown()), + styles: Vec::new(), + start_vv: VersionVector::new(), + covered_vv: VersionVector::new(), + }), + } + } + + /// This should be called after calc_diff + /// + /// TODO: Refactor, this can be simplified + pub fn get_id_latest_pos(&self, id: ID) -> Option { + match &*self.mode { + RichtextCalcMode::Crdt { tracker, .. } => { + tracker.get_target_id_latest_index_at_new_version(id) + } + RichtextCalcMode::Linear { .. } => unreachable!(), + } + } +} + +impl DiffCalculatorTrait for RichtextDiffCalculator { + fn start_tracking( + &mut self, + _oplog: &super::oplog::OpLog, + vv: &crate::VersionVector, + mode: DiffMode, + ) { + match mode { + DiffMode::Linear => { + *self.mode = RichtextCalcMode::Linear { + diff: DeltaRope::new(), + last_style_start: None, + }; + } + _ => { + if !matches!(&*self.mode, RichtextCalcMode::Crdt { .. }) { + unreachable!(); + } + } + } + + match &mut *self.mode { + RichtextCalcMode::Crdt { + tracker, + styles, + start_vv, + covered_vv, + } => { + if !vv.includes_vv(start_vv) || !tracker.all_vv().includes_vv(vv) { + **tracker = RichtextTracker::new_with_unknown(); + styles.clear(); + *start_vv = vv.clone(); + // §patch #21: everything at-or-below the rebuild point is + // frozen prefix — covered by construction. + *covered_vv = vv.clone(); + } + + tracker.checkout(vv); + } + RichtextCalcMode::Linear { .. } => {} + } + } + + fn meet_clamp_floor(&self, meet: &crate::VersionVector) -> Option { + // §patch #21: positional tracker — fully clamp only when every common + // op up to the meet is already tracked (no retain-offset holes); + // otherwise re-feed exactly the coverage gap. Linear mode rebuilds a + // throwaway delta from the full feed: never clamp. + match &*self.mode { + RichtextCalcMode::Crdt { covered_vv, .. } => { + if covered_vv.includes_vv(meet) { + None + } else { + Some(covered_vv.intersection(meet)) + } + } + RichtextCalcMode::Linear { .. } => Some(crate::VersionVector::new()), + } + } + + fn pre_gap_floor( + &self, + lca: &crate::VersionVector, + meet: &crate::VersionVector, + ) -> Option { + // Pre-feed only when this round will REUSE the tracker (same law as + // `start_tracking`) and common ops below the meet are missing. + match &*self.mode { + RichtextCalcMode::Crdt { + tracker, + start_vv, + covered_vv, + .. + } => { + let all_vv = tracker.all_vv(); + if lca.includes_vv(start_vv) && all_vv.includes_vv(lca) && !covered_vv.includes_vv(meet) + { + Some(covered_vv.intersection(meet)) + } else { + None + } + } + RichtextCalcMode::Linear { .. } => None, + } + } + + fn note_covered(&mut self, vv: &crate::VersionVector) { + if let RichtextCalcMode::Crdt { covered_vv, .. } = &mut *self.mode { + covered_vv.extend_to_include_vv(vv.iter()); + } + } + + fn apply_change( + &mut self, + oplog: &super::oplog::OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ) { + match &mut *self.mode { + RichtextCalcMode::Linear { + diff, + last_style_start, + } => match &op.raw_op().content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { .. } + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => { + unreachable!() + } + InnerListOp::InsertText { + slice: _, + unicode_start, + unicode_len: len, + pos, + } => { + let s = oplog.arena.slice_by_unicode( + *unicode_start as usize..(*unicode_start + *len) as usize, + ); + diff.insert_value( + *pos as usize, + RichtextStateChunk::new_text(s, op.id_full()), + (), + ); + } + InnerListOp::Delete(del) => { + diff.delete(del.start() as usize, del.atom_len()); + } + InnerListOp::StyleStart { + start, + end, + key, + info, + value, + } => { + debug_assert!(start < end, "start: {}, end: {}", start, end); + let style_op = Arc::new(StyleOp { + lamport: op.lamport(), + peer: op.peer, + cnt: op.id_start().counter, + key: key.clone(), + value: value.clone(), + info: *info, + }); + + *last_style_start = Some((style_op.clone(), *end)); + diff.insert_value( + *start as usize, + RichtextStateChunk::new_style(style_op, AnchorType::Start), + (), + ); + } + InnerListOp::StyleEnd => { + let (style_op, pos) = match last_style_start.take() { + Some((style_op, pos)) => (style_op, pos), + None => { + let Some(start_op) = oplog.get_op_that_includes(op.id().inc(-1)) + else { + panic!("Unhandled checkout case") + }; + + let InnerListOp::StyleStart { + key, + value, + info, + end, + .. + } = start_op.content.as_list().unwrap() + else { + unreachable!() + }; + let style_op = Arc::new(StyleOp { + lamport: op.lamport() - 1, + peer: op.peer, + cnt: op.id_start().counter - 1, + key: key.clone(), + value: value.clone(), + info: *info, + }); + + (style_op, *end) + } + }; + assert_eq!(style_op.peer, op.peer); + assert_eq!(style_op.cnt, op.id_start().counter - 1); + diff.insert_value( + pos as usize + 1, + RichtextStateChunk::new_style(style_op, AnchorType::End), + (), + ); + } + }, + _ => unreachable!(), + }, + RichtextCalcMode::Crdt { + tracker, + styles, + start_vv: _, + covered_vv: _, + } => { + if let Some(vv) = vv { + tracker.checkout(vv); + } + match &op.raw_op().content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { .. } + | InnerListOp::Move { .. } + | InnerListOp::Set { .. } => { + unreachable!() + } + InnerListOp::InsertText { + slice: _, + unicode_start, + unicode_len: len, + pos, + } => { + tracker.insert( + op.id_full(), + *pos as usize, + RichtextChunk::new_text(*unicode_start..*unicode_start + *len), + ); + } + InnerListOp::Delete(del) => { + tracker.delete( + op.id_start(), + del.id_start, + del.start() as usize, + del.atom_len(), + del.is_reversed(), + ); + } + InnerListOp::StyleStart { + start, + end, + key, + info, + value, + } => { + debug_assert!(start < end, "start: {}, end: {}", start, end); + let style_id = styles.len(); + styles.push(( + StyleOp { + lamport: op.lamport(), + peer: op.peer, + cnt: op.id_start().counter, + key: key.clone(), + value: value.clone(), + info: *info, + }, + *end as usize, + )); + tracker.insert( + op.id_full(), + *start as usize, + RichtextChunk::new_style_anchor(style_id as u32, AnchorType::Start), + ); + } + InnerListOp::StyleEnd => { + let id = op.id(); + if let Some(pos) = styles.iter().rev().position(|(op, _pos)| { + op.peer == id.peer && op.cnt == id.counter - 1 + }) { + let style_id = styles.len() - pos - 1; + let (_start_op, end_pos) = &styles[style_id]; + tracker.insert( + op.id_full(), + // need to shift 1 because we insert the start style anchor before this pos + *end_pos + 1, + RichtextChunk::new_style_anchor( + style_id as u32, + AnchorType::End, + ), + ); + } else { + let Some(start_op) = oplog.get_op_that_includes(op.id().inc(-1)) + else { + // Checkout on richtext that export at a gc version that split + // start style op and end style op apart. Won't fix for now. + // It's such a rare case... + unimplemented!("Unhandled checkout case") + }; + let InnerListOp::StyleStart { + start: _, + end, + key, + value, + info, + } = start_op.content.as_list().unwrap() + else { + unreachable!() + }; + + styles.push(( + StyleOp { + lamport: op.lamport() - 1, + peer: id.peer, + cnt: id.counter - 1, + key: key.clone(), + value: value.clone(), + info: *info, + }, + *end as usize, + )); + let style_id = styles.len() - 1; + tracker.insert( + op.id_full(), + // need to shift 1 because we insert the start style anchor before this pos + *end as usize + 1, + RichtextChunk::new_style_anchor( + style_id as u32, + AnchorType::End, + ), + ); + } + } + }, + _ => unreachable!(), + } + } + } + } + + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + _: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + match &mut *self.mode { + RichtextCalcMode::Linear { diff, .. } => ( + InternalDiff::RichtextRaw(std::mem::take(diff)), + DiffMode::Linear, + ), + RichtextCalcMode::Crdt { + tracker, styles, .. + } => { + // Rolling unicode→byte slice cursors, keyed by raw op id. An + // undo-restore of a large document resolves ~10^5 tiny unknown + // chunks out of a handful of giant non-ASCII `InsertText` ops, + // in ascending order; slicing each chunk through + // `InnerListOp::slice` rescans the op's text from byte 0 per + // chunk (quadratic in the op length). The cursor resumes each + // scan where the previous chunk of the same op ended, making + // the total conversion work linear. + let mut op_slice_cursors: FxHashMap = FxHashMap::default(); + let mut delta = DeltaRope::new(); + for item in tracker.diff(info.from_vv, info.to_vv) { + match item { + CrdtRopeDelta::Retain(len) => { + delta.push_retain(len, ()); + } + CrdtRopeDelta::Insert { + chunk: value, + id, + lamport, + } => match value.value() { + RichtextChunkValue::Text(text) => { + delta.push_insert( + RichtextStateChunk::Text( + // PERF: can be speedup by acquiring lock on arena + TextChunk::new( + oplog.arena.slice_by_unicode( + text.start as usize..text.end as usize, + ), + IdFull::new(id.peer, id.counter, lamport.unwrap()), + ), + ), + (), + ); + } + RichtextChunkValue::StyleAnchor { id, anchor_type } => { + delta.push_insert( + RichtextStateChunk::Style { + style: Arc::new(styles[id as usize].0.clone()), + anchor_type, + }, + (), + ); + } + RichtextChunkValue::Unknown(len) => { + // assert not unknown id + assert_ne!(id.peer, PeerID::MAX); + let mut id = id; + let mut acc_len = 0; + let end = id.counter + len as Counter; + let shallow_root = + oplog.shallow_since_vv().get(&id.peer).copied().unwrap_or(0); + if id.counter < shallow_root { + // need to find the content between id.counter ~ target_end in gc state + let target_end = shallow_root.min(end); + oplog.with_history_cache(|h| { + let chunks = h.find_text_chunks_in( + idx, + IdSpan::new(id.peer, id.counter, target_end), + ); + for c in chunks { + acc_len += c.rle_len(); + delta.push_insert(c, ()); + } + }); + id.counter = shallow_root; + } + + if id.counter < end { + for rich_op in + oplog.iter_ops(IdSpan::new(id.peer, id.counter, end)) + { + acc_len += rich_op.content_len(); + // Slice manually with the rolling cursor instead of + // `rich_op.op()`, whose `InnerListOp::slice` rescans the + // op's text from byte 0 on every non-ASCII slice. + let raw = rich_op.raw_op(); + let lamport = rich_op.lamport(); + let (op_from, op_to) = (rich_op.start(), rich_op.end()); + let content = raw.content.as_list().unwrap(); + match content { + InnerListOp::InsertText { + slice, unicode_len, .. + } => { + let total_unicode = *unicode_len as usize; + let (from_byte, to_byte) = + if slice.len() == total_unicode { + // ASCII fast path. + (op_from, op_to) + } else { + // SAFETY: op text is valid utf8 + let s = unsafe { + std::str::from_utf8_unchecked(slice) + }; + let key = ID::new(id.peer, raw.counter); + let (cursor_unicode, cursor_byte) = + match op_slice_cursors.get(&key) { + Some(&(u, b)) if u <= op_from => { + (u, b) + } + _ => (0, 0), + }; + let from_byte = cursor_byte + + unicode_to_utf8_index( + &s[cursor_byte..], + op_from - cursor_unicode, + ) + .expect("unicode index should be valid"); + let to_byte = from_byte + + unicode_to_utf8_index( + &s[from_byte..], + op_to - op_from, + ) + .expect("unicode index should be valid"); + op_slice_cursors + .insert(key, (op_to, to_byte)); + (from_byte, to_byte) + }; + delta.push_insert( + RichtextStateChunk::Text(TextChunk::new( + slice.slice(from_byte, to_byte), + IdFull::new( + id.peer, + raw.counter + op_from as Counter, + lamport, + ), + )), + (), + ); + } + _ => unreachable!("{:?}", content), + } + } + } + + debug_assert_eq!(acc_len, len as usize); + } + RichtextChunkValue::MoveAnchor => unreachable!(), + }, + CrdtRopeDelta::Delete(len) => { + delta.push_delete(len); + } + } + } + + (InternalDiff::RichtextRaw(delta), DiffMode::Checkout) + } + } + } + + fn finish_this_round(&mut self) { + match &mut *self.mode { + RichtextCalcMode::Crdt { .. } => {} + RichtextCalcMode::Linear { + diff, + last_style_start, + } => { + *diff = DeltaRope::new(); + last_style_start.take(); + } + } + } +} + +#[derive(Debug)] +pub(crate) struct MovableListDiffCalculator { + list: Box, + inner: Box, +} + +#[derive(Debug)] +struct MovableListInner { + changed_elements: FxHashMap, + move_id_to_elem_id: FxHashMap, + current_mode: DiffMode, +} + +impl DiffCalculatorTrait for MovableListDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, vv: &crate::VersionVector, mode: DiffMode) { + if !vv.includes_vv(&self.list.start_vv) || !self.list.tracker.all_vv().includes_vv(vv) { + *self.list.tracker = RichtextTracker::new_with_unknown(); + self.list.start_vv = vv.clone(); + } + + self.list.tracker.checkout(vv); + self.inner.current_mode = mode; + } + + fn meet_clamp_floor(&self, _meet: &crate::VersionVector) -> Option { + // §patch #21: `changed_elements`/`move_id_to_elem_id` accumulate + // per-op element state during the feed that `calculate_diff` consumes; + // tracker coverage alone doesn't prove those complete under a clamp. + // Movable lists are not on flowstate's hot path — never clamp. + Some(crate::VersionVector::new()) + } + + fn apply_change( + &mut self, + oplog: &OpLog, + op: crate::op::RichOp, + vv: Option<&crate::VersionVector>, + ) { + let InnerContent::List(l) = &op.raw_op().content else { + unreachable!() + }; + + // collect the elements that are moved, updated, or inserted + + // If it's checkout mode, we don't need to track the changes + // we only need the element ids + match l { + InnerListOp::Insert { slice, pos: _ } => { + let op_id = op.id_full().idlp(); + for i in 0..slice.atom_len() { + let id = op_id.inc(i as Counter); + let value = oplog.arena.get_value(slice.0.start as usize + i).unwrap(); + + self.inner.changed_elements.insert( + id.compact(), + ElementDelta { + pos: Some(id), + value: value.clone(), + value_updated: true, + value_id: Some(id), + }, + ); + } + } + InnerListOp::Delete(_) => {} + InnerListOp::Move { elem_id, .. } => { + let idlp = IdLp::new(op.peer, op.lamport()); + match self.inner.changed_elements.get_mut(&elem_id.compact()) { + Some(change) => { + if change.pos.is_some() && change.pos.as_ref().unwrap() > &idlp { + } else { + change.pos = Some(idlp); + } + } + None => { + self.inner.changed_elements.insert( + elem_id.compact(), + ElementDelta { + pos: Some(idlp), + value: LoroValue::Null, + value_updated: false, + value_id: None, + }, + ); + } + } + } + InnerListOp::Set { elem_id, value } => { + let idlp = IdLp::new(op.peer, op.lamport()); + match self.inner.changed_elements.get_mut(&elem_id.compact()) { + Some(change) => { + if change.value_id.is_some() && change.value_id.as_ref().unwrap() > &idlp { + } else { + change.value_id = Some(idlp); + change.value = value.clone(); + } + } + None => { + self.inner.changed_elements.insert( + elem_id.compact(), + ElementDelta { + pos: None, + value: value.clone(), + value_updated: true, + value_id: Some(idlp), + }, + ); + } + } + } + + InnerListOp::StyleStart { .. } => unreachable!(), + InnerListOp::StyleEnd => unreachable!(), + InnerListOp::InsertText { .. } => unreachable!(), + } + + let is_checkout = matches!(self.inner.current_mode, DiffMode::Checkout); + + { + // Apply change on the list items + let this = &mut self.list; + if let Some(vv) = vv { + this.tracker.checkout(vv); + } + + let real_op = op.op(); + match &real_op.content { + crate::op::InnerContent::List(l) => match l { + InnerListOp::Insert { slice, pos } => { + this.tracker.insert( + op.id_full(), + *pos, + RichtextChunk::new_text(slice.0.clone()), + ); + } + InnerListOp::Delete(del) => { + this.tracker.delete( + op.id_start(), + del.id_start, + del.start() as usize, + del.atom_len(), + del.is_reversed(), + ); + } + InnerListOp::Move { from, elem_id, to } => { + self.inner.move_id_to_elem_id.insert(op.id(), *elem_id); + if !this.tracker.current_vv().includes_id(op.id()) { + let last_pos = if is_checkout { + // TODO: PERF: this lookup can be optimized + oplog.with_history_cache(|h| { + let list = &h.get_checkout_index().movable_list; + list.last_pos( + *elem_id, + this.tracker.current_vv(), + // TODO: PERF: Provide the lamport of to version + Lamport::MAX, + oplog, + ) + .unwrap() + .id() + }) + } else { + // When it's import or linear mode, we need to use a fake id + // because we want to avoid using the history cache + // + // This ID will not be used. Because it will only be used when + // we switch to an older version. And we know it's for importing and + // to version is always after from version (!is_checkout), so that + // we don't need to checkout to the version before from. + const FAKE_ID: ID = ID { + peer: PeerID::MAX - 2, + counter: 0, + }; + FAKE_ID + }; + this.tracker.move_item( + op.id_full(), + last_pos, + *from as usize, + *to as usize, + ); + } + } + InnerListOp::Set { .. } => { + // don't need to update tracker here + } + InnerListOp::InsertText { .. } + | InnerListOp::StyleStart { .. } + | InnerListOp::StyleEnd => unreachable!(), + }, + _ => unreachable!(), + } + }; + } + + fn finish_this_round(&mut self) { + self.list.finish_this_round(); + } + + #[instrument(skip(self, oplog, on_new_container))] + fn calculate_diff( + &mut self, + idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + let (InternalDiff::ListRaw(list_diff), diff_mode) = + self.list.calculate_diff(idx, oplog, info, |_| {}) + else { + unreachable!() + }; + + assert_eq!(diff_mode, DiffMode::Checkout); + let is_checkout = matches!( + self.inner.current_mode, + DiffMode::Checkout | DiffMode::Import + ); + let mut element_changes: FxHashMap = if is_checkout { + FxHashMap::default() + } else { + std::mem::take(&mut self.inner.changed_elements) + }; + + if is_checkout { + for id in self.inner.changed_elements.keys() { + element_changes.insert(*id, ElementDelta::placeholder()); + } + } + + let list_diff: Delta, ()> = Delta { + vec: list_diff + .iter() + .map(|x| match x { + &DeltaItem::Retain { retain, .. } => DeltaItem::Retain { + retain, + attributes: (), + }, + DeltaItem::Insert { insert, .. } => { + let len = insert.length(); + let id = insert.id; + let mut new_insert = SmallVec::with_capacity(len); + for i in 0..len { + let id = id.inc(i as i32); + let elem_id = + if let Some(e) = self.inner.move_id_to_elem_id.get(&id.id()) { + e.compact() + } else { + insert.elem_id.unwrap_or_else(|| id.idlp().compact()) + }; + if is_checkout { + // add the related element id + element_changes.insert(elem_id, ElementDelta::placeholder()); + } + new_insert.push(id); + } + + DeltaItem::Insert { + insert: new_insert, + attributes: (), + } + } + &DeltaItem::Delete { delete, .. } => DeltaItem::Delete { + delete, + attributes: (), + }, + }) + .collect(), + }; + + if is_checkout { + oplog.with_history_cache(|history_cache| { + let checkout_index = &history_cache.get_checkout_index().movable_list; + element_changes.retain(|id, change| { + let id = id.to_id(); + // It can be None if the target does not exist before the `to` version + // But we don't need to calc from, because the deletion is handled by the diff from list items + + // TODO: PERF: Provide the lamport of to version + let Some(pos) = checkout_index.last_pos(id, info.to_vv, Lamport::MAX, oplog) + else { + return false; + }; + // TODO: PERF: Provide the lamport of to version + let value = checkout_index + .last_value(id, info.to_vv, Lamport::MAX, oplog) + .unwrap(); + // TODO: PERF: Provide the lamport of to version + let old_pos = checkout_index.last_pos(id, info.from_vv, Lamport::MAX, oplog); + // TODO: PERF: Provide the lamport of to version + let old_value = + checkout_index.last_value(id, info.from_vv, Lamport::MAX, oplog); + if old_pos.is_none() && old_value.is_none() { + if let LoroValue::Container(c) = &value.value { + on_new_container(c); + } + *change = ElementDelta { + pos: Some(pos.idlp()), + value: value.value.clone(), + value_id: Some(IdLp::new(value.peer, value.lamport)), + value_updated: true, + }; + } else { + // TODO: PERF: can be filtered based on the list_diff and whether the pos/value are updated + *change = ElementDelta { + pos: Some(pos.idlp()), + value: value.value.clone(), + value_updated: old_value.unwrap().value != value.value, + value_id: Some(IdLp::new(value.peer, value.lamport)), + }; + } + + true + }); + }); + } + + let diff = MovableListInnerDelta { + list: list_diff, + elements: element_changes, + }; + + (InternalDiff::MovableList(diff), self.inner.current_mode) + } +} + +impl MovableListDiffCalculator { + fn new(_container: ContainerIdx) -> MovableListDiffCalculator { + MovableListDiffCalculator { + list: Default::default(), + inner: Box::new(MovableListInner { + changed_elements: Default::default(), + current_mode: DiffMode::Checkout, + move_id_to_elem_id: Default::default(), + }), + } + } +} + +#[test] +fn test_size() { + let text = RichtextDiffCalculator::new(); + let size = std::mem::size_of_val(&text); + assert!(size < 50, "RichtextDiffCalculator size: {}", size); + let list = MovableListDiffCalculator::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::MovableList, + )); + let size = std::mem::size_of_val(&list); + assert!(size < 50, "MovableListDiffCalculator size: {}", size); + let calc = ContainerDiffCalculator::Richtext(text); + let size = std::mem::size_of_val(&calc); + assert!(size < 50, "ContainerDiffCalculator size: {}", size); +} diff --git a/vendor/loro-internal/src/diff_calc/counter.rs b/vendor/loro-internal/src/diff_calc/counter.rs new file mode 100644 index 00000000..bf0663b5 --- /dev/null +++ b/vendor/loro-internal/src/diff_calc/counter.rs @@ -0,0 +1,63 @@ +use std::collections::BTreeMap; + +use loro_common::{ContainerID, ID}; + +use crate::{container::idx::ContainerIdx, event::InternalDiff, OpLog}; + +use super::{DiffCalcVersionInfo, DiffCalculatorTrait, DiffMode}; + +#[derive(Debug)] +pub(crate) struct CounterDiffCalculator { + ops: BTreeMap, +} + +impl CounterDiffCalculator { + pub(crate) fn new(_idx: ContainerIdx) -> Self { + Self { + ops: BTreeMap::new(), + } + } +} + +impl DiffCalculatorTrait for CounterDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, _vv: &crate::VersionVector, _mode: DiffMode) {} + + fn apply_change( + &mut self, + _oplog: &OpLog, + op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + let id = op.id(); + self.ops.insert( + id, + *op.op().content.as_future().unwrap().as_counter().unwrap(), + ); + } + + fn finish_this_round(&mut self) {} + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + _oplog: &OpLog, + info: DiffCalcVersionInfo, + _on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + let mut diff = 0.; + let (b, a) = info.from_vv.diff_iter(info.to_vv); + + for sub in b { + for (_, c) in self.ops.range(sub.norm_id_start()..sub.norm_id_end()) { + diff -= c; + } + } + for sub in a { + for (_, c) in self.ops.range(sub.norm_id_start()..sub.norm_id_end()) { + diff += c; + } + } + + (InternalDiff::Counter(diff), DiffMode::Linear) + } +} diff --git a/vendor/loro-internal/src/diff_calc/tree.rs b/vendor/loro-internal/src/diff_calc/tree.rs new file mode 100644 index 00000000..9fcdae8a --- /dev/null +++ b/vendor/loro-internal/src/diff_calc/tree.rs @@ -0,0 +1,646 @@ +use std::{collections::BTreeSet, sync::Arc}; + +use fractional_index::FractionalIndex; +use itertools::Itertools; +use loro_common::{ContainerID, IdFull, IdLp, Lamport, PeerID, TreeID, ID}; +use rustc_hash::FxHashMap; + +use crate::{ + container::{idx::ContainerIdx, tree::tree_op::TreeOp}, + dag::DagUtils, + delta::{TreeDelta, TreeDeltaItem, TreeInternalDiff}, + event::InternalDiff, + state::TreeParentId, + version::Frontiers, + OpLog, VersionVector, +}; + +use super::{DiffCalcVersionInfo, DiffCalculatorTrait, DiffMode}; + +#[derive(Debug)] +pub(crate) struct TreeDiffCalculator { + container: ContainerIdx, + mode: TreeDiffCalculatorMode, +} + +#[derive(Debug)] +enum TreeDiffCalculatorMode { + Crdt, + Linear(TreeDelta), + ImportGreaterUpdates(TreeDelta), +} + +impl DiffCalculatorTrait for TreeDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, _vv: &crate::VersionVector, mode: DiffMode) { + match mode { + DiffMode::Checkout => { + self.mode = TreeDiffCalculatorMode::Crdt; + } + DiffMode::Import => { + self.mode = TreeDiffCalculatorMode::Crdt; + } + DiffMode::ImportGreaterUpdates => { + self.mode = TreeDiffCalculatorMode::ImportGreaterUpdates(TreeDelta::default()); + } + DiffMode::Linear => { + self.mode = TreeDiffCalculatorMode::Linear(TreeDelta::default()); + } + } + } + + fn meet_clamp_floor(&self, _meet: &crate::VersionVector) -> Option { + // §flowstate vendor patch #21: the IGU/Linear modes accumulate a + // TreeDelta from the fed ops; a clamp would starve it. Trees are not + // on flowstate's hot path — never clamp. + Some(crate::VersionVector::new()) + } + + fn apply_change( + &mut self, + _oplog: &OpLog, + op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + match &mut self.mode { + TreeDiffCalculatorMode::Crdt => {} + TreeDiffCalculatorMode::Linear(ref mut delta) + | TreeDiffCalculatorMode::ImportGreaterUpdates(ref mut delta) => { + let id_full = op.id_full(); + let op = op.op(); + let content = op.content.as_tree().unwrap(); + + let item: TreeDeltaItem = match &**content { + crate::container::tree::tree_op::TreeOp::Create { + target, + parent, + position, + } => TreeDeltaItem { + target: *target, + action: TreeInternalDiff::Create { + parent: (*parent).into(), + position: position.clone(), + }, + last_effective_move_op_id: id_full, + }, + crate::container::tree::tree_op::TreeOp::Move { + target, + parent, + position, + } => TreeDeltaItem { + target: *target, + action: TreeInternalDiff::Move { + parent: (*parent).into(), + position: position.clone(), + }, + last_effective_move_op_id: id_full, + }, + crate::container::tree::tree_op::TreeOp::Delete { target } => TreeDeltaItem { + target: *target, + action: TreeInternalDiff::Delete { + parent: TreeParentId::Deleted, + position: None, + }, + last_effective_move_op_id: id_full, + }, + }; + + delta.diff.push(item); + } + } + } + + fn finish_this_round(&mut self) { + self.mode = TreeDiffCalculatorMode::Crdt; + } + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + oplog: &OpLog, + info: DiffCalcVersionInfo, + mut on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + match &mut self.mode { + TreeDiffCalculatorMode::Crdt => { + let diff = self.diff(oplog, info); + diff.diff.iter().for_each(|d| { + // the metadata could be modified before, so (re)create a node need emit the map container diffs + // `Create` here is because maybe in a diff calc uncreate and then create back + if matches!(d.action, TreeInternalDiff::Create { .. }) { + on_new_container(&d.target.associated_meta_container()) + } + }); + + (InternalDiff::Tree(diff), DiffMode::Checkout) + } + TreeDiffCalculatorMode::Linear(ans) => { + (InternalDiff::Tree(std::mem::take(ans)), DiffMode::Linear) + } + TreeDiffCalculatorMode::ImportGreaterUpdates(ans) => { + let mut ans = std::mem::take(ans); + ans.diff.sort_unstable_by(|a, b| { + a.last_effective_move_op_id + .lamport + .cmp(&b.last_effective_move_op_id.lamport) + .then_with(|| { + a.last_effective_move_op_id + .peer + .cmp(&b.last_effective_move_op_id.peer) + }) + }); + (InternalDiff::Tree(ans), DiffMode::ImportGreaterUpdates) + } + } + } +} + +impl TreeDiffCalculator { + pub(crate) fn new(container: ContainerIdx) -> Self { + Self { + container, + mode: TreeDiffCalculatorMode::Crdt, + } + } + + fn diff(&mut self, oplog: &OpLog, info: DiffCalcVersionInfo) -> TreeDelta { + self.checkout(info.from_vv, info.from_frontiers, oplog); + self.checkout_diff(info, oplog) + } + + fn checkout(&mut self, to: &VersionVector, to_frontiers: &Frontiers, oplog: &OpLog) { + oplog.with_history_cache(|h| { + let mark = h.ensure_importing_caches_exist(); + let tree_ops = h.get_tree(&self.container, mark).unwrap(); + let mut tree_cache = tree_ops.tree().lock(); + let s = format!("checkout current {:?} to {:?}", &tree_cache.current_vv, &to); + let s = tracing::span!(tracing::Level::INFO, "checkout", s = s); + let _e = s.enter(); + if to == &tree_cache.current_vv { + return; + } + let min_lamport = self.get_min_lamport_by_frontiers(to_frontiers, oplog); + // retreat + let mut retreat_ops = vec![]; + for (_target, ops) in tree_cache.tree.iter() { + for op in ops.iter().rev() { + if op.id.lamport < min_lamport { + break; + } + if !to.includes_id(op.id.id()) { + retreat_ops.push(op.clone()); + } + } + } + + for op in retreat_ops { + tree_cache.retreat_op(&op); + } + + // forward and apply + let max_lamport = self.get_max_lamport_by_frontiers(to_frontiers, oplog); + let mut forward_ops = vec![]; + let group = h + .get_importing_cache(&self.container, mark) + .unwrap() + .as_tree() + .unwrap(); + for (idlp, op) in group.ops().range( + IdLp { + lamport: 0, + peer: 0, + }..=IdLp { + lamport: max_lamport, + peer: PeerID::MAX, + }, + ) { + if !tree_cache + .current_vv + .includes_id(ID::new(idlp.peer, op.counter)) + && to.includes_id(ID::new(idlp.peer, op.counter)) + { + forward_ops.push((IdFull::new(idlp.peer, op.counter, idlp.lamport), op)); + } + } + + for (id, op) in forward_ops { + let op = MoveLamportAndID { + id, + op: op.value.clone(), + effected: false, + }; + tree_cache.apply(op); + } + tree_cache.current_vv = to.clone(); + }); + } + + fn checkout_diff(&mut self, info: DiffCalcVersionInfo, oplog: &OpLog) -> TreeDelta { + oplog.with_history_cache(|h| { + let mark = h.ensure_importing_caches_exist(); + let tree_ops = h.get_tree(&self.container, mark).unwrap(); + let mut tree_cache = tree_ops.tree().lock(); + let mut parent_to_children_cache = + TreeParentToChildrenCache::init_from_tree_cache(&tree_cache); + let s = tracing::span!(tracing::Level::INFO, "checkout_diff"); + let _e = s.enter(); + let to_frontiers = info.to_frontiers; + let from_frontiers = info.from_frontiers; + let (common_ancestors, _mode) = + oplog.dag.find_common_ancestor(from_frontiers, to_frontiers); + let lca_vv = oplog.dag.frontiers_to_vv(&common_ancestors).unwrap(); + let lca_frontiers = common_ancestors; + let to_max_lamport = self.get_max_lamport_by_frontiers(to_frontiers, oplog); + let lca_min_lamport = self.get_min_lamport_by_frontiers(&lca_frontiers, oplog); + + // retreat for diff + let mut diffs = vec![]; + + if !(tree_cache.current_vv == lca_vv && &lca_vv == info.from_vv) { + let mut retreat_ops = vec![]; + for (_target, ops) in tree_cache.tree.iter() { + for op in ops.iter().rev() { + if op.id.lamport < lca_min_lamport { + break; + } + if !lca_vv.includes_id(op.id.id()) { + retreat_ops.push(op.clone()); + } + } + } + + for op in retreat_ops.into_iter().sorted().rev() { + tree_cache.retreat_op(&op); + let (old_parent, position, last_effective_move_op_id) = + tree_cache.get_parent_with_id(op.op.target()); + if op.effected { + // we need to know whether old_parent is deleted + let is_parent_deleted = tree_cache.is_parent_deleted(op.op.parent_id()); + let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent); + if op.op.target().id() == op.id.id() { + assert_eq!( + old_parent, + TreeParentId::Unexist, + "old_parent = {:?} instead", + &old_parent + ); + } + parent_to_children_cache.record_change( + op.op.target(), + op.op.parent_id(), + old_parent, + ); + let this_diff = TreeDeltaItem::new( + op.op.target(), + old_parent, + op.op.parent_id(), + last_effective_move_op_id, + is_old_parent_deleted, + is_parent_deleted, + position, + ); + let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. }); + diffs.push(this_diff); + if is_create { + let mut s = vec![op.op.target()]; + while let Some(t) = s.pop() { + let children = tree_cache.get_children_with_id( + TreeParentId::Node(t), + &parent_to_children_cache, + ); + children.iter().for_each(|c| { + diffs.push(TreeDeltaItem { + target: c.0, + action: TreeInternalDiff::Create { + parent: TreeParentId::Node(t), + position: c.1.clone().unwrap(), + }, + last_effective_move_op_id: c.2, + }) + }); + s.extend(children.iter().map(|c| c.0)); + } + } + } + } + } + tree_cache.current_vv = lca_vv; + // forward + let group = h + .get_importing_cache(&self.container, mark) + .unwrap() + .as_tree() + .unwrap(); + for (idlp, op) in group.ops().range( + IdLp { + lamport: lca_min_lamport, + peer: 0, + }..=IdLp { + lamport: to_max_lamport, + peer: PeerID::MAX, + }, + ) { + let id = ID::new(idlp.peer, op.counter); + if !tree_cache.current_vv.includes_id(id) && info.to_vv.includes_id(id) { + let op = MoveLamportAndID { + id: IdFull { + peer: id.peer, + lamport: idlp.lamport, + counter: id.counter, + }, + op: op.value.clone(), + effected: false, + }; + let (old_parent, _position, _id) = + tree_cache.get_parent_with_id(op.op.target()); + let is_parent_deleted = tree_cache.is_parent_deleted(op.op.parent_id()); + let is_old_parent_deleted = tree_cache.is_parent_deleted(old_parent); + let effected = tree_cache.apply(op.clone()); + if effected { + let this_diff = TreeDeltaItem::new( + op.op.target(), + op.op.parent_id(), + old_parent, + op.id_full(), + is_parent_deleted, + is_old_parent_deleted, + op.op.fractional_index(), + ); + parent_to_children_cache.record_change( + op.op.target(), + old_parent, + op.op.parent_id(), + ); + let is_create = matches!(this_diff.action, TreeInternalDiff::Create { .. }); + diffs.push(this_diff); + if is_create { + // TODO: per + let mut s = vec![op.op.target()]; + while let Some(t) = s.pop() { + let children = tree_cache.get_children_with_id( + TreeParentId::Node(t), + &parent_to_children_cache, + ); + children.iter().for_each(|c| { + diffs.push(TreeDeltaItem { + target: c.0, + action: TreeInternalDiff::Create { + parent: TreeParentId::Node(t), + position: c.1.clone().unwrap(), + }, + last_effective_move_op_id: c.2, + }) + }); + s.extend(children.iter().map(|x| x.0)); + } + } + } + } + } + + tree_cache.current_vv = info.to_vv.clone(); + TreeDelta { diff: diffs } + }) + } + + fn get_min_lamport_by_frontiers(&self, frontiers: &Frontiers, oplog: &OpLog) -> Lamport { + frontiers + .iter() + .map(|id| oplog.get_min_lamport_at(id)) + .min() + .unwrap_or(0) + } + + fn get_max_lamport_by_frontiers(&self, frontiers: &Frontiers, oplog: &OpLog) -> Lamport { + frontiers + .iter() + .map(|id| oplog.get_max_lamport_at(id)) + .max() + .unwrap_or(Lamport::MAX) + } +} + +/// All information of an operation for diff calculating of movable tree. +#[derive(Debug, Clone)] +pub struct MoveLamportAndID { + pub(crate) id: IdFull, + pub(crate) op: Arc, + /// Whether this action is applied in the current version. + /// If this action will cause a circular reference, then this action will not be applied. + pub(crate) effected: bool, +} + +impl MoveLamportAndID { + fn id_full(&self) -> IdFull { + self.id + } +} + +impl PartialEq for MoveLamportAndID { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for MoveLamportAndID {} + +impl PartialOrd for MoveLamportAndID { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MoveLamportAndID { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.id + .lamport + .cmp(&other.id.lamport) + .then_with(|| self.id.peer.cmp(&other.id.peer)) + } +} + +#[derive(Clone, Default)] +pub(crate) struct TreeCacheForDiff { + tree: FxHashMap>, + current_vv: VersionVector, +} + +impl std::fmt::Debug for TreeCacheForDiff { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "TreeCacheForDiff {{ tree: ")?; + for (id, ops) in self.tree.iter() { + writeln!(f, " {} -> {:?}", id, ops)?; + } + writeln!(f, " current_vv: {:?}", self.current_vv)?; + Ok(()) + } +} + +impl TreeCacheForDiff { + fn retreat_op(&mut self, op: &MoveLamportAndID) { + self.tree.get_mut(&op.op.target()).unwrap().remove(op); + self.current_vv.set_end(op.id.id()); + } + + fn is_ancestor_of(&self, maybe_ancestor: &TreeID, node_id: &TreeParentId) -> bool { + if !self.tree.contains_key(maybe_ancestor) { + return false; + } + if let TreeParentId::Node(id) = node_id { + if id == maybe_ancestor { + return true; + } + } + match node_id { + TreeParentId::Node(id) => { + let (parent, _, _) = &self.get_parent_with_id(*id); + if parent == node_id { + panic!("is_ancestor_of loop") + } + self.is_ancestor_of(maybe_ancestor, parent) + } + TreeParentId::Deleted | TreeParentId::Root => false, + TreeParentId::Unexist => false, + } + } + + fn apply(&mut self, mut node: MoveLamportAndID) -> bool { + let mut effected = true; + if self.is_ancestor_of(&node.op.target(), &node.op.parent_id()) { + effected = false; + } + node.effected = effected; + self.current_vv.set_last(node.id.id()); + self.tree.entry(node.op.target()).or_default().insert(node); + effected + } + + pub(crate) fn init_tree_with_shallow_root_version(&mut self, nodes: Vec) { + if nodes.is_empty() { + return; + } + + debug_assert!(self.tree.is_empty()); + for node in nodes.into_iter() { + self.current_vv.extend_to_include_last_id(node.id.id()); + self.current_vv + .extend_to_include_last_id(node.op.target().id()); + self.tree.entry(node.op.target()).or_default().insert(node); + } + } + + fn is_parent_deleted(&self, parent: TreeParentId) -> bool { + match parent { + TreeParentId::Deleted => true, + TreeParentId::Node(id) => self.is_parent_deleted(self.get_parent_with_id(id).0), + TreeParentId::Root => false, + TreeParentId::Unexist => false, + } + } + + /// get the parent of the first effected op and its id + fn get_parent_with_id( + &self, + tree_id: TreeID, + ) -> (TreeParentId, Option, IdFull) { + let mut ans = (TreeParentId::Unexist, None, IdFull::NONE_ID); + if let Some(cache) = self.tree.get(&tree_id) { + for op in cache.iter().rev() { + if op.effected { + ans = ( + op.op.parent_id(), + op.op.fractional_index().clone(), + op.id_full(), + ); + break; + } + } + } + ans + } + + /// get the parent of the last effected op + fn get_last_effective_move(&self, tree_id: TreeID) -> Option<&MoveLamportAndID> { + if TreeID::is_deleted_root(&tree_id) { + return None; + } + + let mut ans = None; + if let Some(set) = self.tree.get(&tree_id) { + for op in set.iter().rev() { + if op.effected { + ans = Some(op); + break; + } + } + } + + ans + } + + fn get_children_with_id( + &self, + parent: TreeParentId, + cache: &TreeParentToChildrenCache, + ) -> Vec<(TreeID, Option, IdFull)> { + let Some(children_ids) = cache.get_children(parent) else { + return vec![]; + }; + let mut ans = Vec::with_capacity(children_ids.len()); + for child in children_ids.iter() { + let Some(op) = self.get_last_effective_move(*child) else { + panic!("child {:?} has no last effective move", child); + }; + + assert_eq!(op.op.parent_id(), parent); + ans.push((*child, op.op.fractional_index().clone(), op.id_full())); + } + // The children should be sorted by the position. + // If the fractional index is the same, then sort by the lamport and peer. + ans.sort_by(|a, b| { + a.1.cmp(&b.1) + .then(a.2.lamport.cmp(&b.2.lamport).then(a.2.peer.cmp(&b.2.peer))) + }); + ans + } +} + +#[derive(Debug)] +struct TreeParentToChildrenCache { + cache: FxHashMap>, +} + +impl TreeParentToChildrenCache { + fn get_children(&self, parent: TreeParentId) -> Option<&BTreeSet> { + self.cache.get(&parent) + } + + fn init_from_tree_cache(tree_cache: &TreeCacheForDiff) -> Self { + let mut cache = Self { + cache: FxHashMap::default(), + }; + for (tree_id, _) in tree_cache.tree.iter() { + let Some(op) = tree_cache.get_last_effective_move(*tree_id) else { + continue; + }; + + cache + .cache + .entry(op.op.parent_id()) + .or_default() + .insert(op.op.target()); + } + cache + } + + fn record_change( + &mut self, + target: TreeID, + old_parent: TreeParentId, + new_parent: TreeParentId, + ) { + if !old_parent.is_unexist() { + self.cache.get_mut(&old_parent).unwrap().remove(&target); + } + self.cache.entry(new_parent).or_default().insert(target); + } +} diff --git a/vendor/loro-internal/src/diff_calc/unknown.rs b/vendor/loro-internal/src/diff_calc/unknown.rs new file mode 100644 index 00000000..25a6fbff --- /dev/null +++ b/vendor/loro-internal/src/diff_calc/unknown.rs @@ -0,0 +1,32 @@ +use loro_common::ContainerID; + +use crate::{container::idx::ContainerIdx, event::InternalDiff, OpLog}; + +use super::{DiffCalcVersionInfo, DiffCalculatorTrait, DiffMode}; + +#[derive(Debug, Default)] +pub struct UnknownDiffCalculator; + +impl DiffCalculatorTrait for UnknownDiffCalculator { + fn start_tracking(&mut self, _oplog: &OpLog, _vv: &crate::VersionVector, _mode: DiffMode) {} + + fn apply_change( + &mut self, + _oplog: &OpLog, + _op: crate::op::RichOp, + _vv: Option<&crate::VersionVector>, + ) { + } + + fn finish_this_round(&mut self) {} + + fn calculate_diff( + &mut self, + _idx: ContainerIdx, + _oplog: &OpLog, + _info: DiffCalcVersionInfo, + _on_new_container: impl FnMut(&ContainerID), + ) -> (InternalDiff, DiffMode) { + (InternalDiff::Unknown, DiffMode::Checkout) + } +} diff --git a/vendor/loro-internal/src/encoding.rs b/vendor/loro-internal/src/encoding.rs new file mode 100644 index 00000000..c234fb95 --- /dev/null +++ b/vendor/loro-internal/src/encoding.rs @@ -0,0 +1,598 @@ +pub(crate) mod arena; +pub(crate) mod fast_snapshot; +pub(crate) mod json_schema; +mod outdated_encode_reordered; +mod shallow_snapshot; +pub(crate) mod value; +pub(crate) mod value_register; +pub(crate) use outdated_encode_reordered::{ + decode_op, encode_op, get_op_prop, EncodedDeleteStartId, IterableEncodedDeleteStartId, +}; +use outdated_encode_reordered::{import_changes_to_oplog, ImportChangesResult}; +pub(crate) use value::OwnedValue; + +use crate::change::Change; +use crate::version::{Frontiers, VersionRange}; +use crate::LoroDoc; +use crate::{oplog::OpLog, LoroError, VersionVector}; +use loro_common::{HasIdSpan, IdSpan, InternalString, LoroEncodeError, LoroResult, ID}; +use num_traits::{FromPrimitive, ToPrimitive}; +use std::borrow::Cow; + +/// The mode of the export. +/// +/// Loro CRDT internally consists of two parts: document history and current document state. +/// The export modes offer various options to meet different requirements. +/// +/// - CRDT property: Documents maintain consistent states when they receive the same set of updates. +/// - In real-time collaboration, peers typically only need to synchronize updates +/// (operations/history) to achieve consistency. +/// +/// ## Update Export +/// +/// - Exports only the history part, containing multiple operations. +/// - Suitable for real-time collaboration scenarios where peers only need to synchronize updates. +/// +/// ## Snapshot Export +/// +/// ### Default Snapshot +/// +/// - Includes complete history and current full state. +/// +/// ### Shallow Snapshot +/// +/// - Contains the complete current state. +/// - Retains partial history starting from a specified version. +/// +/// ### State-only Snapshot +/// +/// - Exports the state of the target version. +/// - Includes a minimal set of operation history. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub enum ExportMode<'a> { + /// It contains the full history and the current state of the document. + Snapshot, + /// It contains the history since the `from` version vector. + Updates { from: Cow<'a, VersionVector> }, + /// This mode exports the history in the specified range. + UpdatesInRange { spans: Cow<'a, [IdSpan]> }, + /// The shallow snapshot only contains the history since the target frontiers + ShallowSnapshot(Cow<'a, Frontiers>), + /// The state only snapshot exports the state of the target version + /// with a minimal set of history (a few ops). + /// + /// It's a shallow snapshot with depth=1 at the target version. + /// If the target version is None, it will use the latest version as the target version. + StateOnly(Option>), + /// The snapshot at the specified frontiers. It contains the full history + /// till the target frontiers and the state at the target frontiers. + SnapshotAt { version: Cow<'a, Frontiers> }, +} + +impl<'a> ExportMode<'a> { + /// It contains the full history and the current state of the document. + pub fn snapshot() -> Self { + ExportMode::Snapshot + } + + /// It contains the history since the `from` version vector. + pub fn updates(from: &'a VersionVector) -> Self { + ExportMode::Updates { + from: Cow::Borrowed(from), + } + } + + /// It contains the history since the `from` version vector. + pub fn updates_owned(from: VersionVector) -> Self { + ExportMode::Updates { + from: Cow::Owned(from), + } + } + + /// It contains all the history of the document. + pub fn all_updates() -> Self { + ExportMode::Updates { + from: Cow::Owned(Default::default()), + } + } + + /// This mode exports the history in the specified range. + pub fn updates_in_range(spans: impl Into>) -> Self { + ExportMode::UpdatesInRange { + spans: spans.into(), + } + } + + /// The shallow snapshot only contains the history since the target frontiers. + pub fn shallow_snapshot(frontiers: &'a Frontiers) -> Self { + ExportMode::ShallowSnapshot(Cow::Borrowed(frontiers)) + } + + /// The shallow snapshot only contains the history since the target frontiers. + pub fn shallow_snapshot_owned(frontiers: Frontiers) -> Self { + ExportMode::ShallowSnapshot(Cow::Owned(frontiers)) + } + + /// The shallow snapshot only contains the history since the target frontiers. + pub fn shallow_snapshot_since(id: ID) -> Self { + let frontiers = Frontiers::from_id(id); + ExportMode::ShallowSnapshot(Cow::Owned(frontiers)) + } + + /// The state only snapshot exports the state of the target version + /// with a minimal set of history (a few ops). + /// + /// It's a shallow snapshot with depth=1 at the target version. + /// If the target version is None, it will use the latest version as the target version. + pub fn state_only(frontiers: Option<&'a Frontiers>) -> Self { + ExportMode::StateOnly(frontiers.map(Cow::Borrowed)) + } + + /// The snapshot at the specified frontiers. It contains the full history + /// till the target frontiers and the state at the target frontiers. + pub fn snapshot_at(frontiers: &'a Frontiers) -> Self { + ExportMode::SnapshotAt { + version: Cow::Borrowed(frontiers), + } + } + + /// This mode exports the history within the specified version vector. + pub fn updates_till(vv: &VersionVector) -> ExportMode<'static> { + let mut spans = Vec::with_capacity(vv.len()); + for (peer, counter) in vv.iter() { + if *counter > 0 { + spans.push(IdSpan::new(*peer, 0, *counter)); + } + } + + ExportMode::UpdatesInRange { + spans: Cow::Owned(spans), + } + } +} + +const MAGIC_BYTES: [u8; 4] = *b"loro"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum EncodeMode { + // This is a config option, it won't be used in encoding. + Auto = 255, + OutdatedRle = 1, + OutdatedSnapshot = 2, + FastSnapshot = 3, + FastUpdates = 4, +} + +impl num_traits::FromPrimitive for EncodeMode { + #[allow(trivial_numeric_casts)] + #[inline] + fn from_i64(n: i64) -> Option { + match n { + n if n == EncodeMode::Auto as i64 => Some(EncodeMode::Auto), + n if n == EncodeMode::OutdatedRle as i64 => Some(EncodeMode::OutdatedRle), + n if n == EncodeMode::OutdatedSnapshot as i64 => Some(EncodeMode::OutdatedSnapshot), + n if n == EncodeMode::FastSnapshot as i64 => Some(EncodeMode::FastSnapshot), + n if n == EncodeMode::FastUpdates as i64 => Some(EncodeMode::FastUpdates), + _ => None, + } + } + #[inline] + fn from_u64(n: u64) -> Option { + Self::from_i64(n as i64) + } +} + +impl num_traits::ToPrimitive for EncodeMode { + #[inline] + #[allow(trivial_numeric_casts)] + fn to_i64(&self) -> Option { + Some(match *self { + EncodeMode::Auto => EncodeMode::Auto as i64, + EncodeMode::OutdatedRle => EncodeMode::OutdatedRle as i64, + EncodeMode::OutdatedSnapshot => EncodeMode::OutdatedSnapshot as i64, + EncodeMode::FastSnapshot => EncodeMode::FastSnapshot as i64, + EncodeMode::FastUpdates => EncodeMode::FastUpdates as i64, + }) + } + #[inline] + fn to_u64(&self) -> Option { + self.to_i64().map(|x| x as u64) + } +} + +impl EncodeMode { + pub fn to_bytes(self) -> [u8; 2] { + let value = self.to_u16().unwrap(); + value.to_be_bytes() + } + + pub fn is_snapshot(self) -> bool { + matches!( + self, + EncodeMode::OutdatedSnapshot | EncodeMode::FastSnapshot + ) + } +} + +impl TryFrom<[u8; 2]> for EncodeMode { + type Error = LoroError; + + fn try_from(value: [u8; 2]) -> Result { + let value = u16::from_be_bytes(value); + Self::from_u16(value).ok_or(LoroError::IncompatibleFutureEncodingError(value as usize)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ImportStatus { + pub success: VersionRange, + pub pending: Option, +} + +pub(crate) fn decode_oplog( + oplog: &mut OpLog, + parsed: ParsedHeaderAndBody, +) -> Result { + let changes = decode_oplog_changes(oplog, parsed)?; + let result = apply_decoded_changes_to_oplog(oplog, changes); + if result.has_deps_before_shallow_root { + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + Ok(result.status) +} + +pub(crate) fn decode_oplog_changes( + oplog: &mut OpLog, + parsed: ParsedHeaderAndBody, +) -> Result, LoroError> { + let ParsedHeaderAndBody { mode, body, .. } = parsed; + match mode { + EncodeMode::OutdatedRle | EncodeMode::OutdatedSnapshot => { + Err(LoroError::ImportUnsupportedEncodingMode) + } + EncodeMode::FastSnapshot => fast_snapshot::decode_oplog(oplog, body), + EncodeMode::FastUpdates => fast_snapshot::decode_updates(oplog, body.to_vec().into()), + EncodeMode::Auto => unreachable!(), + } +} + +pub(crate) struct ApplyDecodedChangesResult { + pub status: ImportStatus, + pub has_deps_before_shallow_root: bool, +} + +pub(crate) fn apply_decoded_changes_to_oplog( + oplog: &mut OpLog, + changes: Vec, +) -> ApplyDecodedChangesResult { + let ImportChangesResult { + mut imported, + latest_ids, + pending_changes, + changes_that_have_deps_before_shallow_root, + } = import_changes_to_oplog(changes, oplog); + + let mut pending = VersionRange::default(); + pending_changes.iter().for_each(|c| { + pending.extends_to_include_id_span(c.id_span()); + }); + // TODO: PERF: should we use hashmap to filter latest_ids with the same peer first? + oplog.try_apply_pending(latest_ids, Some(&mut imported)); + oplog + .import_unknown_lamport_pending_changes(pending_changes) + .expect("importing unknown-lamport pending changes is infallible"); + ApplyDecodedChangesResult { + status: ImportStatus { + success: imported, + pending: (!pending.is_empty()).then_some(pending), + }, + has_deps_before_shallow_root: !changes_that_have_deps_before_shallow_root.is_empty(), + } +} + +pub(crate) struct ParsedHeaderAndBody<'a> { + pub checksum: [u8; 16], + pub checksum_body: &'a [u8], + pub mode: EncodeMode, + pub body: &'a [u8], +} + +const XXH_SEED: u32 = u32::from_le_bytes(*b"LORO"); +impl ParsedHeaderAndBody<'_> { + /// Return if the checksum is correct. + fn check_checksum(&self) -> LoroResult<()> { + match self.mode { + EncodeMode::OutdatedRle | EncodeMode::OutdatedSnapshot => { + if md5::compute(self.checksum_body).0 != self.checksum { + return Err(LoroError::DecodeChecksumMismatchError); + } + } + EncodeMode::FastSnapshot | EncodeMode::FastUpdates => { + let mut expected_bytes = [0; 4]; + expected_bytes.copy_from_slice(&self.checksum[12..16]); + let expected = u32::from_le_bytes(expected_bytes); + if xxhash_rust::xxh32::xxh32(self.checksum_body, XXH_SEED) != expected { + return Err(LoroError::DecodeChecksumMismatchError); + } + } + EncodeMode::Auto => { + return Err(LoroError::DecodeError( + "Invalid import mode `Auto` in encoded blob" + .to_string() + .into_boxed_str(), + )); + } + } + + Ok(()) + } +} + +const MIN_HEADER_SIZE: usize = 22; +pub(crate) fn parse_header_and_body( + bytes: &[u8], + check_checksum: bool, +) -> Result, LoroError> { + let reader = &bytes; + if bytes.len() < MIN_HEADER_SIZE { + return Err(LoroError::DecodeError("Invalid import data".into())); + } + + let (magic_bytes, reader) = reader.split_at(4); + if magic_bytes != MAGIC_BYTES { + return Err(LoroError::DecodeError("Invalid magic bytes".into())); + } + + let (checksum, reader) = reader.split_at(16); + let checksum_body = reader; + let (mode_bytes, reader) = reader.split_at(2); + let mode: EncodeMode = [mode_bytes[0], mode_bytes[1]].try_into()?; + if mode == EncodeMode::Auto { + return Err(LoroError::DecodeError( + "Invalid import mode `Auto` in encoded blob" + .to_string() + .into_boxed_str(), + )); + } + let mut checksum_arr = [0; 16]; + checksum_arr.copy_from_slice(checksum); + + let ans = ParsedHeaderAndBody { + mode, + checksum_body, + checksum: checksum_arr, + body: reader, + }; + + if check_checksum { + ans.check_checksum()?; + } + Ok(ans) +} + +pub(crate) fn export_fast_snapshot(doc: &LoroDoc) -> Vec { + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + fast_snapshot::encode_snapshot(doc, ans); + Ok(()) + }) + .unwrap() +} + +pub(crate) fn export_snapshot_at( + doc: &LoroDoc, + frontiers: &Frontiers, +) -> Result, LoroEncodeError> { + check_target_version_reachable(doc, frontiers)?; + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + shallow_snapshot::encode_snapshot_at(doc, frontiers, ans) + }) +} + +pub(crate) fn export_fast_updates(doc: &LoroDoc, vv: &VersionVector) -> Vec { + encode_with(EncodeMode::FastUpdates, &mut |ans| { + fast_snapshot::encode_updates(doc, vv, ans); + Ok(()) + }) + .unwrap() +} + +pub(crate) fn export_fast_updates_in_range(oplog: &OpLog, spans: &[IdSpan]) -> Vec { + encode_with(EncodeMode::FastUpdates, &mut |ans| { + fast_snapshot::encode_updates_in_range(oplog, spans, ans); + Ok(()) + }) + .unwrap() +} + +pub(crate) fn export_shallow_snapshot( + doc: &LoroDoc, + f: &Frontiers, +) -> Result, LoroEncodeError> { + check_target_version_reachable(doc, f)?; + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + shallow_snapshot::export_shallow_snapshot(doc, f, ans)?; + Ok(()) + }) +} + +fn check_target_version_reachable(doc: &LoroDoc, f: &Frontiers) -> Result<(), LoroEncodeError> { + let oplog = doc.oplog.lock(); + if !oplog.dag.can_export_shallow_snapshot_on(f) { + return Err(LoroEncodeError::FrontiersNotFound(format!("{f:?}"))); + } + + Ok(()) +} + +pub(crate) fn export_state_only_snapshot( + doc: &LoroDoc, + f: &Frontiers, +) -> Result, LoroEncodeError> { + check_target_version_reachable(doc, f)?; + encode_with(EncodeMode::FastSnapshot, &mut |ans| { + shallow_snapshot::export_state_only_snapshot(doc, f, ans)?; + Ok(()) + }) +} + +fn encode_with( + mode: EncodeMode, + f: &mut dyn FnMut(&mut Vec) -> Result<(), LoroEncodeError>, +) -> Result, LoroEncodeError> { + // HEADER + let mut ans = Vec::with_capacity(MIN_HEADER_SIZE); + ans.extend(MAGIC_BYTES); + let checksum = [0; 16]; + ans.extend(checksum); + ans.extend(mode.to_bytes()); + + // BODY + f(&mut ans)?; + + // CHECKSUM in HEADER + let checksum_body = &ans[20..]; + let checksum = xxhash_rust::xxh32::xxh32(checksum_body, XXH_SEED); + ans[16..20].copy_from_slice(&checksum.to_le_bytes()); + Ok(ans) +} + +pub(crate) fn decode_snapshot( + doc: &LoroDoc, + mode: EncodeMode, + body: &[u8], + origin: InternalString, +) -> Result { + match mode { + EncodeMode::OutdatedSnapshot => { + return Err(LoroError::ImportUnsupportedEncodingMode); + } + EncodeMode::FastSnapshot => { + fast_snapshot::decode_snapshot(doc, body.to_vec().into(), origin)? + } + _ => { + return Err(LoroError::DecodeError( + format!("Invalid snapshot encoding mode: {mode:?}").into_boxed_str(), + )); + } + }; + Ok(ImportStatus { + success: VersionRange::from_vv(&doc.oplog_vv()), + pending: None, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum EncodedBlobMode { + Snapshot, + OutdatedSnapshot, + ShallowSnapshot, + OutdatedRle, + Updates, +} + +impl std::fmt::Display for EncodedBlobMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + EncodedBlobMode::OutdatedRle => "outdated-update", + EncodedBlobMode::OutdatedSnapshot => "outdated-snapshot", + EncodedBlobMode::Snapshot => "snapshot", + EncodedBlobMode::ShallowSnapshot => "shallow-snapshot", + EncodedBlobMode::Updates => "update", + }) + } +} + +impl EncodedBlobMode { + pub fn is_snapshot(&self) -> bool { + matches!( + self, + EncodedBlobMode::Snapshot + | EncodedBlobMode::ShallowSnapshot + | EncodedBlobMode::OutdatedSnapshot + ) + } +} + +#[derive(Debug, Clone)] +pub struct ImportBlobMetadata { + /// The partial start version vector. + /// + /// Import blob includes all the ops from `partial_start_vv` to `partial_end_vv`. + /// However, it does not constitute a complete version vector, as it only contains counters + /// from peers included within the import blob. + pub partial_start_vv: VersionVector, + /// The partial end version vector. + /// + /// Import blob includes all the ops from `partial_start_vv` to `partial_end_vv`. + /// However, it does not constitute a complete version vector, as it only contains counters + /// from peers included within the import blob. + pub partial_end_vv: VersionVector, + pub start_timestamp: i64, + pub start_frontiers: Frontiers, + pub end_timestamp: i64, + pub change_num: u32, + pub mode: EncodedBlobMode, +} + +impl LoroDoc { + /// Decodes the metadata for an imported blob from the provided bytes. + pub fn decode_import_blob_meta( + blob: &[u8], + check_checksum: bool, + ) -> LoroResult { + let parsed = parse_header_and_body(blob, check_checksum)?; + match parsed.mode { + EncodeMode::Auto => unreachable!(), + EncodeMode::OutdatedRle | EncodeMode::OutdatedSnapshot => { + Err(LoroError::ImportUnsupportedEncodingMode) + } + EncodeMode::FastSnapshot => fast_snapshot::decode_snapshot_blob_meta(parsed), + EncodeMode::FastUpdates => fast_snapshot::decode_updates_blob_meta(parsed), + } + } +} + +#[cfg(test)] +mod test { + + use loro_common::{loro_value, ContainerID, ContainerType, LoroValue, ID}; + + #[test] + fn test_value_encode_size() { + fn assert_size(value: LoroValue, max_size: usize) { + let size = postcard::to_allocvec(&value).unwrap().len(); + assert!( + size <= max_size, + "value: {:?}, size: {}, max_size: {}", + value, + size, + max_size + ); + } + + assert_size(LoroValue::Null, 1); + assert_size(LoroValue::I64(1), 2); + assert_size(LoroValue::Double(1.), 9); + assert_size(LoroValue::Bool(true), 2); + assert_size(LoroValue::String("123".to_string().into()), 5); + assert_size(LoroValue::Binary(vec![1, 2, 3].into()), 5); + assert_size( + loro_value!({ + "a": 1, + "b": 2, + }), + 10, + ); + assert_size(loro_value!([1, 2, 3]), 8); + assert_size( + LoroValue::Container(ContainerID::new_normal(ID::new(1, 1), ContainerType::Map)), + 5, + ); + assert_size( + LoroValue::Container(ContainerID::new_root("a", ContainerType::Map)), + 5, + ); + } +} diff --git a/vendor/loro-internal/src/encoding/arena.rs b/vendor/loro-internal/src/encoding/arena.rs new file mode 100644 index 00000000..5ede1b71 --- /dev/null +++ b/vendor/loro-internal/src/encoding/arena.rs @@ -0,0 +1,277 @@ +use std::{borrow::Cow, ops::Deref}; + +use crate::InternalString; +use loro_common::{ContainerID, ContainerType, Counter, LoroError, LoroResult, PeerID}; +use rustc_hash::FxHashSet; +use serde_columnar::columnar; + +use super::{ + outdated_encode_reordered::PeerIdx, + value::{Value, ValueDecodedArenasTrait, ValueEncodeRegister}, + value_register::ValueRegister, +}; + +#[derive(Debug)] +pub struct EncodedRegisters<'a> { + pub(super) peer: ValueRegister, + pub(super) key: ValueRegister, + pub(super) tree_id: ValueRegister, + pub(super) position: either::Either, ValueRegister<&'a [u8]>>, +} + +impl ValueEncodeRegister for EncodedRegisters<'_> { + fn key_mut(&mut self) -> &mut ValueRegister { + &mut self.key + } + + fn peer_mut(&mut self) -> &mut ValueRegister { + &mut self.peer + } + + fn encode_tree_op( + &mut self, + op: &crate::container::tree::tree_op::TreeOp, + ) -> super::value::Value<'static> { + Value::TreeMove(super::value::EncodedTreeMove::from_tree_op(op, self)) + } +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(crate) struct EncodedContainer { + #[columnar(strategy = "BoolRle")] + is_root: bool, + #[columnar(strategy = "Rle")] + kind: u8, + #[columnar(strategy = "Rle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + key_idx_or_counter: i32, +} + +impl EncodedContainer { + pub fn as_container_id(&self, arenas: &dyn ValueDecodedArenasTrait) -> LoroResult { + if self.is_root { + Ok(ContainerID::Root { + container_type: ContainerType::try_from_u8(self.kind) + .unwrap_or(ContainerType::Unknown(self.kind)), + name: arenas + .keys() + .get(self.key_idx_or_counter as usize) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + }) + } else { + Ok(ContainerID::Normal { + container_type: ContainerType::try_from_u8(self.kind) + .unwrap_or(ContainerType::Unknown(self.kind)), + peer: *(arenas + .peers() + .get(self.peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter: self.key_idx_or_counter, + }) + } + } +} + +#[columnar(ser, de)] +#[derive(Default)] +pub(crate) struct ContainerArena { + #[columnar(class = "vec", iter = "EncodedContainer")] + pub(super) containers: Vec, +} + +impl Deref for ContainerArena { + type Target = [EncodedContainer]; + + fn deref(&self) -> &Self::Target { + &self.containers + } +} + +impl ContainerArena { + pub fn encode(&self) -> Vec { + serde_columnar::to_vec(&self.containers).unwrap() + } + + pub fn decode(bytes: &[u8]) -> LoroResult { + Ok(ContainerArena { + containers: serde_columnar::from_bytes(bytes)?, + }) + } + + pub fn from_containers( + cids: Vec, + peer_register: &mut ValueRegister, + key_reg: &mut ValueRegister, + ) -> Self { + let mut ans = Self { + containers: Vec::with_capacity(cids.len()), + }; + for cid in cids { + ans.push(cid, peer_register, key_reg); + } + + ans + } + + pub fn push( + &mut self, + id: ContainerID, + peer_register: &mut ValueRegister, + register_key: &mut ValueRegister, + ) { + let (is_root, kind, peer_idx, key_idx_or_counter) = match id { + ContainerID::Root { + container_type, + name, + } => (true, container_type, 0, register_key.register(&name) as i32), + ContainerID::Normal { + container_type, + peer, + counter, + } => ( + false, + container_type, + peer_register.register(&peer), + counter, + ), + }; + self.containers.push(EncodedContainer { + is_root, + kind: kind.to_u8(), + peer_idx, + key_idx_or_counter, + }); + } +} + +#[derive(Clone, Hash, PartialEq, Eq, Debug)] +#[columnar(vec, ser, de, iterable)] +pub struct EncodedTreeID { + #[columnar(strategy = "Rle")] + pub peer_idx: PeerIdx, + #[columnar(strategy = "DeltaRle")] + pub counter: Counter, +} + +#[derive(Clone)] +#[columnar(vec, ser, de, iterable)] +pub(super) struct PositionDelta<'a> { + #[columnar(strategy = "Rle")] + common_prefix_length: usize, + #[columnar(borrow)] + rest: Cow<'a, [u8]>, +} + +#[derive(Default)] +#[columnar(ser, de)] +pub(crate) struct PositionArena<'a> { + #[columnar(class = "vec", iter = "PositionDelta<'a>")] + pub(super) positions: Vec>, +} + +impl<'a> PositionArena<'a> { + pub fn from_positions(positions: impl IntoIterator) -> Self { + let iter = positions.into_iter(); + let mut ans = Vec::with_capacity(iter.size_hint().0); + let mut last_bytes: &[u8] = &[]; + for p in iter { + let common = longest_common_prefix_length(last_bytes, p); + let rest = &p[common..]; + last_bytes = p; + ans.push(PositionDelta { + common_prefix_length: common, + rest: Cow::Borrowed(rest), + }) + } + Self { positions: ans } + } + + pub fn try_parse_to_positions(self) -> LoroResult>> { + let mut ans: Vec> = Vec::with_capacity(self.positions.len()); + for PositionDelta { + common_prefix_length, + rest, + } in self.positions + { + if let Some(last_bytes) = ans.last() { + if common_prefix_length > last_bytes.len() { + return Err(LoroError::DecodeError( + "Decode position arena failed: common prefix out of range".into(), + )); + } + } else if common_prefix_length != 0 { + return Err(LoroError::DecodeError( + "Decode position arena failed: first common prefix is nonzero".into(), + )); + } + + // +1 for Fractional Index + let capacity = rest + .len() + .checked_add(common_prefix_length) + .and_then(|len| len.checked_add(1)) + .ok_or_else(|| { + LoroError::DecodeError( + "Decode position arena failed: position length overflow".into(), + ) + })?; + let mut p = Vec::with_capacity(capacity); + if let Some(last_bytes) = ans.last() { + p.extend_from_slice(&last_bytes[0..common_prefix_length]); + } + p.extend_from_slice(rest.as_ref()); + ans.push(p); + } + Ok(ans) + } + + pub fn encode(&self) -> Vec { + serde_columnar::to_vec(&self).unwrap() + } + + pub fn decode<'de: 'a>(bytes: &'de [u8]) -> LoroResult { + Ok(serde_columnar::from_bytes(bytes)?) + } + + pub fn encode_v2(&self) -> Vec { + if self.positions.is_empty() { + return Vec::new(); + } + + serde_columnar::to_vec(&self).unwrap() + } + + pub fn decode_v2<'de: 'a>(bytes: &'de [u8]) -> LoroResult { + if bytes.is_empty() { + return Ok(Self::default()); + } + + Ok(serde_columnar::from_bytes(bytes)?) + } +} + +fn longest_common_prefix_length(a: &[u8], b: &[u8]) -> usize { + a.iter().zip(b.iter()).take_while(|(x, y)| x == y).count() +} + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use super::{PositionArena, PositionDelta}; + + #[test] + fn position_arena_rejects_invalid_common_prefix() { + let arena = PositionArena { + positions: vec![PositionDelta { + common_prefix_length: 1, + rest: Cow::Borrowed(&[]), + }], + }; + + assert!(arena.try_parse_to_positions().is_err()); + } +} diff --git a/vendor/loro-internal/src/encoding/fast_snapshot.rs b/vendor/loro-internal/src/encoding/fast_snapshot.rs new file mode 100644 index 00000000..a3d9a993 --- /dev/null +++ b/vendor/loro-internal/src/encoding/fast_snapshot.rs @@ -0,0 +1,508 @@ +//! Fast snapshot encoding and decoding. +//! +//! # Layout +//! +//! - u32 in little endian for len of bytes for oplog +//! - oplog bytes +//! - u32 in little endian for len of bytes for state +//! - state bytes +//! - u32 in little endian for len of bytes for gc +//! - gc bytes +//! +//! All of `oplog bytes`, `state bytes` and `gc bytes` are encoded KV store bytes. +//! +//! +//! +use std::io::{Read, Write}; + +use crate::{ + change::Change, encoding::shallow_snapshot, oplog::ChangeStore, version::Frontiers, LoroDoc, + OpLog, VersionVector, +}; +use bytes::{Buf, Bytes}; +use loro_common::{HasCounterSpan, IdSpan, InternalString, LoroError, LoroResult}; + +use super::{EncodedBlobMode, ImportBlobMetadata, ParsedHeaderAndBody}; +pub(crate) const EMPTY_MARK: &[u8] = b"E"; +pub(crate) struct Snapshot { + pub oplog_bytes: Bytes, + pub state_bytes: Option, + pub shallow_root_state_bytes: Bytes, +} + +pub(super) fn _encode_snapshot(s: Snapshot, w: &mut W) { + w.write_all(&(s.oplog_bytes.len() as u32).to_le_bytes()) + .unwrap(); + w.write_all(&s.oplog_bytes).unwrap(); + let state_bytes = s + .state_bytes + .unwrap_or_else(|| Bytes::from_static(EMPTY_MARK)); + w.write_all(&(state_bytes.len() as u32).to_le_bytes()) + .unwrap(); + w.write_all(&state_bytes).unwrap(); + w.write_all(&(s.shallow_root_state_bytes.len() as u32).to_le_bytes()) + .unwrap(); + w.write_all(&s.shallow_root_state_bytes).unwrap(); +} + +pub(super) fn _decode_snapshot_bytes(bytes: Bytes) -> LoroResult { + let mut r = bytes.reader(); + let oplog_bytes_len = read_u32_le(&mut r)? as usize; + if r.get_ref().len() < oplog_bytes_len { + return Err(LoroError::DecodeError( + "decode_snapshot: invalid oplog bytes length" + .to_string() + .into_boxed_str(), + )); + } + let oplog_bytes = r.get_mut().copy_to_bytes(oplog_bytes_len); + let state_bytes_len = read_u32_le(&mut r)? as usize; + if r.get_ref().len() < state_bytes_len { + return Err(LoroError::DecodeError( + "decode_snapshot: invalid state bytes length" + .to_string() + .into_boxed_str(), + )); + } + let state_bytes = r.get_mut().copy_to_bytes(state_bytes_len); + let state_bytes = if state_bytes == EMPTY_MARK { + None + } else { + Some(state_bytes) + }; + let shallow_bytes_len = read_u32_le(&mut r)? as usize; + if r.get_ref().len() < shallow_bytes_len { + return Err(LoroError::DecodeError( + "decode_snapshot: invalid shallow root bytes length" + .to_string() + .into_boxed_str(), + )); + } + let shallow_root_state_bytes = r.get_mut().copy_to_bytes(shallow_bytes_len); + if r.get_ref().has_remaining() { + return Err(LoroError::DecodeError( + "decode_snapshot: trailing bytes after snapshot" + .to_string() + .into_boxed_str(), + )); + } + + Ok(Snapshot { + oplog_bytes, + state_bytes, + shallow_root_state_bytes, + }) +} + +pub(super) fn _decode_snapshot_meta_partial(bytes: &[u8]) -> LoroResult<(&[u8], bool)> { + let mut r = bytes; + let oplog_bytes_len = read_u32_le_slice(&mut r)? as usize; + if r.len() < oplog_bytes_len { + return Err(LoroError::DecodeDataCorruptionError); + } + let oplog_bytes = &r[..oplog_bytes_len]; + r = &r[oplog_bytes_len..]; + let state_bytes_len = read_u32_le_slice(&mut r)? as usize; + if r.len() < state_bytes_len { + return Err(LoroError::DecodeDataCorruptionError); + } + r = &r[state_bytes_len..]; + let shallow_bytes_len = read_u32_le_slice(&mut r)? as usize; + if r.len() < shallow_bytes_len { + return Err(LoroError::DecodeDataCorruptionError); + } + r = &r[shallow_bytes_len..]; + if !r.is_empty() { + return Err(LoroError::DecodeDataCorruptionError); + } + + Ok((oplog_bytes, shallow_bytes_len > 0)) +} + +fn read_u32_le_slice(r: &mut &[u8]) -> LoroResult { + let mut buf = [0; 4]; + r.read_exact(&mut buf) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + Ok(u32::from_le_bytes(buf)) +} + +fn read_u32_le(r: &mut bytes::buf::Reader) -> LoroResult { + let mut buf = [0; 4]; + r.read_exact(&mut buf).map_err(|_| { + LoroError::DecodeError( + "decode_snapshot: unexpected end of snapshot bytes" + .to_string() + .into_boxed_str(), + ) + })?; + Ok(u32::from_le_bytes(buf)) +} + +pub(crate) fn decode_snapshot( + doc: &LoroDoc, + bytes: Bytes, + origin: InternalString, +) -> LoroResult<()> { + let snapshot = _decode_snapshot_bytes(bytes)?; + decode_snapshot_inner(snapshot, doc, origin) +} + +pub(crate) fn decode_snapshot_inner( + snapshot: Snapshot, + doc: &LoroDoc, + origin: InternalString, +) -> Result<(), LoroError> { + let Snapshot { + oplog_bytes, + state_bytes, + shallow_root_state_bytes, + } = snapshot; + ensure_cov::notify_cov("loro_internal::import::fast_snapshot::decode_snapshot"); + let mut oplog = doc.oplog().lock(); + if !oplog.is_empty() { + return Err(LoroError::DecodeError( + "decode_snapshot: cannot import snapshot into a non-empty doc" + .to_string() + .into_boxed_str(), + )); + } + + let mut state = doc.app_state().lock(); + + state.check_before_decode_snapshot()?; + + if !state.frontiers.is_empty() { + return Err(LoroError::DecodeError( + "decode_snapshot: app state frontiers must be empty before import" + .to_string() + .into_boxed_str(), + )); + } + + if !oplog.frontiers().is_empty() { + return Err(LoroError::DecodeError( + "decode_snapshot: oplog frontiers must be empty before import" + .to_string() + .into_boxed_str(), + )); + } + let need_calc = state_bytes.is_none(); + + let arena_checkpoint = oplog.arena.checkpoint_for_rollback(); + // FLOWSTATE §A14.4.0: env-gated decode stage probe. + static DECODE_PROBE: std::sync::OnceLock = std::sync::OnceLock::new(); + let decode_probe = + *DECODE_PROBE.get_or_init(|| std::env::var_os("FLOWSTATE_DECODE_PROBE").is_some()); + let decode_result = (|| -> LoroResult<()> { + let probe_t = std::time::Instant::now(); + oplog.decode_change_store(oplog_bytes)?; + if decode_probe { + eprintln!( + "[flowstate-decode-probe] change_store={:?}", + probe_t.elapsed() + ); + } + let probe_t = std::time::Instant::now(); + let state_frontiers; + if shallow_root_state_bytes.is_empty() { + ensure_cov::notify_cov("loro_internal::import::snapshot::normal"); + if let Some(bytes) = state_bytes { + state.store.decode(bytes)?; + } + state_frontiers = oplog.frontiers().clone(); + } else { + ensure_cov::notify_cov("loro_internal::import::snapshot::gc"); + let shallow_root_state_frontiers = state.store.decode_gc( + shallow_root_state_bytes.clone(), + oplog.dag().shallow_since_frontiers().clone(), + doc.config.clone(), + )?; + if decode_probe { + eprintln!( + "[flowstate-decode-probe] gc_store={:?}", + probe_t.elapsed() + ); + } + state.store.decode_state_by_two_bytes( + shallow_root_state_bytes, + state_bytes.unwrap_or_default(), + )?; + + let shallow_root_store = state.shallow_root_store().cloned(); + oplog.with_history_cache(|h| { + h.set_shallow_root_store(shallow_root_store); + }); + + if need_calc { + ensure_cov::notify_cov("shallow_snapshot::need_calc"); + state_frontiers = shallow_root_state_frontiers.ok_or_else(|| { + LoroError::DecodeError( + "decode_snapshot: shallow root frontiers are missing" + .to_string() + .into_boxed_str(), + ) + })?; + } else { + ensure_cov::notify_cov("shallow_snapshot::dont_need_calc"); + state_frontiers = oplog.frontiers().clone(); + } + } + + if decode_probe { + eprintln!( + "[flowstate-decode-probe] state_store={:?}", + probe_t.elapsed() + ); + } + let probe_t = std::time::Instant::now(); + state.init_with_states_and_version(state_frontiers, &oplog, vec![], false, origin)?; + if decode_probe { + eprintln!( + "[flowstate-decode-probe] init_states={:?}", + probe_t.elapsed() + ); + } + Ok(()) + })(); + + if let Err(e) = decode_result { + state.reset_to_empty_for_failed_snapshot_import(); + oplog.reset_to_empty_for_failed_snapshot_import(arena_checkpoint); + return Err(e); + } + drop(state); + drop(oplog); + if need_calc { + doc.set_detached(true); + if let Err(e) = doc._checkout_to_latest_without_commit(false) { + doc.set_detached(false); + doc.app_state() + .lock() + .reset_to_empty_for_failed_snapshot_import(); + doc.oplog() + .lock() + .reset_to_empty_for_failed_snapshot_import(arena_checkpoint); + return Err(e); + } + debug_assert_eq!(doc.state_frontiers(), doc.oplog_frontiers()); + } + + Ok(()) +} + +impl OpLog { + pub(super) fn decode_change_store(&mut self, bytes: bytes::Bytes) -> LoroResult<()> { + let v = self.change_store().import_all(bytes)?; + self.dag.set_version_by_fast_snapshot_import(v); + self.refresh_visible_op_count(); + Ok(()) + } +} + +pub(crate) fn encode_snapshot(doc: &LoroDoc, w: &mut W) { + let snapshot = encode_snapshot_inner(doc); + _encode_snapshot(snapshot, w); +} + +pub(crate) fn encode_snapshot_inner(doc: &LoroDoc) -> Snapshot { + assert!(doc.drop_pending_events().is_empty()); + let old_state_frontiers = doc.state_frontiers(); + let was_detached = doc.is_detached(); + let oplog = doc.oplog().lock(); + let mut state = doc.app_state().lock(); + let is_gc = state.store.shallow_root_store().is_some(); + if is_gc { + // TODO: PERF: this can be optimized by reusing the bytes of gc store + let f = oplog.shallow_since_frontiers().clone(); + drop(state); + drop(oplog); + let (snapshot, _) = shallow_snapshot::export_shallow_snapshot_inner(doc, &f).unwrap(); + return snapshot; + } + + assert!(!state.is_in_txn()); + let oplog_bytes = oplog.encode_change_store(); + if oplog.is_shallow() { + assert_eq!( + oplog.shallow_since_frontiers(), + state.store.shallow_root_frontiers().unwrap() + ); + } + if was_detached { + let latest = oplog.frontiers().clone(); + drop(state); + drop(oplog); + doc._checkout_without_emitting(&latest, false, true) + .unwrap(); + state = doc.app_state().lock(); + } + state.ensure_all_alive_containers(); + let state_bytes = state.store.encode(); + let snapshot = Snapshot { + oplog_bytes, + state_bytes: Some(state_bytes), + shallow_root_state_bytes: Bytes::new(), + }; + if was_detached { + drop(state); + doc._checkout_without_emitting(&old_state_frontiers, false, true) + .unwrap(); + doc.drop_pending_events(); + } + + snapshot +} + +pub(crate) fn decode_oplog(oplog: &mut OpLog, bytes: &[u8]) -> Result, LoroError> { + let oplog_len = bytes + .get(0..4) + .ok_or_else(|| LoroError::DecodeError("decode_oplog: missing length prefix".into()))?; + let oplog_len = u32::from_le_bytes( + oplog_len + .try_into() + .expect("slice length checked to be exactly 4"), + ) as usize; + let oplog_bytes = bytes + .get(4..4 + oplog_len) + .ok_or_else(|| LoroError::DecodeError("decode_oplog: invalid oplog length".into()))?; + let mut changes = ChangeStore::decode_snapshot_for_updates( + oplog_bytes.to_vec().into(), + &oplog.arena, + oplog.vv(), + )?; + changes.sort_unstable_by_key(|x| x.lamport); + Ok(changes) +} + +pub(crate) fn encode_updates(doc: &LoroDoc, vv: &VersionVector, w: &mut W) { + let oplog = doc.oplog().lock(); + oplog.export_blocks_from(vv, w); +} + +pub(crate) fn encode_updates_in_range( + oplog: &OpLog, + spans: &[IdSpan], + w: &mut W, +) { + oplog.export_blocks_in_range(spans, w); +} + +pub(crate) fn decode_updates(oplog: &mut OpLog, body: Bytes) -> Result, LoroError> { + let mut reader: &[u8] = body.as_ref(); + let mut index = 0; + let self_vv = oplog.vv(); + let mut changes = Vec::new(); + while !reader.is_empty() { + let old_reader_len = reader.len(); + let len = leb128::read::unsigned(&mut reader) + .map_err(|_| LoroError::DecodeError("decode_updates: invalid block length".into()))? + as usize; + index += old_reader_len - reader.len(); + let end = index.checked_add(len).ok_or_else(|| { + LoroError::DecodeError("decode_updates: block length overflow".into()) + })?; + if end > body.len() { + return Err(LoroError::DecodeError( + "decode_updates: truncated block payload".into(), + )); + } + let block_bytes = body.slice(index..end); + let new_changes = ChangeStore::decode_block_bytes(block_bytes, &oplog.arena, self_vv)?; + changes.extend(new_changes); + index = end; + reader = &reader[len..]; + } + + changes.sort_unstable_by_key(|x| x.lamport); + Ok(changes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_updates_rejects_truncated_block() { + let doc = LoroDoc::new(); + let mut oplog = doc.oplog.lock(); + let err = decode_updates(&mut oplog, Bytes::from_static(&[0x02, 0x01])) + .expect_err("truncated update block should be rejected"); + assert!(matches!(err, LoroError::DecodeError(_))); + } +} + +pub(crate) fn decode_snapshot_blob_meta( + parsed: ParsedHeaderAndBody, +) -> LoroResult { + let (oplog_bytes, is_shallow) = _decode_snapshot_meta_partial(parsed.body)?; + let mode = if is_shallow { + EncodedBlobMode::ShallowSnapshot + } else { + EncodedBlobMode::Snapshot + }; + + let doc = LoroDoc::new(); + let mut oplog = doc.oplog.lock(); + oplog.decode_change_store(oplog_bytes.to_vec().into())?; + let timestamp = oplog.get_greatest_timestamp(oplog.dag.frontiers()); + let f = oplog.dag.shallow_since_frontiers().clone(); + let start_timestamp = oplog.get_timestamp_of_version(&f); + let change_num = oplog.change_store().change_num() as u32; + + Ok(ImportBlobMetadata { + mode, + partial_start_vv: oplog.dag.shallow_since_vv().to_vv(), + partial_end_vv: oplog.vv().clone(), + start_timestamp, + start_frontiers: f, + end_timestamp: timestamp, + change_num, + }) +} + +pub(crate) fn decode_updates_blob_meta( + parsed: ParsedHeaderAndBody, +) -> LoroResult { + let doc = LoroDoc::new(); + let mut oplog = doc.oplog.lock(); + let changes = decode_updates(&mut oplog, parsed.body.to_vec().into())?; + let mut start_vv = VersionVector::new(); + let mut end_vv = VersionVector::new(); + for c in changes.iter() { + match start_vv.get(&c.id.peer).copied() { + Some(start) if start <= c.id.counter => {} + _ => { + start_vv.insert(c.id.peer, c.id.counter); + } + } + match end_vv.get(&c.id.peer).copied() { + Some(end) if end >= c.ctr_end() => {} + _ => { + end_vv.insert(c.id.peer, c.ctr_end()); + } + } + } + + let mut start_frontiers = Frontiers::new(); + for c in changes.iter() { + for dep in c.deps().iter() { + if let Some(start_counter) = start_vv.get(&dep.peer) { + if *start_counter > dep.counter { + start_frontiers.push(dep); + } + } else if end_vv.get(&dep.peer).is_none() { + start_frontiers.push(dep); + } + } + } + + Ok(ImportBlobMetadata { + mode: EncodedBlobMode::Updates, + partial_start_vv: start_vv, + partial_end_vv: end_vv, + start_timestamp: changes.first().map(|x| x.timestamp).unwrap_or(0), + start_frontiers, + end_timestamp: changes.last().map(|x| x.timestamp).unwrap_or(0), + change_num: changes.len() as u32, + }) +} diff --git a/vendor/loro-internal/src/encoding/json_schema.rs b/vendor/loro-internal/src/encoding/json_schema.rs new file mode 100644 index 00000000..2f846c31 --- /dev/null +++ b/vendor/loro-internal/src/encoding/json_schema.rs @@ -0,0 +1,1912 @@ +use super::outdated_encode_reordered::ValueRegister; +use crate::{ + arena::SharedArena, + change::{Change, ChangeRef}, + container::{ + list::list_op::{DeleteSpan, DeleteSpanWithId, InnerListOp}, + map::MapSet, + richtext::TextStyleInfoFlag, + tree::tree_op::TreeOp, + }, + op::{FutureInnerContent, InnerContent, Op, SliceRange}, + oplog::BlockChangeRef, + version::Frontiers, + OpLog, VersionVector, +}; +use either::Either; +use itertools::Itertools; +use json::{JsonChange, JsonOpContent, JsonSchema}; +use loro_common::{ + ContainerID, ContainerType, Counter, HasCounterSpan, HasId, IdLp, IdSpan, LoroError, + LoroResult, LoroValue, PeerID, TreeID, ID, +}; +use rle::{HasLength, RleVec, Sliceable}; +use std::sync::Arc; + +const SCHEMA_VERSION: u8 = 1; + +fn refine_vv(vv: &VersionVector, oplog: &OpLog) -> VersionVector { + let mut refined = VersionVector::new(); + for (&peer, &counter) in vv.iter() { + if counter <= 0 { + continue; + } + let end = oplog.vv().get(&peer).copied().unwrap_or(0); + if end <= counter { + refined.insert(peer, end); + } else { + refined.insert(peer, counter); + } + } + refined +} + +pub(crate) fn export_json<'a, 'c: 'a>( + oplog: &'c OpLog, + start_vv: &VersionVector, + end_vv: &VersionVector, + with_peer_compression: bool, +) -> JsonSchema { + let actual_start_vv = refine_vv(start_vv, oplog); + let actual_end_vv = refine_vv(end_vv, oplog); + + let frontiers = oplog.dag.vv_to_frontiers(&actual_start_vv); + + let diff_changes = init_encode(oplog, &actual_start_vv, &actual_end_vv); + if with_peer_compression { + let mut peer_register = ValueRegister::::new(); + let changes = encode_changes(&diff_changes, &oplog.arena, Some(&mut peer_register)); + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: Some(peer_register.unwrap_vec()), + start_version: frontiers, + } + } else { + let changes = encode_changes(&diff_changes, &oplog.arena, None); + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: None, + start_version: frontiers, + } + } +} + +pub(crate) fn export_json_in_id_span(oplog: &OpLog, mut id_span: IdSpan) -> Vec { + id_span.normalize_(); + if id_span.counter.end <= 0 { + return vec![]; + } + id_span.counter.start = id_span.counter.start.max(0); + id_span.counter.end = id_span.counter.end.max(0); + + let end = oplog.vv().get(&id_span.peer).copied().unwrap_or(0); + if id_span.counter.start >= end { + return vec![]; + } + + id_span.counter.end = id_span.counter.end.min(end); + let mut diff_changes: Vec> = Vec::new(); + while id_span.counter.start < id_span.counter.end { + let Some(change) = oplog.get_change_at(id_span.id_start()) else { + break; + }; + let ctr_end = change.ctr_end(); + if change.id.counter >= id_span.counter.start && change.ctr_end() <= id_span.counter.end { + diff_changes.push(Either::Left(change)); + } else { + let start = if change.id.counter < id_span.counter.start { + (id_span.counter.start - change.id.counter) as usize + } else { + 0 + }; + + let end = if change.ctr_end() > id_span.counter.end { + (id_span.counter.end - change.id.counter) as usize + } else { + change.atom_len() + }; + + diff_changes.push(Either::Right(change.slice(start, end))); + } + + id_span.counter.start = ctr_end; + } + + encode_changes(&diff_changes, &oplog.arena, None) +} + +pub(crate) fn decode_json_changes( + json: JsonSchema, + arena: &SharedArena, +) -> LoroResult> { + decode_changes(json, arena) +} + +fn init_encode<'s, 'a: 's>( + oplog: &'a OpLog, + start_vv: &VersionVector, + end_vv: &VersionVector, +) -> Vec> { + let mut diff_changes: Vec> = Vec::new(); + for change in oplog.iter_changes_peer_by_peer(start_vv, end_vv) { + let start_cnt = start_vv.get(&change.id.peer).copied().unwrap_or(0); + let end_cnt = end_vv.get(&change.id.peer).copied().unwrap_or(0); + if change.id.counter < start_cnt { + if change.ctr_end() <= start_cnt { + continue; + } + + let offset = start_cnt - change.id.counter; + let to = change + .atom_len() + .min((end_cnt - change.id.counter) as usize); + diff_changes.push(Either::Right(change.slice(offset as usize, to))); + } else if change.id.counter + change.atom_len() as i32 > end_cnt { + let len = end_cnt - change.id.counter; + diff_changes.push(Either::Right(change.slice(0, len as usize))); + } else { + diff_changes.push(Either::Left(change)); + } + } + diff_changes.sort_by_key(|x| match x { + Either::Left(c) => c.lamport, + Either::Right(c) => c.lamport, + }); + diff_changes +} + +fn register_id(id: &ID, peer_register: Option<&mut ValueRegister>) -> ID { + let peer = match peer_register { + Some(peer_register) => peer_register.register(&id.peer) as PeerID, + None => id.peer, + }; + ID::new(peer as PeerID, id.counter) +} + +fn register_idlp(idlp: &IdLp, peer_register: Option<&mut ValueRegister>) -> IdLp { + let peer = match peer_register { + Some(peer_register) => peer_register.register(&idlp.peer) as PeerID, + None => idlp.peer, + }; + IdLp { + peer, + lamport: idlp.lamport, + } +} + +fn register_tree_id(tree: &TreeID, peer_register: Option<&mut ValueRegister>) -> TreeID { + TreeID { + peer: match peer_register { + Some(peer_register) => peer_register.register(&tree.peer) as PeerID, + None => tree.peer, + }, + counter: tree.counter, + } +} + +fn register_container_id( + container: ContainerID, + peer_register: Option<&mut ValueRegister>, +) -> ContainerID { + match container { + ContainerID::Normal { + peer, + counter, + container_type, + } => ContainerID::Normal { + peer: match peer_register { + Some(peer_register) => peer_register.register(&peer) as PeerID, + None => peer, + }, + counter, + container_type, + }, + r => r, + } +} + +fn convert_container_id( + container: ContainerID, + peers: &Option>, +) -> LoroResult { + match container { + ContainerID::Normal { + peer, + counter, + container_type, + } => Ok(ContainerID::Normal { + peer: get_peer_from_peers(peers, peer)?, + counter, + container_type, + }), + r => Ok(r), + } +} + +pub(crate) fn get_peer_from_peers(peers: &Option>, peer: PeerID) -> LoroResult { + match peers { + Some(peers) => peers.get(peer as usize).copied().ok_or_else(|| { + LoroError::DecodeError("peer index out of range in compressed peer table".into()) + }), + None => Ok(peer), + } +} + +fn convert_id(id: &ID, peers: &Option>) -> LoroResult { + Ok(ID { + peer: get_peer_from_peers(peers, id.peer)?, + counter: id.counter, + }) +} + +fn convert_idlp(idlp: &IdLp, peers: &Option>) -> LoroResult { + Ok(IdLp { + lamport: idlp.lamport, + peer: get_peer_from_peers(peers, idlp.peer)?, + }) +} + +fn convert_tree_id(tree: &TreeID, peers: &Option>) -> LoroResult { + Ok(TreeID { + peer: get_peer_from_peers(peers, tree.peer)?, + counter: tree.counter, + }) +} +pub(crate) fn encode_change_to_json(change: ChangeRef<'_>, arena: &SharedArena) -> JsonSchema { + let f = change.deps.clone(); + let changes = vec![encode_change(change, arena, None)]; + JsonSchema { + changes, + schema_version: SCHEMA_VERSION, + peers: None, + start_version: f, + } +} + +fn encode_changes( + diff_changes: &[Either], + arena: &SharedArena, + mut peer_register: Option<&mut ValueRegister>, +) -> Vec { + let mut changes = Vec::with_capacity(diff_changes.len()); + for change in diff_changes.iter() { + let change: &Change = match change { + Either::Left(c) => c, + Either::Right(c) => c, + }; + let c = encode_change( + ChangeRef::from_change(change), + arena, + peer_register.as_deref_mut(), + ); + changes.push(c); + } + changes +} + +pub(crate) fn encode_change( + change: ChangeRef<'_, Op>, + arena: &SharedArena, + mut peer_register: Option<&mut ValueRegister>, +) -> JsonChange { + let mut ops = Vec::with_capacity(change.ops.len()); + for Op { + counter, + container, + content, + } in change.ops.iter() + { + let mut container = arena.get_container_id(*container).unwrap(); + if container.is_normal() { + container = register_container_id(container, peer_register.as_deref_mut()); + } + let op = match container.container_type() { + ContainerType::List => match content { + InnerContent::List(list) => JsonOpContent::List(match list { + InnerListOp::Insert { slice, pos } => { + let mut values = + arena.get_values(slice.0.start as usize..slice.0.end as usize); + values.iter_mut().for_each(|x| { + if let LoroValue::Container(id) = x { + if id.is_normal() { + *id = register_container_id( + id.clone(), + peer_register.as_deref_mut(), + ); + } + } + }); + json::ListOp::Insert { + pos: *pos as u32, + value: values, + } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => json::ListOp::Delete { + pos: *pos as i32, + len: *signed_len as i32, + start_id: register_id(id_start, peer_register.as_deref_mut()), + }, + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::MovableList => match content { + InnerContent::List(list) => JsonOpContent::MovableList(match list { + InnerListOp::Insert { slice, pos } => { + let mut values = + arena.get_values(slice.0.start as usize..slice.0.end as usize); + values.iter_mut().for_each(|x| { + if let LoroValue::Container(id) = x { + if id.is_normal() { + *id = register_container_id( + id.clone(), + peer_register.as_deref_mut(), + ); + } + } + }); + json::MovableListOp::Insert { + pos: *pos as u32, + value: values, + } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => json::MovableListOp::Delete { + pos: *pos as i32, + len: *signed_len as i32, + start_id: register_id(id_start, peer_register.as_deref_mut()), + }, + InnerListOp::Move { + from, + elem_id: from_id, + to, + } => json::MovableListOp::Move { + from: *from, + to: *to, + elem_id: register_idlp(from_id, peer_register.as_deref_mut()), + }, + InnerListOp::Set { elem_id, value } => { + let value = if let LoroValue::Container(id) = value { + if id.is_normal() { + LoroValue::Container(register_container_id( + id.clone(), + peer_register.as_deref_mut(), + )) + } else { + value.clone() + } + } else { + value.clone() + }; + json::MovableListOp::Set { + elem_id: register_idlp(elem_id, peer_register.as_deref_mut()), + value, + } + } + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::Text => match content { + InnerContent::List(list) => JsonOpContent::Text(match list { + InnerListOp::InsertText { + slice, + unicode_start: _, + unicode_len: _, + pos, + } => { + let text = String::from_utf8(slice.as_bytes().to_vec()).unwrap(); + json::TextOp::Insert { pos: *pos, text } + } + InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { pos, signed_len }, + }) => json::TextOp::Delete { + pos: *pos as i32, + len: *signed_len as i32, + start_id: register_id(id_start, peer_register.as_deref_mut()), + }, + InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => json::TextOp::Mark { + start: *start, + end: *end, + style_key: key.to_string(), + style_value: value.clone(), + info: info.to_byte(), + }, + InnerListOp::StyleEnd => json::TextOp::MarkEnd, + _ => unreachable!(), + }), + _ => unreachable!(), + }, + ContainerType::Map => match content { + InnerContent::Map(MapSet { key, value }) => { + JsonOpContent::Map(if let Some(v) = value { + let value = if let LoroValue::Container(id) = v { + if id.is_normal() { + LoroValue::Container(register_container_id( + id.clone(), + peer_register.as_deref_mut(), + )) + } else { + v.clone() + } + } else { + v.clone() + }; + json::MapOp::Insert { + key: key.to_string(), + value, + } + } else { + json::MapOp::Delete { + key: key.to_string(), + } + }) + } + + _ => unreachable!(), + }, + + ContainerType::Tree => match content { + InnerContent::Tree(op) => JsonOpContent::Tree(match &**op { + TreeOp::Create { + target, + parent, + position, + } => json::TreeOp::Create { + target: register_tree_id(target, peer_register.as_deref_mut()), + parent: parent.map(|p| register_tree_id(&p, peer_register.as_deref_mut())), + fractional_index: position.clone(), + }, + TreeOp::Move { + target, + parent, + position, + } => json::TreeOp::Move { + target: register_tree_id(target, peer_register.as_deref_mut()), + parent: parent.map(|p| register_tree_id(&p, peer_register.as_deref_mut())), + fractional_index: position.clone(), + }, + TreeOp::Delete { target } => json::TreeOp::Delete { + target: register_tree_id(target, peer_register.as_deref_mut()), + }, + }), + _ => unreachable!(), + }, + ContainerType::Unknown(_) => { + let InnerContent::Future(FutureInnerContent::Unknown { prop, value }) = content + else { + unreachable!(); + }; + JsonOpContent::Future(json::FutureOpWrapper { + prop: *prop, + value: json::FutureOp::Unknown((**value).clone()), + }) + } + #[cfg(feature = "counter")] + ContainerType::Counter => { + let InnerContent::Future(f) = content else { + unreachable!() + }; + match f { + FutureInnerContent::Counter(x) => { + JsonOpContent::Future(json::FutureOpWrapper { + prop: 0, + value: json::FutureOp::Counter(super::OwnedValue::F64(*x)), + }) + } + _ => unreachable!(), + } + } + }; + ops.push(json::JsonOp { + counter: *counter, + container, + content: op, + }); + } + let c = json::JsonChange { + id: register_id(change.id, peer_register.as_deref_mut()), + ops, + deps: change + .deps + .iter() + // Make sure the order is deterministic + .sorted() + .map(|id| register_id(&id, peer_register.as_deref_mut())) + .collect(), + lamport: *change.lamport, + timestamp: *change.timestamp, + msg: change.commit_msg.as_deref().map(|x| x.to_string()), + }; + c +} + +fn decode_changes(json: JsonSchema, arena: &SharedArena) -> LoroResult> { + let JsonSchema { + schema_version, + start_version, + peers, + changes, + } = json; + if schema_version != SCHEMA_VERSION { + return Err(LoroError::DecodeError( + format!( + "unsupported json schema version: expected {SCHEMA_VERSION}, got {schema_version}" + ) + .into_boxed_str(), + )); + } + validate_json_frontiers(&start_version)?; + + let mut ans = Vec::with_capacity(changes.len()); + for json::JsonChange { + id, + timestamp, + deps, + lamport, + msg, + ops: json_ops, + } in changes + { + let id = convert_id(&id, &peers)?; + validate_json_id_counter(id, "change id")?; + let mut ops: RleVec<[Op; 1]> = RleVec::new(); + let mut expected_counter = id.counter; + if json_ops.is_empty() { + return Err(LoroError::DecodeError( + "invalid json change: change must contain at least one op".into(), + )); + } + + for op in json_ops { + let next_counter = validate_json_op_counter(expected_counter, &op)?; + let op = decode_op(op, arena, &peers)?; + validate_json_op_created_container_ids(id.peer, &op, arena)?; + ops.push(op); + expected_counter = next_counter; + } + + let deps = deps + .into_iter() + .map(|id| convert_id(&id, &peers)) + .collect::>>()?; + for dep in &deps { + validate_json_id_counter(*dep, "dependency id")?; + } + + let change = Change { + id, + timestamp, + deps: Frontiers::from_iter(deps), + lamport, + ops, + commit_msg: msg.map(|x| x.into()), + }; + ans.push(change); + } + Ok(ans) +} + +fn validate_json_frontiers(frontiers: &Frontiers) -> LoroResult<()> { + for id in frontiers.iter() { + validate_json_id_counter(id, "start version id")?; + } + + Ok(()) +} + +fn validate_json_id_counter(id: ID, name: &str) -> LoroResult<()> { + if id.counter < 0 { + return Err(LoroError::DecodeError( + format!("invalid json counter: {name} counter must be non-negative").into_boxed_str(), + )); + } + + Ok(()) +} + +fn validate_json_tree_id_counter(id: &TreeID, name: &str) -> LoroResult<()> { + if id.counter < 0 { + return Err(LoroError::DecodeError( + format!("invalid json counter: {name} counter must be non-negative").into_boxed_str(), + )); + } + + Ok(()) +} + +fn validate_json_container_id_counter(id: &ContainerID, name: &str) -> LoroResult<()> { + if let ContainerID::Normal { counter, .. } = id { + if *counter < 0 { + return Err(LoroError::DecodeError( + format!("invalid json counter: {name} counter must be non-negative") + .into_boxed_str(), + )); + } + } + + Ok(()) +} + +fn validate_json_op_counter(expected_counter: Counter, op: &json::JsonOp) -> LoroResult { + let op_len = op.content.op_len(); + if op_len == 0 { + return Err(LoroError::DecodeError( + "invalid json op counter: op length must be greater than zero".into(), + )); + } + + if expected_counter < 0 || op.counter < 0 { + return Err(LoroError::DecodeError( + "invalid json op counter: op counters must be non-negative".into(), + )); + } + + let op_len = Counter::try_from(op_len).map_err(|_| { + LoroError::DecodeError("invalid json op counter: op length is too large".into()) + })?; + if op.counter != expected_counter { + return Err(LoroError::DecodeError( + "invalid json op counter: op counters must be contiguous with the change id".into(), + )); + } + + expected_counter + .checked_add(op_len) + .ok_or_else(|| LoroError::DecodeError("invalid json op counter: counter overflow".into())) +} + +fn validate_json_op_created_container_ids( + peer: PeerID, + op: &Op, + arena: &SharedArena, +) -> LoroResult<()> { + match &op.content { + InnerContent::List(list) => match list { + InnerListOp::Insert { slice, .. } => { + for (offset, value) in arena.iter_value_slice(slice.to_range()).enumerate() { + let LoroValue::Container(id) = value else { + validate_json_value_has_no_container_refs(&value)?; + continue; + }; + let offset = Counter::try_from(offset).map_err(|_| { + LoroError::DecodeError( + "invalid json container id: list offset is too large".into(), + ) + })?; + let counter = op.counter.checked_add(offset).ok_or_else(|| { + LoroError::DecodeError("invalid json container id: counter overflow".into()) + })?; + validate_json_created_container_id(&id, ID::new(peer, counter))?; + } + } + InnerListOp::Set { value, .. } => { + if let LoroValue::Container(id) = value { + validate_json_created_container_id(id, ID::new(peer, op.counter))?; + } else { + validate_json_value_has_no_container_refs(value)?; + } + } + InnerListOp::Move { .. } + | InnerListOp::InsertText { .. } + | InnerListOp::Delete(_) + | InnerListOp::StyleEnd => {} + InnerListOp::StyleStart { value, .. } => { + validate_json_value_has_no_container_refs(value)?; + } + }, + InnerContent::Map(map) => { + if let Some(value) = &map.value { + if let LoroValue::Container(id) = value { + validate_json_created_container_id(id, ID::new(peer, op.counter))?; + } else { + validate_json_value_has_no_container_refs(value)?; + } + } + } + InnerContent::Tree(tree) => { + if let TreeOp::Create { target, .. } = tree.as_ref() { + validate_json_tree_create_target(target, ID::new(peer, op.counter))?; + } + } + InnerContent::Future(_) => {} + } + + Ok(()) +} + +fn validate_json_value_has_no_container_refs(value: &LoroValue) -> LoroResult<()> { + let mut stack = vec![value]; + while let Some(value) = stack.pop() { + match value { + LoroValue::Container(_) => { + return Err(LoroError::DecodeError( + "invalid json container id: container values must not be nested".into(), + )); + } + LoroValue::List(list) => { + stack.extend(list.iter()); + } + LoroValue::Map(map) => { + stack.extend(map.values()); + } + LoroValue::Null + | LoroValue::Bool(_) + | LoroValue::Double(_) + | LoroValue::I64(_) + | LoroValue::Binary(_) + | LoroValue::String(_) => {} + } + } + + Ok(()) +} + +fn validate_json_created_container_id(id: &ContainerID, expected_id: ID) -> LoroResult<()> { + let ContainerID::Normal { peer, counter, .. } = id else { + return Err(LoroError::DecodeError( + "invalid json container id: created child containers must be normal".into(), + )); + }; + + if *peer != expected_id.peer || *counter != expected_id.counter { + return Err(LoroError::DecodeError( + "invalid json container id: created child container id must match the op id".into(), + )); + } + + Ok(()) +} + +fn validate_json_tree_create_target(target: &TreeID, expected_id: ID) -> LoroResult<()> { + if target.peer != expected_id.peer || target.counter != expected_id.counter { + return Err(LoroError::DecodeError( + "invalid json tree target: tree create target must match the op id".into(), + )); + } + + Ok(()) +} + +fn decode_op(op: json::JsonOp, arena: &SharedArena, peers: &Option>) -> LoroResult { + let json::JsonOp { + counter, + container, + content, + } = op; + let container = convert_container_id(container, peers)?; + validate_json_container_id_counter(&container, "op container")?; + let idx = arena.register_container(&container); + let content = match container.container_type() { + ContainerType::Text => match content { + JsonOpContent::Text(text) => match text { + json::TextOp::Insert { pos, text } => { + let (slice, result) = arena.alloc_str_with_slice(&text); + InnerContent::List(InnerListOp::InsertText { + slice, + unicode_start: result.start as u32, + unicode_len: (result.end - result.start) as u32, + pos, + }) + } + json::TextOp::Delete { + pos, + len, + start_id: id_start, + } => { + let id_start = convert_id(&id_start, peers)?; + validate_json_id_counter(id_start, "text delete start id")?; + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start, + span: DeleteSpan { + pos: pos as isize, + signed_len: len as isize, + }, + })) + } + json::TextOp::Mark { + start, + end, + style_key, + style_value, + info, + } => InnerContent::List(InnerListOp::StyleStart { + start, + end, + key: style_key.into(), + value: style_value, + info: TextStyleInfoFlag::from_byte(info), + }), + json::TextOp::MarkEnd => InnerContent::List(InnerListOp::StyleEnd), + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for text container".into(), + )) + } + }, + ContainerType::List => match content { + JsonOpContent::List(list) => match list { + json::ListOp::Insert { + pos, + value: mut values, + } => { + for v in values.iter_mut() { + if let LoroValue::Container(id) = v { + if id.is_normal() { + *id = convert_container_id(id.clone(), peers)?; + } + } + } + let range = arena.alloc_values(values.iter().cloned()); + InnerContent::List(InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos: pos as usize, + }) + } + json::ListOp::Delete { pos, len, start_id } => { + let start_id = convert_id(&start_id, peers)?; + validate_json_id_counter(start_id, "list delete start id")?; + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start: start_id, + span: DeleteSpan { + pos: pos as isize, + signed_len: len as isize, + }, + })) + } + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for list container".into(), + )) + } + }, + ContainerType::MovableList => match content { + JsonOpContent::MovableList(list) => match list { + json::MovableListOp::Insert { + pos, + value: mut values, + } => { + for v in values.iter_mut() { + if let LoroValue::Container(id) = v { + if id.is_normal() { + *id = convert_container_id(id.clone(), peers)?; + } + } + } + let range = arena.alloc_values(values.iter().cloned()); + InnerContent::List(InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos: pos as usize, + }) + } + json::MovableListOp::Delete { pos, len, start_id } => { + let start_id = convert_id(&start_id, peers)?; + validate_json_id_counter(start_id, "movable list delete start id")?; + InnerContent::List(InnerListOp::Delete(DeleteSpanWithId { + id_start: start_id, + span: DeleteSpan { + pos: pos as isize, + signed_len: len as isize, + }, + })) + } + json::MovableListOp::Move { + from, + elem_id: from_id, + to, + } => { + let from_id = convert_idlp(&from_id, peers)?; + InnerContent::List(InnerListOp::Move { + from, + elem_id: from_id, + to, + }) + } + json::MovableListOp::Set { elem_id, mut value } => { + let elem_id = convert_idlp(&elem_id, peers)?; + if let LoroValue::Container(id) = &mut value { + *id = convert_container_id(id.clone(), peers)?; + } + InnerContent::List(InnerListOp::Set { elem_id, value }) + } + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for movable list container".into(), + )) + } + }, + ContainerType::Map => match content { + JsonOpContent::Map(map) => match map { + json::MapOp::Insert { key, mut value } => { + if let LoroValue::Container(id) = &mut value { + *id = convert_container_id(id.clone(), peers)?; + } + InnerContent::Map(MapSet { + key: key.into(), + value: Some(value), + }) + } + json::MapOp::Delete { key } => InnerContent::Map(MapSet { + key: key.into(), + value: None, + }), + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for map container".into(), + )) + } + }, + ContainerType::Tree => match content { + JsonOpContent::Tree(tree) => match tree { + json::TreeOp::Create { + target, + parent, + fractional_index, + } => { + let target = convert_tree_id(&target, peers)?; + validate_json_tree_id_counter(&target, "tree create target")?; + let parent = match parent { + Some(p) => { + let parent = convert_tree_id(&p, peers)?; + validate_json_tree_id_counter(&parent, "tree create parent")?; + Some(parent) + } + None => None, + }; + InnerContent::Tree(Arc::new(TreeOp::Create { + target, + parent, + position: fractional_index, + })) + } + json::TreeOp::Move { + target, + parent, + fractional_index, + } => { + let target = convert_tree_id(&target, peers)?; + validate_json_tree_id_counter(&target, "tree move target")?; + let parent = match parent { + Some(p) => { + let parent = convert_tree_id(&p, peers)?; + validate_json_tree_id_counter(&parent, "tree move parent")?; + Some(parent) + } + None => None, + }; + InnerContent::Tree(Arc::new(TreeOp::Move { + target, + parent, + position: fractional_index, + })) + } + json::TreeOp::Delete { target } => { + let target = convert_tree_id(&target, peers)?; + validate_json_tree_id_counter(&target, "tree delete target")?; + InnerContent::Tree(Arc::new(TreeOp::Delete { target })) + } + }, + _ => { + return Err(LoroError::DecodeError( + "invalid op content for tree container".into(), + )) + } + }, + ContainerType::Unknown(_) => match content { + JsonOpContent::Future(json::FutureOpWrapper { + prop, + value: json::FutureOp::Unknown(value), + }) => InnerContent::Future(FutureInnerContent::Unknown { + prop, + value: Box::new(value), + }), + _ => { + return Err(LoroError::DecodeError( + "invalid op content for unknown container".into(), + )) + } + }, + #[cfg(feature = "counter")] + ContainerType::Counter => { + let JsonOpContent::Future(json::FutureOpWrapper { prop: _, value }) = content else { + return Err(LoroError::DecodeError( + "invalid op content for counter container".into(), + )); + }; + use crate::encoding::OwnedValue; + match value { + json::FutureOp::Counter(OwnedValue::F64(c)) + | json::FutureOp::Unknown(OwnedValue::F64(c)) => { + InnerContent::Future(FutureInnerContent::Counter(c)) + } + json::FutureOp::Counter(OwnedValue::I64(c)) + | json::FutureOp::Unknown(OwnedValue::I64(c)) => { + InnerContent::Future(FutureInnerContent::Counter(c as f64)) + } + _ => { + return Err(LoroError::DecodeError( + "invalid counter op value type".into(), + )) + } + } + } // Note: The Future Type need try to parse Op from the unknown content + }; + Ok(Op { + counter, + container: idx, + content, + }) +} + +impl TryFrom<&str> for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: &str) -> Result { + serde_json::from_str(value) + } +} + +impl TryFrom<&String> for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: &String) -> Result { + serde_json::from_str(value) + } +} + +impl TryFrom for JsonSchema { + type Error = serde_json::Error; + + fn try_from(value: String) -> Result { + serde_json::from_str(&value) + } +} + +pub mod json { + use crate::{ + encoding::OwnedValue, + version::{Frontiers, VersionRange}, + }; + use fractional_index::FractionalIndex; + use loro_common::{ContainerID, Counter, IdLp, Lamport, LoroValue, PeerID, TreeID, ID}; + use serde::{Deserialize, Serialize}; + use std::ops::Range; + + use super::{get_peer_from_peers, redact_value}; + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct JsonSchema { + pub schema_version: u8, + #[serde(with = "self::serde_impl::frontiers")] + pub start_version: Frontiers, + #[serde(with = "self::serde_impl::peer_id")] + pub peers: Option>, + pub changes: Vec, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct JsonChange { + #[serde(with = "self::serde_impl::id")] + pub id: ID, + pub timestamp: i64, + #[serde(with = "self::serde_impl::deps")] + pub deps: Vec, + pub lamport: Lamport, + pub msg: Option, + pub ops: Vec, + } + + impl JsonChange { + pub fn checked_op_len(&self) -> Option { + let Some(last_op) = self.ops.last() else { + return Some(0); + }; + if last_op.counter < self.id.counter { + return None; + } + + let counter_delta = last_op.counter.checked_sub(self.id.counter)?; + let counter_delta = usize::try_from(counter_delta).ok()?; + counter_delta.checked_add(last_op.content.op_len()) + } + + pub fn op_len(&self) -> usize { + self.checked_op_len().unwrap_or(0) + } + } + + #[derive(Debug, Clone)] + pub struct JsonOp { + pub content: JsonOpContent, + pub container: ContainerID, + pub counter: i32, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(untagged)] + pub enum JsonOpContent { + List(ListOp), + MovableList(MovableListOp), + Map(MapOp), + Text(TextOp), + Tree(TreeOp), + // #[serde(with = "self::serde_impl::future_op")] + Future(FutureOpWrapper), + } + + impl JsonOpContent { + pub fn op_len(&self) -> usize { + match self { + JsonOpContent::List(list_op) => list_op.op_len(), + JsonOpContent::MovableList(movable_list_op) => movable_list_op.op_len(), + JsonOpContent::Map(..) => 1, + JsonOpContent::Text(text_op) => text_op.op_len(), + JsonOpContent::Tree(..) => 1, + JsonOpContent::Future(..) => 1, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + pub struct FutureOpWrapper { + #[serde(flatten)] + pub value: FutureOp, + pub prop: i32, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum ListOp { + Insert { + pos: u32, + value: Vec, + }, + Delete { + pos: i32, + len: i32, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + } + + impl ListOp { + fn op_len(&self) -> usize { + match self { + ListOp::Insert { value: values, .. } => values.len(), + ListOp::Delete { len, .. } => (*len).unsigned_abs() as usize, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum MovableListOp { + Insert { + pos: u32, + value: Vec, + }, + Delete { + pos: i32, + len: i32, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + Move { + from: u32, + to: u32, + #[serde(with = "self::serde_impl::idlp")] + elem_id: IdLp, + }, + Set { + #[serde(with = "self::serde_impl::idlp")] + elem_id: IdLp, + value: LoroValue, + }, + } + + impl MovableListOp { + fn op_len(&self) -> usize { + match self { + MovableListOp::Insert { value: values, .. } => values.len(), + MovableListOp::Delete { len, .. } => (*len).unsigned_abs() as usize, + MovableListOp::Move { .. } => 1, + MovableListOp::Set { .. } => 1, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum MapOp { + Insert { key: String, value: LoroValue }, + Delete { key: String }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum TextOp { + Insert { + pos: u32, + text: String, + }, + Delete { + pos: i32, + len: i32, + #[serde(with = "self::serde_impl::id")] + start_id: ID, + }, + Mark { + start: u32, + end: u32, + style_key: String, + style_value: LoroValue, + info: u8, + }, + MarkEnd, + } + + impl TextOp { + fn op_len(&self) -> usize { + match self { + TextOp::Insert { text, .. } => text.chars().count(), + TextOp::Delete { len, .. } => len.unsigned_abs() as usize, + TextOp::Mark { .. } => 1, + TextOp::MarkEnd => 1, + } + } + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum TreeOp { + Create { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + #[serde(with = "self::serde_impl::option_tree_id")] + parent: Option, + #[serde(default, with = "self::serde_impl::fractional_index")] + fractional_index: FractionalIndex, + }, + Move { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + #[serde(with = "self::serde_impl::option_tree_id")] + parent: Option, + #[serde(default, with = "self::serde_impl::fractional_index")] + fractional_index: FractionalIndex, + }, + Delete { + #[serde(with = "self::serde_impl::tree_id")] + target: TreeID, + }, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(tag = "type", rename_all = "snake_case")] + pub enum FutureOp { + #[cfg(feature = "counter")] + Counter(OwnedValue), + Unknown(OwnedValue), + } + + mod serde_impl { + + use loro_common::{ContainerID, ContainerType}; + use serde::{ + de::{Error, IgnoredAny, MapAccess, Visitor}, + ser::SerializeStruct, + Deserialize, Deserializer, Serialize, Serializer, + }; + + #[allow(unused_imports)] + use crate::encoding::OwnedValue; + + impl Serialize for super::JsonOp { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut s = serializer.serialize_struct("Op", 3)?; + s.serialize_field("container", &self.container.to_string())?; + s.serialize_field("content", &self.content)?; + s.serialize_field("counter", &self.counter)?; + s.end() + } + } + + impl<'de> Deserialize<'de> for super::JsonOp { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct __Visitor; + + impl<'de> Visitor<'de> for __Visitor { + type Value = super::JsonOp; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("struct Op") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut container = None; + let mut content = None; + let mut counter = None; + + while let Some(key) = map.next_key::()? { + match key.as_str() { + "container" => { + if container.is_some() { + return Err(A::Error::duplicate_field("container")); + } + let value = map.next_value::()?; + container = + Some(ContainerID::try_from(value.as_str()).map_err( + |_| A::Error::custom("invalid container id"), + )?); + } + "content" => { + if content.is_some() { + return Err(A::Error::duplicate_field("content")); + } + content = Some(map.next_value::()?); + } + "counter" => { + if counter.is_some() { + return Err(A::Error::duplicate_field("counter")); + } + counter = Some(map.next_value::()?); + } + _ => { + let _ = map.next_value::()?; + } + } + } + + let container = + container.ok_or_else(|| A::Error::missing_field("container"))?; + let content = content.ok_or_else(|| A::Error::missing_field("content"))?; + let op = json_op_content_from_value(&container, content)?; + let counter = counter.ok_or_else(|| A::Error::missing_field("counter"))?; + Ok(super::JsonOp { + container, + content: op, + counter, + }) + } + } + const FIELDS: &[&str] = &["container", "content", "counter"]; + deserializer.deserialize_struct("JsonOp", FIELDS, __Visitor) + } + } + + fn json_op_content_from_value( + container: &ContainerID, + value: serde_json::Value, + ) -> Result { + match container.container_type() { + ContainerType::List => serde_json::from_value(value) + .map(super::JsonOpContent::List) + .map_err(E::custom), + ContainerType::MovableList => serde_json::from_value(value) + .map(super::JsonOpContent::MovableList) + .map_err(E::custom), + ContainerType::Map => serde_json::from_value(value) + .map(super::JsonOpContent::Map) + .map_err(E::custom), + ContainerType::Text => serde_json::from_value(value) + .map(super::JsonOpContent::Text) + .map_err(E::custom), + ContainerType::Tree => serde_json::from_value(value) + .map(super::JsonOpContent::Tree) + .map_err(E::custom), + ContainerType::Unknown(_) => serde_json::from_value(value) + .map(super::JsonOpContent::Future) + .map_err(E::custom), + #[cfg(feature = "counter")] + ContainerType::Counter => { + match serde_json::from_value::(value.clone()) { + Ok(op) => Ok(super::JsonOpContent::Future(op)), + Err(_) => { + let value = + serde_json::from_value::(value).map_err(E::custom)?; + Ok(super::JsonOpContent::Future(super::FutureOpWrapper { + prop: 0, + value: super::FutureOp::Counter(value), + })) + } + } + } + } + } + + pub mod id { + use loro_common::ID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &ID, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&id.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + // NOTE: https://github.com/serde-rs/serde/issues/2467 we use String here + let str: String = Deserialize::deserialize(d)?; + let id: ID = ID::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid ID format"))?; + Ok(id) + } + } + + pub mod frontiers { + use itertools::Itertools; + use loro_common::ID; + use serde::{ser::SerializeMap, Deserializer, Serializer}; + + use crate::version::Frontiers; + + pub fn serialize(f: &Frontiers, s: S) -> Result + where + S: Serializer, + { + let mut map = s.serialize_map(Some(f.len()))?; + for id in f + .iter() + // Make sure the order is deterministic + .sorted() + { + map.serialize_entry(&id.peer.to_string(), &id.counter)?; + } + map.end() + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + struct __Visitor; + impl<'de> serde::de::Visitor<'de> for __Visitor { + type Value = Frontiers; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a Frontiers") + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let mut f = Frontiers::default(); + while let Some((k, v)) = map.next_entry::()? { + f.push(ID::new( + k.parse().map_err(|_| { + serde::de::Error::custom("invalid peer id in frontiers") + })?, + v, + )) + } + Ok(f) + } + } + d.deserialize_map(__Visitor) + } + } + + pub mod deps { + use loro_common::ID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(deps: &[ID], s: S) -> Result + where + S: Serializer, + { + s.collect_seq(deps.iter().map(|x| x.to_string())) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let deps: Vec = Deserialize::deserialize(d)?; + deps.into_iter() + .map(|x| { + ID::try_from(x.as_str()) + .map_err(|_| serde::de::Error::custom("invalid ID in deps")) + }) + .collect::, _>>() + } + } + + pub mod peer_id { + use loro_common::PeerID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(peers: &Option>, s: S) -> Result + where + S: Serializer, + { + match peers { + Some(peers) => s.collect_seq(peers.iter().map(|x| x.to_string())), + None => s.serialize_none(), + } + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + let peers: Option> = Deserialize::deserialize(d)?; + peers + .map(|x| { + x.into_iter() + .map(|x| { + x.parse() + .map_err(|_| serde::de::Error::custom("invalid peer id")) + }) + .collect::, _>>() + }) + .transpose() + } + } + + pub mod idlp { + use loro_common::IdLp; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(idlp: &IdLp, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&idlp.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let id: IdLp = IdLp::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid IdLp format"))?; + Ok(id) + } + } + + pub mod tree_id { + use loro_common::TreeID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &TreeID, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&id.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + let id: TreeID = TreeID::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid TreeID format"))?; + Ok(id) + } + } + + pub mod option_tree_id { + use loro_common::TreeID; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(id: &Option, s: S) -> Result + where + S: Serializer, + { + match id { + Some(id) => s.serialize_str(&id.to_string()), + None => s.serialize_none(), + } + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let str: Option = Deserialize::deserialize(d)?; + match str { + Some(str) => { + let id: TreeID = TreeID::try_from(str.as_str()) + .map_err(|_| serde::de::Error::custom("invalid TreeID format"))?; + Ok(Some(id)) + } + None => Ok(None), + } + } + } + + pub mod fractional_index { + use fractional_index::FractionalIndex; + use serde::{Deserialize, Deserializer, Serializer}; + + pub fn serialize(fi: &FractionalIndex, s: S) -> Result + where + S: Serializer, + { + s.serialize_str(&fi.to_string()) + } + + pub fn deserialize<'de, 'a, D>(d: D) -> Result + where + D: Deserializer<'de>, + { + let str: String = Deserialize::deserialize(d)?; + if !str.len().is_multiple_of(2) { + return Err(serde::de::Error::custom( + "invalid fractional index hex length", + )); + } + + let mut bytes = Vec::with_capacity(str.len() / 2); + for i in 0..str.len() / 2 { + let byte = u8::from_str_radix(&str[i * 2..i * 2 + 2], 16).map_err(|e| { + serde::de::Error::custom(format!("invalid fractional index hex: {e}")) + })?; + bytes.push(byte); + } + + Ok(FractionalIndex::from_bytes(bytes)) + } + } + } + + #[derive(thiserror::Error, Debug, PartialEq, Eq)] + pub enum RedactError { + #[error("unknown operation type")] + UnknownOperationType, + #[error("invalid schema: {0}")] + InvalidSchema(String), + } + + fn invalid_schema(message: impl Into) -> RedactError { + RedactError::InvalidSchema(message.into()) + } + + fn checked_redact_op_len(op: &JsonOp) -> Result { + let len = op.content.op_len(); + if len == 0 { + return Err(invalid_schema("op length must be greater than zero")); + } + + Counter::try_from(len).map_err(|_| invalid_schema("op length is too large")) + } + + fn validate_redactable_change(change: &JsonChange) -> Result { + if change.id.counter < 0 { + return Err(invalid_schema("change id counter must be non-negative")); + } + + let mut expected_counter = change.id.counter; + for op in &change.ops { + if op.counter < 0 { + return Err(invalid_schema("op counter must be non-negative")); + } + if op.counter != expected_counter { + return Err(invalid_schema( + "op counters must be contiguous with the change id", + )); + } + + let len = checked_redact_op_len(op)?; + expected_counter = expected_counter + .checked_add(len) + .ok_or_else(|| invalid_schema("op counter overflow"))?; + } + + let len = expected_counter + .checked_sub(change.id.counter) + .ok_or_else(|| invalid_schema("change counter overflow"))?; + usize::try_from(len).map_err(|_| invalid_schema("change length is too large")) + } + + /// Redacts sensitive content within the specified range by replacing it with default values. + /// + /// This method applies the following redaction rules: + /// + /// - Preserves delete and move operations without changes + /// - Replaces text insertion content with the Unicode replacement character (U+FFFD) + /// - Substitutes list and map insert values with `LoroValue::Null` + /// - Maintains child container creation operations + /// - Replaces text mark values with `LoroValue::Null` + /// - Preserves map insertion and text annotation keys + /// - Resets counter operations to zero + /// - Leaves unknown operation types (from future Loro versions) unchanged + /// + /// This approach ensures sensitive data removal while preserving the document's overall + /// structure. Redacted documents maintain seamless collaboration capabilities with both + /// redacted and non-redacted versions. + pub fn redact(json: &mut JsonSchema, range: VersionRange) -> Result<(), RedactError> { + let peers = json.peers.clone(); + let mut errors = Vec::new(); + for change in json.changes.iter_mut() { + let real_peer = get_peer_from_peers(&peers, change.id.peer) + .map_err(|_| RedactError::InvalidSchema("peer index out of range".to_string()))?; + let real_id = ID::new(real_peer, change.id.counter); + let change_len = validate_redactable_change(change)?; + if !range.has_overlap_with(real_id.to_span(change_len)) { + continue; + } + + let redact_range = range.get(&real_peer).copied().unwrap(); + for op in change.ops.iter_mut() { + if op.counter >= redact_range.1 { + break; + } + + let len = checked_redact_op_len(op)?; + let op_end = op + .counter + .checked_add(len) + .ok_or_else(|| invalid_schema("op counter overflow"))?; + if op_end <= redact_range.0 { + continue; + } + + let result = redact_op( + &mut op.content, + redact_range.0.saturating_sub(op.counter).max(0).min(len) + ..redact_range.1.saturating_sub(op.counter).max(0).min(len), + ); + match result { + Ok(()) => {} + Err(e) => errors.push(e), + } + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors.pop().unwrap()) + } + } + + fn redact_op(op: &mut JsonOpContent, range: Range) -> Result<(), RedactError> { + match op { + JsonOpContent::List(list_op) => { + match list_op { + ListOp::Insert { value: values, .. } => { + for i in range { + redact_value(&mut values[i as usize]); + } + } + ListOp::Delete { .. } => { + // Delete op won't be changed + } + } + } + JsonOpContent::MovableList(movable_list_op) => { + match movable_list_op { + MovableListOp::Insert { value: values, .. } => { + for i in range { + redact_value(&mut values[i as usize]); + } + } + MovableListOp::Delete { .. } | MovableListOp::Move { .. } => { + // Delete and move ops won't be changed + } + MovableListOp::Set { value, .. } => { + assert!(range.start == 0 && range.len() == 1); + redact_value(value); + } + } + } + JsonOpContent::Map(map_op) => { + match map_op { + MapOp::Insert { value, .. } => { + assert!(range.start == 0 && range.len() == 1); + redact_value(value); + } + MapOp::Delete { .. } => { + // Delete op won't be changed + } + } + } + JsonOpContent::Text(text_op) => { + match text_op { + TextOp::Insert { text, .. } => { + let mut chars = vec![]; + for (i, c) in text.chars().enumerate() { + if i < range.start as usize || i >= range.end as usize { + chars.push(c); + } else { + chars.push("� ".chars().next().unwrap()); + } + } + *text = chars.into_iter().collect(); + } + TextOp::Delete { .. } => { + // Delete op won't be changed + } + TextOp::Mark { style_value, .. } => { + assert!(range.start == 0 && range.len() == 1); + *style_value = LoroValue::Null; + } + TextOp::MarkEnd => { + // MarkEnd won't be changed + } + } + } + JsonOpContent::Tree(..) => { + // Creation of child container won't be changed + } + JsonOpContent::Future(future_op_wrapper) => match &mut future_op_wrapper.value { + #[cfg(feature = "counter")] + FutureOp::Counter(owned_value) => { + *owned_value = OwnedValue::I64(0); + } + FutureOp::Unknown(..) => { + return Err(RedactError::UnknownOperationType); + } + }, + } + + Ok(()) + } +} + +fn redact_value(v: &mut LoroValue) { + match v { + LoroValue::Container(_) => {} + _ => *v = LoroValue::Null, + } +} + +#[cfg(test)] +mod tests { + use crate::{LoroDoc, VersionVector}; + + #[test] + fn json_range_version() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let list = doc.get_list("list"); + list.insert(0, "a").unwrap(); + list.insert(0, "b").unwrap(); + list.insert(0, "c").unwrap(); + let json = doc.export_json_updates( + &VersionVector::from_iter(vec![(0, 1)]), + &VersionVector::from_iter(vec![(0, 2)]), + true, + ); + assert_eq!(json.changes[0].ops.len(), 1); + let json = doc.export_json_updates( + &VersionVector::from_iter(vec![(0, 0)]), + &VersionVector::from_iter(vec![(0, 2)]), + true, + ); + assert_eq!(json.changes[0].ops.len(), 2); + } +} diff --git a/vendor/loro-internal/src/encoding/outdated_encode_reordered.rs b/vendor/loro-internal/src/encoding/outdated_encode_reordered.rs new file mode 100644 index 00000000..e2f2c504 --- /dev/null +++ b/vendor/loro-internal/src/encoding/outdated_encode_reordered.rs @@ -0,0 +1,575 @@ +pub(crate) use encode::{encode_op, get_op_prop}; +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, ContainerType, Counter, HasCounterSpan, HasIdSpan, IdLp, LoroError, LoroResult, + TreeID, ID, +}; +use serde_columnar::{columnar, ColumnarError}; +use std::sync::Arc; + +use crate::version::VersionRange; +use crate::{ + arena::SharedArena, + change::{Change, Lamport}, + container::{ + list::list_op::DeleteSpanWithId, richtext::TextStyleInfoFlag, tree::tree_op::TreeOp, + }, + op::{FutureInnerContent, SliceRange}, + OpLog, +}; + +use super::value::{Value, ValueDecodedArenasTrait}; + +pub(crate) use crate::encoding::value_register::ValueRegister; + +#[allow(unused_imports)] +use super::value::FutureValue; + +/// If any collection in the document is longer than this, we will not decode it. +/// It will return an data corruption error instead. +pub(super) const MAX_COLLECTION_SIZE: usize = 1 << 28; + +pub(crate) struct ImportChangesResult { + pub latest_ids: Vec, + pub pending_changes: Vec, + pub changes_that_have_deps_before_shallow_root: Vec, + pub imported: VersionRange, +} + +/// NOTE: This method expects that the remote_changes are already sorted by lamport value +pub(crate) fn import_changes_to_oplog( + changes: Vec, + oplog: &mut OpLog, +) -> ImportChangesResult { + let mut pending_changes = Vec::new(); + let mut latest_ids = Vec::new(); + let mut changes_before_shallow_root = Vec::new(); + let mut imported = VersionRange::default(); + for mut change in changes { + if change.ctr_end() <= oplog.vv().get(&change.id.peer).copied().unwrap_or(0) { + // skip included changes + continue; + } + + if oplog.dag.is_before_shallow_root(&change.deps) { + changes_before_shallow_root.push(change); + continue; + } + + latest_ids.push(change.id_last()); + // calc lamport or pending if its deps are not satisfied + match oplog.dag.get_change_lamport_from_deps(&change.deps) { + Some(lamport) => change.lamport = lamport, + None => { + pending_changes.push(change); + continue; + } + } + + let Some(change) = oplog.trim_the_known_part_of_change(change) else { + continue; + }; + + imported.extends_to_include_id_span(change.id_span()); + oplog.insert_new_change(change, false); + } + + ImportChangesResult { + latest_ids, + pending_changes, + changes_that_have_deps_before_shallow_root: changes_before_shallow_root, + imported, + } +} + +mod encode { + use loro_common::ContainerType; + + use crate::{ + arena::SharedArena, + encoding::value::{MarkStart, Value, ValueEncodeRegister, ValueKind, ValueWriter}, + op::{FutureInnerContent, Op}, + }; + + use super::EncodedDeleteStartId; + + fn get_future_op_prop(op: &FutureInnerContent) -> i32 { + match op { + #[cfg(feature = "counter")] + FutureInnerContent::Counter(_) => 0, + FutureInnerContent::Unknown { prop, .. } => *prop, + } + } + + pub(crate) fn get_op_prop(op: &Op, registers: &mut dyn ValueEncodeRegister) -> i32 { + match &op.content { + crate::op::InnerContent::List(list) => match list { + crate::container::list::list_op::InnerListOp::Move { to, .. } => *to as i32, + crate::container::list::list_op::InnerListOp::Set { .. } => 0, + crate::container::list::list_op::InnerListOp::Insert { pos, .. } => *pos as i32, + crate::container::list::list_op::InnerListOp::InsertText { pos, .. } => *pos as i32, + crate::container::list::list_op::InnerListOp::Delete(span) => span.span.pos as i32, + crate::container::list::list_op::InnerListOp::StyleStart { start, .. } => { + *start as i32 + } + crate::container::list::list_op::InnerListOp::StyleEnd => 0, + }, + crate::op::InnerContent::Map(map) => { + let key = registers.key_mut().register(&map.key); + key as i32 + } + crate::op::InnerContent::Tree(_) => 0, + crate::op::InnerContent::Future(f) => get_future_op_prop(f), + } + } + + #[inline] + pub(crate) fn encode_op<'p, 'a: 'p>( + op: &'a Op, + arena: &SharedArena, + delete_start: &mut Vec, + value_writer: &mut ValueWriter, + registers: &mut dyn ValueEncodeRegister, + ) -> ValueKind { + let value = match &op.content { + crate::op::InnerContent::List(list) => match list { + crate::container::list::list_op::InnerListOp::Insert { slice, .. } => { + assert!(matches!( + op.container.get_type(), + ContainerType::List | ContainerType::MovableList + )); + let value = arena.get_values(slice.0.start as usize..slice.0.end as usize); + Value::LoroValue(value.into()) + } + crate::container::list::list_op::InnerListOp::InsertText { slice, .. } => { + Value::Str(std::str::from_utf8(slice.as_bytes()).unwrap()) + } + crate::container::list::list_op::InnerListOp::Delete(span) => { + delete_start.push(EncodedDeleteStartId { + peer_idx: registers.peer_mut().register(&span.id_start.peer), + counter: span.id_start.counter, + len: span.span.signed_len, + }); + Value::DeleteSeq + } + crate::container::list::list_op::InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => Value::MarkStart(MarkStart { + len: *end - *start, + key: key.clone(), + value: value.clone(), + info: info.to_byte(), + }), + crate::container::list::list_op::InnerListOp::Set { elem_id, value } => { + Value::ListSet { + peer_idx: registers.peer_mut().register(&elem_id.peer), + lamport: elem_id.lamport, + value: value.clone(), + } + } + crate::container::list::list_op::InnerListOp::StyleEnd => Value::Null, + crate::container::list::list_op::InnerListOp::Move { + from, + elem_id: from_id, + to: _, + } => Value::ListMove { + from: *from as usize, + from_idx: registers.peer_mut().register(&from_id.peer), + lamport: from_id.lamport as usize, + }, + }, + crate::op::InnerContent::Map(map) => { + assert_eq!(op.container.get_type(), ContainerType::Map); + match &map.value { + Some(v) => Value::LoroValue(v.clone()), + None => Value::DeleteOnce, + } + } + crate::op::InnerContent::Tree(t) => { + assert_eq!(op.container.get_type(), ContainerType::Tree); + registers.encode_tree_op(t) + } + crate::op::InnerContent::Future(f) => match f { + #[cfg(feature = "counter")] + FutureInnerContent::Counter(c) => { + let c_abs = c.abs(); + if c_abs.fract() < f64::EPSILON && (c_abs as i64) < (2 << 26) { + Value::I64(*c as i64) + } else { + Value::F64(*c) + } + } + FutureInnerContent::Unknown { value, .. } => Value::from_owned(value), + }, + }; + let (k, _) = value.encode(value_writer, registers); + k + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn decode_op( + cid: &ContainerID, + value: Value<'_>, + del_iter: &mut impl Iterator>, + shared_arena: &SharedArena, + arenas: &dyn ValueDecodedArenasTrait, + positions: &[Vec], + prop: i32, + op_id: ID, +) -> LoroResult { + let content = match cid.container_type() { + ContainerType::Text => match value { + Value::Str(s) => { + let (slice, result) = shared_arena.alloc_str_with_slice(s); + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::InsertText { + slice, + unicode_start: result.start as u32, + unicode_len: (result.end - result.start) as u32, + pos: prop as u32, + }, + ) + } + Value::DeleteSeq => { + let del_start = del_iter + .next() + .ok_or(LoroError::DecodeDataCorruptionError)??; + let peer_idx = del_start.peer_idx; + let cnt = del_start.counter; + let len = del_start.len; + crate::op::InnerContent::List(crate::container::list::list_op::InnerListOp::Delete( + DeleteSpanWithId::new( + ID::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + cnt as Counter, + ), + prop as isize, + len, + ), + )) + } + Value::MarkStart(mark) => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::StyleStart { + start: prop as u32, + end: prop as u32 + mark.len, + key: mark.key, + value: mark.value, + info: TextStyleInfoFlag::from_byte(mark.info), + }, + ), + Value::Null => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::StyleEnd, + ), + _ => return Err(LoroError::DecodeDataCorruptionError), + }, + ContainerType::Map => { + let key = arenas + .keys() + .get(prop as usize) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(); + match value { + Value::DeleteOnce => { + crate::op::InnerContent::Map(crate::container::map::MapSet { key, value: None }) + } + Value::LoroValue(v) => { + crate::op::InnerContent::Map(crate::container::map::MapSet { + key, + value: Some(v.clone()), + }) + } + _ => return Err(LoroError::DecodeDataCorruptionError), + } + } + ContainerType::List => { + let pos = prop as usize; + match value { + Value::LoroValue(arr) => { + let list = arr + .into_list() + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let range = shared_arena.alloc_values(list.iter().cloned()); + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos, + }, + ) + } + Value::DeleteSeq => { + let del_start = del_iter + .next() + .ok_or(LoroError::DecodeDataCorruptionError)??; + let peer_idx = del_start.peer_idx; + let cnt = del_start.counter; + let len = del_start.len; + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Delete( + DeleteSpanWithId::new( + ID::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + cnt as Counter, + ), + pos as isize, + len, + ), + ), + ) + } + _ => return Err(LoroError::DecodeDataCorruptionError), + } + } + ContainerType::Tree => match value { + Value::TreeMove(op) => crate::op::InnerContent::Tree(Arc::new( + arenas.decode_tree_op(positions, op, op_id)?, + )), + Value::RawTreeMove(op) => { + let subject = TreeID::new( + *arenas + .peers() + .get(op.subject_peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + op.subject_cnt as Counter, + ); + let parent = if op.is_parent_null { + None + } else { + let parent_id = TreeID::new( + *arenas + .peers() + .get(op.parent_peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + op.parent_cnt as Counter, + ); + if parent_id.is_deleted_root() { + return Ok(crate::op::InnerContent::Tree(Arc::new(TreeOp::Delete { + target: subject, + }))); + } + + Some(parent_id) + }; + + let fi = FractionalIndex::from_bytes( + positions + .get(op.position_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + ); + let is_create = subject.id() == op_id; + let ans = if is_create { + TreeOp::Create { + target: subject, + parent, + position: fi, + } + } else { + TreeOp::Move { + target: subject, + parent, + position: fi, + } + }; + crate::op::InnerContent::Tree(Arc::new(ans)) + } + _ => return Err(LoroError::DecodeDataCorruptionError), + }, + ContainerType::MovableList => { + let pos = prop as usize; + match value { + Value::LoroValue(arr) => { + let list = arr + .into_list() + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let range = shared_arena.alloc_values(list.iter().cloned()); + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Insert { + slice: SliceRange::new(range.start as u32..range.end as u32), + pos, + }, + ) + } + Value::DeleteSeq => { + let del_start = del_iter + .next() + .ok_or(LoroError::DecodeDataCorruptionError)??; + let peer_idx = del_start.peer_idx; + let cnt = del_start.counter; + let len = del_start.len; + crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Delete( + DeleteSpanWithId::new( + ID::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + cnt as Counter, + ), + pos as isize, + len, + ), + ), + ) + } + Value::ListMove { + from, + from_idx, + lamport, + } => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Move { + from: from as u32, + elem_id: IdLp::new( + *arenas + .peers() + .get(from_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + lamport as Lamport, + ), + to: prop as u32, + }, + ), + Value::ListSet { + peer_idx, + lamport, + value, + } => crate::op::InnerContent::List( + crate::container::list::list_op::InnerListOp::Set { + elem_id: IdLp::new( + *arenas + .peers() + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?, + lamport as Lamport, + ), + value, + }, + ), + _ => return Err(LoroError::DecodeDataCorruptionError), + } + } + #[cfg(feature = "counter")] + ContainerType::Counter => match value { + Value::F64(c) => crate::op::InnerContent::Future(FutureInnerContent::Counter(c)), + Value::I64(c) => crate::op::InnerContent::Future(FutureInnerContent::Counter(c as f64)), + _ => return Err(LoroError::DecodeDataCorruptionError), + }, + // NOTE: The future container type need also try to parse the unknown type + ContainerType::Unknown(_) => crate::op::InnerContent::Future(FutureInnerContent::Unknown { + prop, + value: Box::new(value.into_owned()), + }), + }; + + Ok(content) +} + +pub type PeerIdx = usize; + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +pub(crate) struct EncodedDeleteStartId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + len: isize, +} + +#[cfg(test)] +mod test { + + use loro_common::LoroValue; + use rustc_hash::FxHashSet; + + use crate::{ + encoding::{ + arena::EncodedRegisters, + value::{ValueReader, ValueWriter}, + value_register::ValueRegister, + }, + fx_map, + }; + + use super::*; + + fn test_loro_value_read_write(v: impl Into, container_id: Option) { + let v = v.into(); + let id = match &container_id { + Some(ContainerID::Root { .. }) => ID::new(u64::MAX, 0), + Some(ContainerID::Normal { peer, counter, .. }) => ID::new(*peer, *counter), + None => ID::new(u64::MAX, 0), + }; + + let mut registers = EncodedRegisters { + key: ValueRegister::new(), + peer: ValueRegister::new(), + tree_id: ValueRegister::new(), + position: either::Either::Left(FxHashSet::default()), + }; + + let mut writer = ValueWriter::new(); + let (kind, _) = writer.write_value_content(&v, &mut registers); + + let binding = writer.finish(); + let mut reader = ValueReader::new(binding.as_slice()); + + let ans = reader + .read_value_content(kind, ®isters.key.unwrap_vec(), id) + .unwrap(); + assert_eq!(v, ans) + } + + #[test] + fn test_value_read_write() { + test_loro_value_read_write(true, None); + test_loro_value_read_write(false, None); + test_loro_value_read_write(123, None); + test_loro_value_read_write(1.23, None); + test_loro_value_read_write(LoroValue::Null, None); + test_loro_value_read_write( + LoroValue::Binary((vec![123, 223, 255, 0, 1, 2, 3]).into()), + None, + ); + test_loro_value_read_write("sldk;ajfas;dlkfas测试", None); + // we won't encode root container by `value content` + // test_loro_value_read_write( + // LoroValue::Container(ContainerID::new_root("name", ContainerType::Text)), + // Some(ContainerID::new_root("name", ContainerType::Text)), + // ); + test_loro_value_read_write( + LoroValue::Container(ContainerID::new_normal( + ID::new(u64::MAX, 123), + ContainerType::Tree, + )), + Some(ContainerID::new_normal( + ID::new(u64::MAX, 123), + ContainerType::Tree, + )), + ); + test_loro_value_read_write(vec![1i32, 2, 3], None); + test_loro_value_read_write( + LoroValue::Map( + (fx_map![ + "1".into() => 123.into(), + "2".into() => "123".into(), + "3".into() => vec![true].into() + ]) + .into(), + ), + None, + ); + } +} diff --git a/vendor/loro-internal/src/encoding/shallow_snapshot.rs b/vendor/loro-internal/src/encoding/shallow_snapshot.rs new file mode 100644 index 00000000..fcdc4f55 --- /dev/null +++ b/vendor/loro-internal/src/encoding/shallow_snapshot.rs @@ -0,0 +1,422 @@ +use bytes::Bytes; +use rle::HasLength; +use std::collections::BTreeSet; + +use loro_common::{ContainerID, ContainerType, LoroEncodeError, LoroError, ID}; + +use crate::{ + container::list::list_op::InnerListOp, + dag::DagUtils, + encoding::fast_snapshot::{_encode_snapshot, Snapshot}, + state::container_store::FRONTIERS_KEY, + version::{Frontiers, VersionVector}, + LoroDoc, +}; + +#[cfg(test)] +const MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE: usize = 16; +#[cfg(not(test))] +const MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE: usize = 256; + +#[tracing::instrument(skip_all)] +pub(crate) fn export_shallow_snapshot( + doc: &LoroDoc, + start_from: &Frontiers, + w: &mut W, +) -> Result { + let (snapshot, start_from) = export_shallow_snapshot_inner(doc, start_from)?; + _encode_snapshot(snapshot, w); + Ok(start_from) +} + +pub(crate) fn export_shallow_snapshot_inner( + doc: &LoroDoc, + start_from: &Frontiers, +) -> Result<(Snapshot, Frontiers), LoroEncodeError> { + let oplog = doc.oplog().lock(); + let start_from = calc_shallow_doc_start(&oplog, start_from); + let mut start_vv = frontiers_to_vv_for_export(&oplog, &start_from, "export_shallow_snapshot")?; + for id in start_from.iter() { + // we need to include the ops in start_from, this can make things easier + start_vv.insert(id.peer, id.counter); + } + + #[cfg(debug_assertions)] + { + use crate::dag::Dag; + if !start_from.is_empty() { + assert!(start_from.len() == 1); + let id = start_from.as_single().unwrap(); + let node = oplog.dag.get(id).unwrap(); + if id.counter == node.cnt { + let vv = oplog.dag().frontiers_to_vv(&node.deps).unwrap(); + assert_eq!(vv, start_vv); + } else { + let vv = oplog + .dag() + .frontiers_to_vv(&Frontiers::from(id.inc(-1))) + .unwrap(); + assert_eq!(vv, start_vv); + } + } + } + + loro_common::debug!( + "start version vv={:?} frontiers={:?}", + &start_vv, + &start_from, + ); + + let latest_frontiers = oplog.frontiers().clone(); + let state_frontiers = doc.state_frontiers(); + let is_attached = !doc.is_detached(); + let oplog_bytes = oplog.export_change_store_from(&start_vv, &start_from); + let latest_vv = oplog.vv(); + let ops_num: usize = latest_vv.sub_iter(&start_vv).map(|x| x.atom_len()).sum(); + if &start_from == oplog.shallow_since_frontiers() + && state_frontiers == latest_frontiers + && ops_num <= MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE + { + let state = doc.app_state().lock(); + if let Some(shallow_root_state_bytes) = state.store.encode_shallow_root_state() { + return Ok(( + Snapshot { + oplog_bytes, + state_bytes: None, + shallow_root_state_bytes, + }, + start_from, + )); + } + } + if &start_from == oplog.shallow_since_frontiers() + && state_frontiers == latest_frontiers + && ops_num > MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE + { + let mut state = doc.app_state().lock(); + if let Some((shallow_root_state_bytes, shallow_root_state_kv)) = + state.store.shallow_root_state_for_export() + { + let mut alive_c_bytes = shallow_root_state_kv.keys(); + if has_unknown_container_key(alive_c_bytes.iter()) { + return Err(LoroEncodeError::UnknownContainer); + } + + state.ensure_all_alive_containers(); + state.store.flush(); + + // All the containers that are created after start_from need to be encoded. + for cid in state.store.iter_all_container_ids() { + if let ContainerID::Normal { peer, counter, .. } = cid { + let temp_id = ID::new(peer, counter); + if !start_from.contains(&temp_id) { + alive_c_bytes.insert(cid.to_bytes()); + } + } else { + alive_c_bytes.insert(cid.to_bytes()); + } + } + + let new_kv = state.store.get_kv_clone(); + new_kv.remove_same(&shallow_root_state_kv); + new_kv.retain_keys(&alive_c_bytes); + + return Ok(( + Snapshot { + oplog_bytes, + state_bytes: Some(new_kv.export()), + shallow_root_state_bytes, + }, + start_from, + )); + } + } + drop(oplog); + let result = (|| -> Result { + doc._checkout_without_emitting(&start_from, false, false) + .map_err(LoroEncodeError::from)?; + let mut state = doc.app_state().lock(); + let alive_containers = state.ensure_all_alive_containers(); + if has_unknown_container(alive_containers.iter()) { + return Err(LoroEncodeError::UnknownContainer); + } + let mut alive_c_bytes: BTreeSet> = + alive_containers.iter().map(|x| x.to_bytes()).collect(); + state.store.flush(); + let shallow_root_state_kv = state.store.get_kv_clone(); + drop(state); + doc._checkout_without_emitting(&latest_frontiers, false, false) + .map_err(LoroEncodeError::from)?; + let state_bytes = if ops_num > MAX_OPS_NUM_TO_ENCODE_WITHOUT_LATEST_STATE { + let mut state = doc.app_state().lock(); + state.ensure_all_alive_containers(); + state.store.encode(); + // All the containers that are created after start_from need to be encoded + for cid in state.store.iter_all_container_ids() { + if let ContainerID::Normal { peer, counter, .. } = cid { + let temp_id = ID::new(peer, counter); + if !start_from.contains(&temp_id) { + alive_c_bytes.insert(cid.to_bytes()); + } + } else { + alive_c_bytes.insert(cid.to_bytes()); + } + } + + let new_kv = state.store.get_kv_clone(); + new_kv.remove_same(&shallow_root_state_kv); + new_kv.retain_keys(&alive_c_bytes); + Some(new_kv.export()) + } else { + None + }; + + shallow_root_state_kv.retain_keys(&alive_c_bytes); + shallow_root_state_kv.insert(FRONTIERS_KEY, start_from.encode().into()); + let shallow_root_state_bytes = shallow_root_state_kv.export(); + + Ok(Snapshot { + oplog_bytes, + state_bytes, + shallow_root_state_bytes, + }) + })(); + + restore_export_doc_state(doc, &state_frontiers, is_attached)?; + doc.drop_pending_events(); + Ok((result?, start_from)) +} + +fn has_unknown_container<'a>(mut cids: impl Iterator) -> bool { + cids.any(|cid| matches!(cid.container_type(), ContainerType::Unknown(_))) +} + +fn has_unknown_container_key<'a>(mut keys: impl Iterator>) -> bool { + keys.any(|key| ContainerID::from_bytes(key).is_unknown()) +} + +pub(crate) fn export_state_only_snapshot( + doc: &LoroDoc, + target_frontiers: &Frontiers, + w: &mut W, +) -> Result { + let oplog = doc.oplog().lock(); + let start_from = calc_shallow_doc_start(&oplog, target_frontiers); + let mut start_vv = + frontiers_to_vv_for_export(&oplog, &start_from, "export_state_only_snapshot")?; + for id in start_from.iter() { + // we need to include the ops in start_from, this can make things easier + start_vv.insert(id.peer, id.counter); + } + + loro_common::debug!( + "start version vv={:?} frontiers={:?}", + &start_vv, + &start_from, + ); + + let to_vv = frontiers_to_vv_for_export(&oplog, target_frontiers, "export_state_only_snapshot")?; + + let oplog_bytes = + oplog.export_change_store_in_range(&start_vv, &start_from, &to_vv, target_frontiers); + let state_frontiers = doc.state_frontiers(); + let is_attached = !doc.is_detached(); + drop(oplog); + let result = (|| -> Result<(), LoroEncodeError> { + doc._checkout_without_emitting(&start_from, false, false) + .map_err(LoroEncodeError::from)?; + let mut state = doc.app_state().lock(); + let alive_containers = state.ensure_all_alive_containers(); + let alive_c_bytes = cids_to_bytes(alive_containers); + state.store.flush(); + let shallow_state_kv = state.store.get_kv_clone(); + drop(state); + shallow_state_kv.retain_keys(&alive_c_bytes); + shallow_state_kv.insert(FRONTIERS_KEY, start_from.encode().into()); + let shallow_state_bytes = shallow_state_kv.export(); + let snapshot = Snapshot { + oplog_bytes, + state_bytes: None, + shallow_root_state_bytes: shallow_state_bytes, + }; + _encode_snapshot(snapshot, w); + Ok(()) + })(); + + restore_export_doc_state(doc, &state_frontiers, is_attached)?; + doc.drop_pending_events(); + result?; + Ok(start_from) +} + +fn cids_to_bytes( + alive_containers: std::collections::HashSet, +) -> BTreeSet> { + let alive_c_bytes: BTreeSet> = alive_containers.iter().map(|x| x.to_bytes()).collect(); + alive_c_bytes +} + +fn frontiers_to_vv_for_export( + oplog: &crate::OpLog, + frontiers: &Frontiers, + context: &str, +) -> Result { + oplog.dag().frontiers_to_vv(frontiers).ok_or_else(|| { + LoroEncodeError::FrontiersNotFound(format!( + "{context}: unreachable frontiers {frontiers:?}" + )) + }) +} + +fn restore_export_doc_state( + doc: &LoroDoc, + state_frontiers: &Frontiers, + was_attached: bool, +) -> Result<(), LoroEncodeError> { + if &doc.state_frontiers() != state_frontiers { + doc._checkout_without_emitting(state_frontiers, false, false) + .map_err(LoroEncodeError::from)?; + } + + if was_attached { + doc.set_detached(false); + } + + Ok(()) +} + +/// Calculates optimal starting version for the shallow doc +/// +/// It should be the LCA of the user given version and the latest version. +/// Otherwise, users cannot replay the history from the initial version till the latest version. +fn calc_shallow_doc_start(oplog: &crate::OpLog, frontiers: &Frontiers) -> Frontiers { + // Find the LCA of the given frontiers by iteratively pairwise GCA. + // This converges to a single frontier or empty if there is no common ancestor. + let mut current = frontiers.clone(); + while current.len() > 1 { + let ids: Vec = current.iter().collect(); + let mut next = Frontiers::new(); + let mut i = 0; + while i < ids.len() { + if i + 1 < ids.len() { + let (gca, _) = oplog + .dag() + .find_common_ancestor(&Frontiers::from(ids[i]), &Frontiers::from(ids[i + 1])); + for id in gca.iter() { + next.push(id); + } + } else { + next.push(ids[i]); + } + i += 2; + } + if next == current { + // Cannot converge further (pairwise GCAs are the nodes themselves). + // Fall back to empty frontiers, meaning export full history. + return Frontiers::default(); + } + current = next; + } + + let mut ans = Frontiers::new(); + for id in current.iter() { + let mut processed = false; + if let Some(op) = oplog.get_op_that_includes(id) { + if let crate::op::InnerContent::List(InnerListOp::StyleStart { .. }) = &op.content { + // StyleStart and StyleEnd operations must be kept together in the GC snapshot. + // Splitting them could lead to an weird document state that cannot be + // properly encoded. To ensure they stay together, we advance the frontier by + // one step to include both operations. + + // > Id.counter + 1 is guaranteed to be the StyleEnd Op + ans.push(id.inc(1)); + processed = true; + } + } + + if !processed { + ans.push(id); + } + } + + ans +} + +pub(crate) fn encode_snapshot_at( + doc: &LoroDoc, + frontiers: &Frontiers, + w: &mut W, +) -> Result<(), LoroEncodeError> { + let was_detached = doc.is_detached(); + let version_before_start = doc.state_frontiers().clone(); + doc._checkout_without_emitting(frontiers, true, false) + .map_err(LoroEncodeError::from)?; + let result = 'block: { + let oplog = doc.oplog().lock(); + let mut state = doc.app_state().lock(); + let is_shallow = state.store.shallow_root_store().is_some(); + if is_shallow { + break 'block Err(LoroEncodeError::from(LoroError::NotImplemented( + "fork_at on shallow docs", + ))); + } + + if state.is_in_txn() { + break 'block Err(LoroEncodeError::internal( + "encode_snapshot_at: state is unexpectedly still in a transaction", + )); + } + let Some(oplog_bytes) = oplog.fork_changes_up_to(frontiers) else { + break 'block Err(LoroEncodeError::FrontiersNotFound(format!( + "frontiers: {:?} when export in SnapshotAt mode", + frontiers + ))); + }; + + if oplog.is_shallow() { + let Some(shallow_root_frontiers) = state.store.shallow_root_frontiers() else { + break 'block Err(LoroEncodeError::internal( + "encode_snapshot_at: shallow oplog is missing shallow root frontiers", + )); + }; + if oplog.shallow_since_frontiers() != shallow_root_frontiers { + break 'block Err(LoroEncodeError::internal( + "encode_snapshot_at: shallow root frontiers are inconsistent", + )); + } + } + + let alive_containers = state.ensure_all_alive_containers(); + if has_unknown_container(alive_containers.iter()) { + break 'block Err(LoroEncodeError::UnknownContainer); + } + + let alive_c_bytes = cids_to_bytes(alive_containers); + state.store.flush(); + let state_kv = state.store.get_kv_clone(); + state_kv.retain_keys(&alive_c_bytes); + let bytes = state_kv.export(); + _encode_snapshot( + Snapshot { + oplog_bytes, + state_bytes: Some(bytes), + shallow_root_state_bytes: Bytes::new(), + }, + w, + ); + + Ok(()) + }; + let restore_result = doc + ._checkout_without_emitting(&version_before_start, false, false) + .map_err(LoroEncodeError::from); + if !was_detached { + doc.set_detached(false); + } + doc.app_state().lock().take_events(); + + match result { + Err(err) => Err(err), + Ok(()) => restore_result, + } +} diff --git a/vendor/loro-internal/src/encoding/value.rs b/vendor/loro-internal/src/encoding/value.rs new file mode 100644 index 00000000..908ef5c9 --- /dev/null +++ b/vendor/loro-internal/src/encoding/value.rs @@ -0,0 +1,1515 @@ +use enum_as_inner::EnumAsInner; +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, ContainerType, Counter, InternalString, LoroError, LoroResult, LoroValue, PeerID, + TreeID, ID, +}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; + +use crate::{ + change::Lamport, container::tree::tree_op::TreeOp, + encoding::outdated_encode_reordered::MAX_COLLECTION_SIZE, +}; + +use super::{ + arena::{EncodedRegisters, EncodedTreeID}, + value_register::ValueRegister, +}; + +pub trait ValueDecodedArenasTrait { + fn keys(&self) -> &[InternalString]; + fn peers(&self) -> &[PeerID]; + fn decode_tree_op( + &self, + positions: &[Vec], + op: super::value::EncodedTreeMove, + id: ID, + ) -> LoroResult; +} + +pub trait ValueEncodeRegister { + fn key_mut(&mut self) -> &mut ValueRegister; + fn peer_mut(&mut self) -> &mut ValueRegister; + fn encode_tree_op(&mut self, op: &TreeOp) -> Value<'static>; +} + +#[derive(Debug)] +pub enum ValueKind { + Null, // 0 + True, // 1 + False, // 2 + I64, // 3 + F64, // 4 + Str, // 5 + Binary, // 6 + ContainerType, // 7 + DeleteOnce, // 8 + DeleteSeq, // 9 + DeltaInt, // 10 + LoroValue, // 11 + MarkStart, // 12 + TreeMove, // 13 + ListMove, // 14 + ListSet, // 15 + Future(FutureValueKind), + // 16 Counter + RawTreeMove, // 17 +} + +#[derive(Debug)] +pub enum LoroValueKind { + Null, + True, + False, + I64, + F64, + Binary, + Str, + List, + Map, + ContainerType, +} +impl LoroValueKind { + fn from_u8(kind: u8) -> Option { + match kind { + 0 => Some(LoroValueKind::Null), + 1 => Some(LoroValueKind::True), + 2 => Some(LoroValueKind::False), + 3 => Some(LoroValueKind::I64), + 4 => Some(LoroValueKind::F64), + 5 => Some(LoroValueKind::Str), + 6 => Some(LoroValueKind::Binary), + 7 => Some(LoroValueKind::List), + 8 => Some(LoroValueKind::Map), + 9 => Some(LoroValueKind::ContainerType), + _ => None, + } + } + + fn to_u8(&self) -> u8 { + match self { + LoroValueKind::Null => 0, + LoroValueKind::True => 1, + LoroValueKind::False => 2, + LoroValueKind::I64 => 3, + LoroValueKind::F64 => 4, + LoroValueKind::Str => 5, + LoroValueKind::Binary => 6, + LoroValueKind::List => 7, + LoroValueKind::Map => 8, + LoroValueKind::ContainerType => 9, + } + } +} + +#[derive(Debug)] +pub enum FutureValueKind { + Unknown(u8), +} + +impl ValueKind { + pub(crate) fn to_u8(&self) -> u8 { + match self { + ValueKind::Null => 0, + ValueKind::True => 1, + ValueKind::False => 2, + ValueKind::I64 => 3, + ValueKind::F64 => 4, + ValueKind::Str => 5, + ValueKind::Binary => 6, + ValueKind::ContainerType => 7, + ValueKind::DeleteOnce => 8, + ValueKind::DeleteSeq => 9, + ValueKind::DeltaInt => 10, + ValueKind::LoroValue => 11, + ValueKind::MarkStart => 12, + ValueKind::TreeMove => 13, + ValueKind::ListMove => 14, + ValueKind::ListSet => 15, + ValueKind::Future(future_value_kind) => match future_value_kind { + FutureValueKind::Unknown(u8) => *u8 | 0x80, + }, + ValueKind::RawTreeMove => 16, + } + } + + pub(crate) fn from_u8(mut kind: u8) -> Self { + kind &= 0x7F; + match kind { + 0 => ValueKind::Null, + 1 => ValueKind::True, + 2 => ValueKind::False, + 3 => ValueKind::I64, + 4 => ValueKind::F64, + 5 => ValueKind::Str, + 6 => ValueKind::Binary, + 7 => ValueKind::ContainerType, + 8 => ValueKind::DeleteOnce, + 9 => ValueKind::DeleteSeq, + 10 => ValueKind::DeltaInt, + 11 => ValueKind::LoroValue, + 12 => ValueKind::MarkStart, + 13 => ValueKind::TreeMove, + 14 => ValueKind::ListMove, + 15 => ValueKind::ListSet, + 16 => ValueKind::RawTreeMove, + _ => ValueKind::Future(FutureValueKind::Unknown(kind)), + } + } +} + +#[derive(Debug, EnumAsInner)] +pub enum Value<'a> { + Null, + True, + False, + I64(i64), + F64(f64), + Str(&'a str), + Binary(&'a [u8]), + ContainerIdx(usize), + DeleteOnce, + DeleteSeq, + DeltaInt(i32), + #[allow(clippy::enum_variant_names)] + LoroValue(LoroValue), + MarkStart(MarkStart), + TreeMove(EncodedTreeMove), + RawTreeMove(RawTreeMove), + ListMove { + from: usize, + from_idx: usize, + lamport: usize, + }, + ListSet { + peer_idx: usize, + lamport: Lamport, + value: LoroValue, + }, + Future(FutureValue<'a>), +} + +#[derive(Debug)] +pub enum FutureValue<'a> { + // The future value cannot depend on the arena for encoding. + Unknown { kind: u8, data: &'a [u8] }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "value_type", content = "value", rename_all = "snake_case")] +pub enum OwnedValue { + Null, + True, + False, + I64(i64), + F64(f64), + Str(Arc), + Binary(Arc>), + ContainerIdx(usize), + DeleteOnce, + DeleteSeq, + DeltaInt(i32), + LoroValue(LoroValue), + MarkStart(MarkStart), + TreeMove(EncodedTreeMove), + RawTreeMove(RawTreeMove), + ListMove { + from: usize, + from_idx: usize, + lamport: usize, + }, + ListSet { + peer_idx: usize, + lamport: Lamport, + value: LoroValue, + }, + #[serde(untagged)] + Future(OwnedFutureValue), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "value_type", content = "value")] +pub enum OwnedFutureValue { + // The future value cannot depend on the arena for encoding. + Unknown { kind: u8, data: Arc> }, +} + +impl<'a> Value<'a> { + pub fn from_owned(owned_value: &'a OwnedValue) -> Self { + match owned_value { + OwnedValue::Null => Value::Null, + OwnedValue::True => Value::True, + OwnedValue::False => Value::False, + OwnedValue::DeleteOnce => Value::DeleteOnce, + OwnedValue::I64(x) => Value::I64(*x), + OwnedValue::ContainerIdx(x) => Value::ContainerIdx(*x), + OwnedValue::F64(x) => Value::F64(*x), + OwnedValue::Str(x) => Value::Str(x.as_str()), + OwnedValue::DeleteSeq => Value::DeleteSeq, + OwnedValue::DeltaInt(x) => Value::DeltaInt(*x), + OwnedValue::LoroValue(x) => Value::LoroValue(x.clone()), + OwnedValue::MarkStart(x) => Value::MarkStart(x.clone()), + OwnedValue::Binary(x) => Value::Binary(x.as_slice()), + OwnedValue::TreeMove(x) => Value::TreeMove(x.clone()), + OwnedValue::ListMove { + from, + from_idx, + lamport, + } => Value::ListMove { + from: *from, + from_idx: *from_idx, + lamport: *lamport, + }, + OwnedValue::ListSet { + peer_idx, + lamport, + value, + } => Value::ListSet { + peer_idx: *peer_idx, + lamport: *lamport, + value: value.clone(), + }, + OwnedValue::Future(value) => match value { + OwnedFutureValue::Unknown { kind, data } => Value::Future(FutureValue::Unknown { + kind: *kind, + data: data.as_slice(), + }), + }, + OwnedValue::RawTreeMove(t) => Value::RawTreeMove(t.clone()), + } + } + + pub fn into_owned(self) -> OwnedValue { + match self { + Value::Null => OwnedValue::Null, + Value::True => OwnedValue::True, + Value::False => OwnedValue::False, + Value::DeleteOnce => OwnedValue::DeleteOnce, + Value::I64(x) => OwnedValue::I64(x), + Value::ContainerIdx(x) => OwnedValue::ContainerIdx(x), + Value::F64(x) => OwnedValue::F64(x), + Value::Str(x) => OwnedValue::Str(Arc::new(x.to_owned())), + Value::DeleteSeq => OwnedValue::DeleteSeq, + Value::DeltaInt(x) => OwnedValue::DeltaInt(x), + Value::LoroValue(x) => OwnedValue::LoroValue(x), + Value::MarkStart(x) => OwnedValue::MarkStart(x), + Value::Binary(x) => OwnedValue::Binary(Arc::new(x.to_owned())), + Value::TreeMove(x) => OwnedValue::TreeMove(x), + Value::ListMove { + from, + from_idx, + lamport, + } => OwnedValue::ListMove { + from, + from_idx, + lamport, + }, + Value::ListSet { + peer_idx, + lamport, + value, + } => OwnedValue::ListSet { + peer_idx, + lamport, + value, + }, + Value::Future(value) => match value { + FutureValue::Unknown { kind, data } => { + OwnedValue::Future(OwnedFutureValue::Unknown { + kind, + data: Arc::new(data.to_owned()), + }) + } + }, + Value::RawTreeMove(t) => OwnedValue::RawTreeMove(t), + } + } + + fn decode_without_arena<'r: 'a>( + future_kind: FutureValueKind, + value_reader: &'r mut ValueReader, + ) -> LoroResult { + let bytes = value_reader.read_binary()?; + let value = match future_kind { + FutureValueKind::Unknown(kind) => { + Value::Future(FutureValue::Unknown { kind, data: bytes }) + } + }; + Ok(value) + } + + pub(crate) fn decode<'r: 'a>( + kind: ValueKind, + value_reader: &'r mut ValueReader, + arenas: &'a dyn ValueDecodedArenasTrait, + id: ID, + ) -> LoroResult { + Ok(match kind { + ValueKind::Null => Value::Null, + ValueKind::True => Value::True, + ValueKind::False => Value::False, + ValueKind::I64 => Value::I64(value_reader.read_i64()?), + ValueKind::F64 => Value::F64(value_reader.read_f64()?), + ValueKind::Str => Value::Str(value_reader.read_str()?), + ValueKind::Binary => Value::Binary(value_reader.read_binary()?), + ValueKind::ContainerType => Value::ContainerIdx(value_reader.read_usize()?), + ValueKind::DeleteOnce => Value::DeleteOnce, + ValueKind::DeleteSeq => Value::DeleteSeq, + ValueKind::DeltaInt => Value::DeltaInt(value_reader.read_i32()?), + ValueKind::LoroValue => { + Value::LoroValue(value_reader.read_value_type_and_content(arenas.keys(), id)?) + } + ValueKind::MarkStart => Value::MarkStart(value_reader.read_mark(arenas.keys(), id)?), + ValueKind::TreeMove => Value::TreeMove(value_reader.read_tree_move()?), + ValueKind::RawTreeMove => Value::RawTreeMove(value_reader.read_raw_tree_move()?), + ValueKind::ListMove => { + let from = value_reader.read_usize()?; + let from_idx = value_reader.read_usize()?; + let lamport = value_reader.read_usize()?; + Value::ListMove { + from, + from_idx, + lamport, + } + } + ValueKind::ListSet => { + let peer_idx = value_reader.read_usize()?; + let lamport = value_reader.read_usize()? as u32; + let value = value_reader.read_value_type_and_content(arenas.keys(), id)?; + Value::ListSet { + peer_idx, + lamport, + value, + } + } + ValueKind::Future(future_kind) => { + Self::decode_without_arena(future_kind, value_reader)? + } + }) + } + + fn encode_without_registers( + value: FutureValue, + value_writer: &mut ValueWriter, + ) -> (FutureValueKind, usize) { + // Note: we should encode FutureValue as binary data. + // [binary data length, binary data] + // when decoding, we will use reader.read_binary() to read the binary data. + // So such as FutureValue::Counter, we should write 0 as the length of binary data first. + match value { + FutureValue::Unknown { kind, data } => ( + FutureValueKind::Unknown(kind), + value_writer.write_binary(data), + ), + } + } + + pub(super) fn encode( + self, + value_writer: &mut ValueWriter, + registers: &mut dyn ValueEncodeRegister, + ) -> (ValueKind, usize) { + match self { + Value::Null => (ValueKind::Null, 0), + Value::True => (ValueKind::True, 0), + Value::False => (ValueKind::False, 0), + Value::I64(x) => (ValueKind::I64, value_writer.write_i64(x)), + Value::F64(x) => (ValueKind::F64, value_writer.write_f64(x)), + Value::Str(x) => (ValueKind::Str, value_writer.write_str(x)), + Value::Binary(x) => (ValueKind::Binary, value_writer.write_binary(x)), + Value::ContainerIdx(x) => (ValueKind::ContainerType, value_writer.write_usize(x)), + Value::DeleteOnce => (ValueKind::DeleteOnce, 0), + Value::DeleteSeq => (ValueKind::DeleteSeq, 0), + Value::DeltaInt(x) => (ValueKind::DeltaInt, value_writer.write_i32(x)), + Value::LoroValue(x) => ( + ValueKind::LoroValue, + value_writer.write_value_type_and_content(&x, registers), + ), + Value::MarkStart(x) => (ValueKind::MarkStart, value_writer.write_mark(x, registers)), + Value::TreeMove(tree) => (ValueKind::TreeMove, value_writer.write_tree_move(&tree)), + Value::ListMove { + from, + from_idx, + lamport, + } => ( + ValueKind::ListMove, + value_writer.write_usize(from) + + value_writer.write_usize(from_idx) + + value_writer.write_usize(lamport), + ), + Value::ListSet { + peer_idx, + lamport, + value, + } => ( + ValueKind::ListSet, + value_writer.write_usize(peer_idx) + + value_writer.write_usize(lamport as usize) + + value_writer.write_value_type_and_content(&value, registers), + ), + Value::Future(value) => { + let (k, i) = Self::encode_without_registers(value, value_writer); + (ValueKind::Future(k), i) + } + Value::RawTreeMove(x) => (ValueKind::RawTreeMove, value_writer.write_raw_tree_move(x)), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct MarkStart { + pub len: u32, + pub key: InternalString, + pub value: LoroValue, + pub info: u8, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RawTreeMove { + pub subject_peer_idx: usize, + pub subject_cnt: Counter, + pub is_parent_null: bool, + pub parent_peer_idx: usize, + pub parent_cnt: Counter, + pub position_idx: usize, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EncodedTreeMove { + pub target_idx: usize, + pub is_parent_null: bool, + pub parent_idx: usize, + pub position: usize, +} + +impl EncodedTreeMove { + pub fn as_tree_op( + &self, + peer_ids: &[u64], + positions: &[Vec], + tree_ids: &[EncodedTreeID], + op_id: ID, + ) -> LoroResult { + let parent = if self.is_parent_null { + None + } else { + let EncodedTreeID { peer_idx, counter } = tree_ids[self.parent_idx]; + Some(TreeID::new( + *(peer_ids + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter as Counter, + )) + }; + let is_delete = parent.is_some_and(|p| TreeID::is_deleted_root(&p)); + let position = if is_delete { + None + } else { + let bytes = &positions[self.position]; + Some(FractionalIndex::from_bytes(bytes.clone())) + }; + let EncodedTreeID { peer_idx, counter } = tree_ids[self.target_idx]; + let target = TreeID::new( + *(peer_ids + .get(peer_idx) + .ok_or(LoroError::DecodeDataCorruptionError)?), + counter as Counter, + ); + + let is_create = target.id() == op_id; + let op = if is_delete { + TreeOp::Delete { target } + } else if is_create { + TreeOp::Create { + target, + parent, + position: position.unwrap(), + } + } else { + TreeOp::Move { + target, + parent, + position: position.unwrap(), + } + }; + Ok(op) + } + + pub fn from_tree_op<'p, 'a: 'p>(op: &'a TreeOp, registers: &mut EncodedRegisters) -> Self { + match op { + TreeOp::Create { + target, + parent, + position, + } + | TreeOp::Move { + target, + parent, + position, + } => { + let bytes = position.as_bytes(); + let either::Right(position_register) = &mut registers.position else { + unreachable!() + }; + let position = position_register.get(&bytes).unwrap(); + let target_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&target.peer), + counter: target.counter, + }); + + let parent_idx = parent.map(|x| { + registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&x.peer), + counter: x.counter, + }) + }); + EncodedTreeMove { + target_idx, + is_parent_null: parent.is_none(), + parent_idx: parent_idx.unwrap_or(0), + position, + } + } + TreeOp::Delete { target } => { + let target_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&target.peer), + counter: target.counter, + }); + let parent_idx = registers.tree_id.register(&EncodedTreeID { + peer_idx: registers.peer.register(&TreeID::delete_root().peer), + counter: TreeID::delete_root().counter, + }); + EncodedTreeMove { + target_idx, + is_parent_null: false, + parent_idx, + position: 0, + } + } + } + } +} + +pub struct ValueWriter { + buffer: Vec, +} + +pub struct ValueReader<'a> { + raw: &'a [u8], +} + +impl<'a> ValueReader<'a> { + pub fn new(raw: &'a [u8]) -> Self { + ValueReader { raw } + } + + pub fn read_value_type_and_content( + &mut self, + keys: &[InternalString], + id: ID, + ) -> LoroResult { + let kind = self.read_u8()?; + let kind = LoroValueKind::from_u8(kind).ok_or(LoroError::DecodeDataCorruptionError)?; + self.read_value_content(kind, keys, id) + } + + pub fn read_value_content( + &mut self, + kind: LoroValueKind, + keys: &[InternalString], + id: ID, + ) -> LoroResult { + Ok(match kind { + LoroValueKind::Null => LoroValue::Null, + LoroValueKind::True => LoroValue::Bool(true), + LoroValueKind::False => LoroValue::Bool(false), + LoroValueKind::I64 => LoroValue::I64(self.read_i64()?), + LoroValueKind::F64 => LoroValue::Double(self.read_f64()?), + LoroValueKind::Str => LoroValue::String((self.read_str()?.to_owned()).into()), + LoroValueKind::List => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + let mut ans = Vec::with_capacity(len); + for i in 0..len { + ans.push(self.recursive_read_value_type_and_content(keys, id.inc(i as i32))?); + } + ans.into() + } + LoroValueKind::Map => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + let mut ans = FxHashMap::with_capacity_and_hasher(len, Default::default()); + for _ in 0..len { + let key_idx = self.read_usize()?; + let key = keys + .get(key_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .to_string(); + let value = self.recursive_read_value_type_and_content(keys, id)?; + ans.insert(key, value); + } + ans.into() + } + LoroValueKind::Binary => LoroValue::Binary(self.read_binary()?.to_owned().into()), + LoroValueKind::ContainerType => { + let u8 = self.read_u8()?; + let container_id = ContainerID::new_normal( + id, + ContainerType::try_from_u8(u8).unwrap_or(ContainerType::Unknown(u8)), + ); + + LoroValue::Container(container_id) + } + }) + } + + /// Read a value that may be very deep efficiently. + /// + /// This method avoids using recursive calls to read deeply nested values. + /// Otherwise, it may cause stack overflow. + fn recursive_read_value_type_and_content( + &mut self, + keys: &[InternalString], + id: ID, + ) -> LoroResult { + #[derive(Debug)] + enum Task { + Init, + ReadList { + left: usize, + vec: Vec, + key_idx_in_parent: usize, + }, + ReadMap { + left: usize, + map: FxHashMap, + key_idx_in_parent: usize, + }, + } + impl Task { + fn should_read(&self) -> bool { + !matches!( + self, + Self::ReadList { left: 0, .. } | Self::ReadMap { left: 0, .. } + ) + } + + fn key_idx(&self) -> usize { + match self { + Self::ReadList { + key_idx_in_parent, .. + } => *key_idx_in_parent, + Self::ReadMap { + key_idx_in_parent, .. + } => *key_idx_in_parent, + _ => unreachable!(), + } + } + + fn into_value(self) -> LoroValue { + match self { + Self::ReadList { vec, .. } => vec.into(), + Self::ReadMap { map, .. } => map.into(), + _ => unreachable!(), + } + } + } + let mut stack = vec![Task::Init]; + while let Some(mut task) = stack.pop() { + if task.should_read() { + let key_idx = if matches!(task, Task::ReadMap { .. }) { + self.read_usize()? + } else { + 0 + }; + let kind = self.read_u8()?; + let kind = + LoroValueKind::from_u8(kind).ok_or(LoroError::DecodeDataCorruptionError)?; + let value = match kind { + LoroValueKind::Null => LoroValue::Null, + LoroValueKind::True => LoroValue::Bool(true), + LoroValueKind::False => LoroValue::Bool(false), + LoroValueKind::I64 => LoroValue::I64(self.read_i64()?), + LoroValueKind::F64 => LoroValue::Double(self.read_f64()?), + LoroValueKind::Str => LoroValue::String((self.read_str()?.to_owned()).into()), + LoroValueKind::List => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + let ans = Vec::with_capacity(len); + stack.push(task); + stack.push(Task::ReadList { + left: len, + vec: ans, + key_idx_in_parent: key_idx, + }); + continue; + } + LoroValueKind::Map => { + let len = self.read_usize()?; + if len > MAX_COLLECTION_SIZE { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = FxHashMap::with_capacity_and_hasher(len, Default::default()); + stack.push(task); + stack.push(Task::ReadMap { + left: len, + map: ans, + key_idx_in_parent: key_idx, + }); + continue; + } + LoroValueKind::Binary => { + LoroValue::Binary(self.read_binary()?.to_owned().into()) + } + LoroValueKind::ContainerType => { + let u8 = self.read_u8()?; + let container_id = ContainerID::new_normal( + id, + ContainerType::try_from_u8(u8).unwrap_or(ContainerType::Unknown(u8)), + ); + + LoroValue::Container(container_id) + } + }; + + task = match task { + Task::Init => { + return Ok(value); + } + Task::ReadList { + mut left, + mut vec, + key_idx_in_parent, + } => { + left -= 1; + vec.push(value); + let task = Task::ReadList { + left, + vec, + key_idx_in_parent, + }; + if left != 0 { + stack.push(task); + continue; + } + + task + } + Task::ReadMap { + mut left, + mut map, + key_idx_in_parent, + } => { + left -= 1; + let key = keys + .get(key_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .to_string(); + map.insert(key, value); + let task = Task::ReadMap { + left, + map, + key_idx_in_parent, + }; + if left != 0 { + stack.push(task); + continue; + } + task + } + }; + } + + let key_index = task.key_idx(); + let value = task.into_value(); + if let Some(last) = stack.last_mut() { + match last { + Task::Init => { + return Ok(value); + } + Task::ReadList { left, vec, .. } => { + *left -= 1; + vec.push(value); + } + Task::ReadMap { left, map, .. } => { + *left -= 1; + let key = keys + .get(key_index) + .ok_or(LoroError::DecodeDataCorruptionError)? + .to_string(); + map.insert(key, value); + } + } + } else { + return Ok(value); + } + } + + unreachable!(); + } + + pub fn read_i64(&mut self) -> LoroResult { + leb128::read::signed(&mut self.raw).map_err(|_| LoroError::DecodeDataCorruptionError) + } + + #[allow(unused)] + pub fn read_u64(&mut self) -> LoroResult { + leb128::read::unsigned(&mut self.raw).map_err(|_| LoroError::DecodeDataCorruptionError) + } + + #[allow(unused)] + pub fn read_i32(&mut self) -> LoroResult { + leb128::read::signed(&mut self.raw) + .map(|x| x as i32) + .map_err(|_| LoroError::DecodeDataCorruptionError) + } + + pub fn read_f64(&mut self) -> LoroResult { + if self.raw.len() < 8 { + return Err(LoroError::DecodeDataCorruptionError); + } + + let mut bytes = [0; 8]; + bytes.copy_from_slice(&self.raw[..8]); + self.raw = &self.raw[8..]; + Ok(f64::from_be_bytes(bytes)) + } + + pub fn read_usize(&mut self) -> LoroResult { + Ok(leb128::read::unsigned(&mut self.raw) + .map_err(|_| LoroError::DecodeDataCorruptionError)? as usize) + } + + pub fn read_str(&mut self) -> LoroResult<&'a str> { + let len = self.read_usize()?; + if self.raw.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = std::str::from_utf8(&self.raw[..len]) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + self.raw = &self.raw[len..]; + Ok(ans) + } + + #[allow(unused)] + fn read_binary_vec(&mut self) -> LoroResult> { + let len = self.read_usize()?; + if self.raw.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + let ans = &self.raw[..len]; + self.raw = &self.raw[len..]; + Ok(ans.to_vec()) + } + + fn read_u8(&mut self) -> LoroResult { + if self.raw.is_empty() { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = self.raw[0]; + self.raw = &self.raw[1..]; + Ok(ans) + } + + pub fn read_binary(&mut self) -> LoroResult<&'a [u8]> { + let len = self.read_usize()?; + if self.raw.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + + let ans = &self.raw[..len]; + self.raw = &self.raw[len..]; + Ok(ans) + } + + pub fn read_mark<'s: 'm, 'm>( + &mut self, + keys: &'s [InternalString], + id: ID, + ) -> LoroResult { + let info = self.read_u8()?; + let len = self.read_usize()?; + let key_idx = self.read_usize()?; + let value = self.read_value_type_and_content(keys, id)?; + Ok(MarkStart { + len: len as u32, + key: keys + .get(key_idx) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + value, + info, + }) + } + + pub fn read_tree_move(&mut self) -> LoroResult { + let subject_idx = self.read_usize()?; + let is_parent_null = self.read_u8()? != 0; + let position = self.read_usize()?; + let mut parent_idx = 0; + if !is_parent_null { + parent_idx = self.read_usize()?; + } + Ok(EncodedTreeMove { + target_idx: subject_idx, + is_parent_null, + parent_idx, + position, + }) + } + + fn read_raw_tree_move(&mut self) -> LoroResult { + let subject_peer_idx = self.read_usize()?; + let subject_cnt = self.read_usize()?; + let position_idx = self.read_usize()?; + let is_parent_null = self.read_u8()? != 0; + let mut parent_peer_idx = 0; + let mut parent_cnt = 0; + if !is_parent_null { + parent_peer_idx = self.read_usize()?; + parent_cnt = self.read_usize()?; + } + + Ok(RawTreeMove { + subject_peer_idx, + subject_cnt: subject_cnt as i32, + position_idx, + is_parent_null, + parent_peer_idx, + parent_cnt: parent_cnt as i32, + }) + } +} + +impl ValueWriter { + pub fn new() -> Self { + ValueWriter { buffer: Vec::new() } + } + + pub fn write_value_type_and_content( + &mut self, + value: &LoroValue, + registers: &mut dyn ValueEncodeRegister, + ) -> usize { + let len = self.write_u8(get_loro_value_kind(value).to_u8()); + let (_, l) = self.write_value_content(value, registers); + len + l + } + + pub fn write_value_content( + &mut self, + value: &LoroValue, + registers: &mut dyn ValueEncodeRegister, + ) -> (LoroValueKind, usize) { + match value { + LoroValue::Null => (LoroValueKind::Null, 0), + LoroValue::Bool(true) => (LoroValueKind::True, 0), + LoroValue::Bool(false) => (LoroValueKind::False, 0), + LoroValue::I64(value) => (LoroValueKind::I64, self.write_i64(*value)), + LoroValue::Double(value) => (LoroValueKind::F64, self.write_f64(*value)), + LoroValue::String(value) => (LoroValueKind::Str, self.write_str(value)), + LoroValue::List(value) => { + let mut len = self.write_usize(value.len()); + for value in value.iter() { + let l = self.write_value_type_and_content(value, registers); + len += l; + } + (LoroValueKind::List, len) + } + LoroValue::Map(value) => { + let mut len = self.write_usize(value.len()); + for (key, value) in value.iter() { + let key_idx = registers.key_mut().register(&key.as_str().into()); + len += self.write_usize(key_idx); + let l = self.write_value_type_and_content(value, registers); + len += l; + } + (LoroValueKind::Map, len) + } + LoroValue::Binary(value) => (LoroValueKind::Binary, self.write_binary(value)), + LoroValue::Container(c) => ( + LoroValueKind::ContainerType, + self.write_u8(c.container_type().to_u8()), + ), + } + } + + pub fn write_i64(&mut self, value: i64) -> usize { + let len = self.buffer.len(); + leb128::write::signed(&mut self.buffer, value).unwrap(); + self.buffer.len() - len + } + + fn write_i32(&mut self, value: i32) -> usize { + let len = self.buffer.len(); + leb128::write::signed(&mut self.buffer, value as i64).unwrap(); + self.buffer.len() - len + } + + #[allow(unused)] + fn write_u64(&mut self, value: u64) -> usize { + let len = self.buffer.len(); + leb128::write::unsigned(&mut self.buffer, value).unwrap(); + self.buffer.len() - len + } + + fn write_usize(&mut self, value: usize) -> usize { + let len = self.buffer.len(); + leb128::write::unsigned(&mut self.buffer, value as u64).unwrap(); + self.buffer.len() - len + } + + fn write_f64(&mut self, value: f64) -> usize { + let len = self.buffer.len(); + self.buffer.extend_from_slice(&value.to_be_bytes()); + self.buffer.len() - len + } + + fn write_str(&mut self, value: &str) -> usize { + let len = self.buffer.len(); + self.write_usize(value.len()); + self.buffer.extend_from_slice(value.as_bytes()); + self.buffer.len() - len + } + + fn write_u8(&mut self, value: u8) -> usize { + let len = self.buffer.len(); + self.buffer.push(value); + self.buffer.len() - len + } + + fn write_binary(&mut self, value: &[u8]) -> usize { + let len = self.buffer.len(); + self.write_usize(value.len()); + self.buffer.extend_from_slice(value); + self.buffer.len() - len + } + + fn write_mark(&mut self, mark: MarkStart, registers: &mut dyn ValueEncodeRegister) -> usize { + let key_idx = registers.key_mut().register(&mark.key); + let len = self.buffer.len(); + self.write_u8(mark.info); + self.write_usize(mark.len as usize); + self.write_usize(key_idx); + self.write_value_type_and_content(&mark.value, registers); + self.buffer.len() - len + } + + fn write_tree_move(&mut self, op: &EncodedTreeMove) -> usize { + let len = self.buffer.len(); + self.write_usize(op.target_idx); + self.write_u8(op.is_parent_null as u8); + self.write_usize(op.position); + if op.is_parent_null { + return self.buffer.len() - len; + } + self.write_usize(op.parent_idx); + self.buffer.len() - len + } + + pub(crate) fn finish(self) -> Vec { + self.buffer + } + + fn write_raw_tree_move(&mut self, op: RawTreeMove) -> usize { + let len = self.buffer.len(); + self.write_usize(op.subject_peer_idx); + self.write_usize(op.subject_cnt as usize); + self.write_usize(op.position_idx); + self.write_u8(op.is_parent_null as u8); + if op.is_parent_null { + return self.buffer.len() - len; + } + self.write_usize(op.parent_peer_idx); + self.write_usize(op.parent_cnt as usize); + self.buffer.len() - len + } +} + +fn get_loro_value_kind(value: &LoroValue) -> LoroValueKind { + match value { + LoroValue::Null => LoroValueKind::Null, + LoroValue::Bool(true) => LoroValueKind::True, + LoroValue::Bool(false) => LoroValueKind::False, + LoroValue::I64(_) => LoroValueKind::I64, + LoroValue::Double(_) => LoroValueKind::F64, + LoroValue::String(_) => LoroValueKind::Str, + LoroValue::List(_) => LoroValueKind::List, + LoroValue::Map(_) => LoroValueKind::Map, + LoroValue::Binary(_) => LoroValueKind::Binary, + LoroValue::Container(_) => LoroValueKind::ContainerType, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{container::tree::tree_op::TreeOp, fx_map}; + + #[derive(Default)] + struct TestRegisters { + key: ValueRegister, + peer: ValueRegister, + } + + impl ValueEncodeRegister for TestRegisters { + fn key_mut(&mut self) -> &mut ValueRegister { + &mut self.key + } + + fn peer_mut(&mut self) -> &mut ValueRegister { + &mut self.peer + } + + fn encode_tree_op(&mut self, op: &TreeOp) -> Value<'static> { + let target = match op { + TreeOp::Create { target, .. } + | TreeOp::Move { target, .. } + | TreeOp::Delete { target } => *target, + }; + + Value::TreeMove(EncodedTreeMove { + target_idx: self.peer.register(&target.peer), + is_parent_null: true, + parent_idx: 0, + position: 0, + }) + } + } + + struct TestArenas { + keys: Vec, + peers: Vec, + } + + impl ValueDecodedArenasTrait for TestArenas { + fn keys(&self) -> &[InternalString] { + &self.keys + } + + fn peers(&self) -> &[PeerID] { + &self.peers + } + + fn decode_tree_op( + &self, + positions: &[Vec], + op: EncodedTreeMove, + id: ID, + ) -> LoroResult { + op.as_tree_op(&self.peers, positions, &[], id) + } + } + + fn encode_then_decode_owned(value: Value<'_>) -> OwnedValue { + let mut writer = ValueWriter::new(); + let mut registers = TestRegisters::default(); + let (kind, _) = value.encode(&mut writer, &mut registers); + let bytes = writer.finish(); + let arenas = TestArenas { + keys: registers.key.unwrap_vec(), + peers: registers.peer.unwrap_vec(), + }; + let mut reader = ValueReader::new(&bytes); + Value::decode(kind, &mut reader, &arenas, ID::new(9, 1)) + .unwrap() + .into_owned() + } + + #[test] + fn value_kind_tags_roundtrip_including_future_kinds() { + let kinds = [ + ValueKind::Null, + ValueKind::True, + ValueKind::False, + ValueKind::I64, + ValueKind::F64, + ValueKind::Str, + ValueKind::Binary, + ValueKind::ContainerType, + ValueKind::DeleteOnce, + ValueKind::DeleteSeq, + ValueKind::DeltaInt, + ValueKind::LoroValue, + ValueKind::MarkStart, + ValueKind::TreeMove, + ValueKind::ListMove, + ValueKind::ListSet, + ValueKind::RawTreeMove, + ]; + + for kind in kinds { + let tag = kind.to_u8(); + assert_eq!(ValueKind::from_u8(tag).to_u8(), tag); + } + + let future = ValueKind::Future(FutureValueKind::Unknown(42)); + assert_eq!(future.to_u8(), 42 | 0x80); + match ValueKind::from_u8(future.to_u8()) { + ValueKind::Future(FutureValueKind::Unknown(kind)) => assert_eq!(kind, 42), + other => panic!("expected future kind, got {other:?}"), + } + + let loro_kinds = [ + LoroValueKind::Null, + LoroValueKind::True, + LoroValueKind::False, + LoroValueKind::I64, + LoroValueKind::F64, + LoroValueKind::Str, + LoroValueKind::Binary, + LoroValueKind::List, + LoroValueKind::Map, + LoroValueKind::ContainerType, + ]; + for kind in loro_kinds { + assert_eq!( + LoroValueKind::from_u8(kind.to_u8()) + .expect("known value kind tag should decode") + .to_u8(), + kind.to_u8() + ); + } + } + + #[test] + fn owned_value_conversion_is_lossless_for_all_current_variants() { + let mark = MarkStart { + len: 3, + key: "bold".into(), + value: true.into(), + info: 7, + }; + let tree_move = EncodedTreeMove { + target_idx: 2, + is_parent_null: false, + parent_idx: 1, + position: 4, + }; + let raw_tree_move = RawTreeMove { + subject_peer_idx: 3, + subject_cnt: 9, + is_parent_null: false, + parent_peer_idx: 4, + parent_cnt: 8, + position_idx: 6, + }; + let values = vec![ + OwnedValue::Null, + OwnedValue::True, + OwnedValue::False, + OwnedValue::I64(-123), + OwnedValue::F64(1.25), + OwnedValue::Str(Arc::new("hello".to_string())), + OwnedValue::Binary(Arc::new(vec![1, 2, 3])), + OwnedValue::ContainerIdx(42), + OwnedValue::DeleteOnce, + OwnedValue::DeleteSeq, + OwnedValue::DeltaInt(-9), + OwnedValue::LoroValue( + fx_map!( + "k".to_string() => + LoroValue::from(vec![LoroValue::from(1_i64), LoroValue::from("v")]) + ) + .into(), + ), + OwnedValue::MarkStart(mark), + OwnedValue::TreeMove(tree_move), + OwnedValue::RawTreeMove(raw_tree_move), + OwnedValue::ListMove { + from: 1, + from_idx: 2, + lamport: 3, + }, + OwnedValue::ListSet { + peer_idx: 4, + lamport: 5, + value: "set".into(), + }, + OwnedValue::Future(OwnedFutureValue::Unknown { + kind: 99, + data: Arc::new(vec![4, 5, 6]), + }), + ]; + + for owned in values { + assert_eq!(Value::from_owned(&owned).into_owned(), owned); + } + } + + #[test] + fn value_encode_decode_roundtrips_non_loro_value_variants() { + let cases = vec![ + (Value::Null, OwnedValue::Null), + (Value::True, OwnedValue::True), + (Value::False, OwnedValue::False), + (Value::I64(-42), OwnedValue::I64(-42)), + (Value::F64(3.5), OwnedValue::F64(3.5)), + ( + Value::Str("encoded"), + OwnedValue::Str(Arc::new("encoded".to_string())), + ), + ( + Value::Binary(&[9, 8, 7]), + OwnedValue::Binary(Arc::new(vec![9, 8, 7])), + ), + (Value::ContainerIdx(17), OwnedValue::ContainerIdx(17)), + (Value::DeleteOnce, OwnedValue::DeleteOnce), + (Value::DeleteSeq, OwnedValue::DeleteSeq), + (Value::DeltaInt(-17), OwnedValue::DeltaInt(-17)), + ( + Value::LoroValue(fx_map!("nested".to_string() => LoroValue::from(1_i64)).into()), + OwnedValue::LoroValue( + fx_map!("nested".to_string() => LoroValue::from(1_i64)).into(), + ), + ), + ( + Value::MarkStart(MarkStart { + len: 2, + key: "italic".into(), + value: "yes".into(), + info: 3, + }), + OwnedValue::MarkStart(MarkStart { + len: 2, + key: "italic".into(), + value: "yes".into(), + info: 3, + }), + ), + ( + Value::TreeMove(EncodedTreeMove { + target_idx: 5, + is_parent_null: true, + parent_idx: 0, + position: 7, + }), + OwnedValue::TreeMove(EncodedTreeMove { + target_idx: 5, + is_parent_null: true, + parent_idx: 0, + position: 7, + }), + ), + ( + Value::RawTreeMove(RawTreeMove { + subject_peer_idx: 1, + subject_cnt: 2, + is_parent_null: true, + parent_peer_idx: 0, + parent_cnt: 0, + position_idx: 3, + }), + OwnedValue::RawTreeMove(RawTreeMove { + subject_peer_idx: 1, + subject_cnt: 2, + is_parent_null: true, + parent_peer_idx: 0, + parent_cnt: 0, + position_idx: 3, + }), + ), + ( + Value::ListMove { + from: 3, + from_idx: 4, + lamport: 5, + }, + OwnedValue::ListMove { + from: 3, + from_idx: 4, + lamport: 5, + }, + ), + ( + Value::ListSet { + peer_idx: 1, + lamport: 2, + value: vec![LoroValue::from(1_i64), LoroValue::from(2_i64)].into(), + }, + OwnedValue::ListSet { + peer_idx: 1, + lamport: 2, + value: vec![LoroValue::from(1_i64), LoroValue::from(2_i64)].into(), + }, + ), + ( + Value::Future(FutureValue::Unknown { + kind: 100, + data: &[1, 3, 5], + }), + OwnedValue::Future(OwnedFutureValue::Unknown { + kind: 100, + data: Arc::new(vec![1, 3, 5]), + }), + ), + ]; + + for (value, expected) in cases { + assert_eq!(encode_then_decode_owned(value), expected); + } + } + + #[test] + fn loro_value_writer_reader_roundtrips_nested_scalars_collections_and_containers() { + let container = ContainerID::new_normal(ID::new(7, 2), ContainerType::Tree); + let value: LoroValue = fx_map!( + "null".to_string() => LoroValue::Null, + "true".to_string() => true.into(), + "false".to_string() => false.into(), + "int".to_string() => (-9_i64).into(), + "float".to_string() => 2.25.into(), + "string".to_string() => "text".into(), + "binary".to_string() => vec![1_u8, 2, 3].into(), + "container".to_string() => container.into(), + "list".to_string() => vec![ + LoroValue::Null, + true.into(), + fx_map!("child".to_string() => "value".into()).into(), + ].into() + ) + .into(); + + let mut writer = ValueWriter::new(); + let mut registers = TestRegisters::default(); + writer.write_value_type_and_content(&value, &mut registers); + let keys = registers.key.unwrap_vec(); + assert!(keys.iter().any(|key| key.as_ref() == "child")); + + let bytes = writer.finish(); + let mut reader = ValueReader::new(&bytes); + let decoded = reader + .read_value_type_and_content(&keys, ID::new(7, 2)) + .unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn reader_reports_corruption_for_truncated_or_invalid_value_payloads() { + assert!(ValueReader::new(&[]).read_i64().is_err()); + assert!(ValueReader::new(&[1, 2, 3]).read_f64().is_err()); + assert!(ValueReader::new(&[5, b'a']).read_str().is_err()); + assert!(ValueReader::new(&[5, 1, 2]).read_binary().is_err()); + + let mut writer = ValueWriter::new(); + writer.write_usize(1); + writer.write_usize(9); + writer.write_value_type_and_content(&"value".into(), &mut TestRegisters::default()); + let bytes = writer.finish(); + let mut reader = ValueReader::new(&bytes); + assert!(reader + .read_value_content(LoroValueKind::Map, &[], ID::new(1, 0)) + .is_err()); + } +} diff --git a/vendor/loro-internal/src/encoding/value_register.rs b/vendor/loro-internal/src/encoding/value_register.rs new file mode 100644 index 00000000..97ff5933 --- /dev/null +++ b/vendor/loro-internal/src/encoding/value_register.rs @@ -0,0 +1,51 @@ +use rustc_hash::FxHashMap; + +#[derive(Debug)] +pub struct ValueRegister { + map_value_to_index: FxHashMap, + vec: Vec, +} + +impl Default for ValueRegister { + fn default() -> Self { + Self::new() + } +} + +impl ValueRegister { + pub fn new() -> Self { + Self { + map_value_to_index: FxHashMap::default(), + vec: Vec::new(), + } + } + + /// Return the index of the given value. If it does not exist, + /// insert it and return the new index. + pub fn register(&mut self, key: &T) -> usize { + if let Some(index) = self.map_value_to_index.get(key) { + *index + } else { + let idx = self.vec.len(); + self.vec.push(key.clone()); + self.map_value_to_index.insert(key.clone(), idx); + idx + } + } + + pub fn get(&self, key: &T) -> Option { + self.map_value_to_index.get(key).copied() + } + + pub fn unwrap_vec(self) -> Vec { + self.vec + } + + pub fn get_value(&self, index: usize) -> Option<&T> { + self.vec.get(index) + } + + pub fn vec(&self) -> &Vec { + &self.vec + } +} diff --git a/vendor/loro-internal/src/error.rs b/vendor/loro-internal/src/error.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/vendor/loro-internal/src/error.rs @@ -0,0 +1 @@ + diff --git a/vendor/loro-internal/src/estimated_size.rs b/vendor/loro-internal/src/estimated_size.rs new file mode 100644 index 00000000..64718f56 --- /dev/null +++ b/vendor/loro-internal/src/estimated_size.rs @@ -0,0 +1,27 @@ +use loro_common::LoroValue; + +pub trait EstimatedSize { + /// Estimate the storage size of the object in bytes + fn estimate_storage_size(&self) -> usize; +} + +impl EstimatedSize for LoroValue { + fn estimate_storage_size(&self) -> usize { + match self { + LoroValue::Null => 1, + LoroValue::Bool(_) => 1, + LoroValue::Double(_) => 8, + LoroValue::I64(i) => 1 + (64 - i.leading_zeros()) as usize / 7, + LoroValue::Binary(b) => b.len() + 1, + LoroValue::String(s) => s.len() + 1, + LoroValue::List(l) => 3 + l.iter().map(|x| x.estimate_storage_size()).sum::(), + LoroValue::Map(m) => { + 3 + m + .iter() + .map(|(k, v)| k.len() + 3 + v.estimate_storage_size()) + .sum::() + } + LoroValue::Container(_) => 6, + } + } +} diff --git a/vendor/loro-internal/src/event.rs b/vendor/loro-internal/src/event.rs new file mode 100644 index 00000000..9430c3f5 --- /dev/null +++ b/vendor/loro-internal/src/event.rs @@ -0,0 +1,522 @@ +use enum_as_inner::EnumAsInner; +use itertools::Itertools; +use loro_delta::{array_vec::ArrayVec, delta_trait::DeltaAttr, DeltaItem, DeltaRope}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use crate::{ + container::richtext::richtext_state::RichtextStateChunk, + delta::{Delta, MapDelta, Meta, MovableListInnerDelta, ResolvedMapDelta, TreeDelta, TreeDiff}, + diff_calc::DiffMode, + handler::ValueOrHandler, + op::SliceWithId, + utils::string_slice::StringSlice, + InternalString, +}; + +use std::{borrow::Cow, hash::Hash}; + +use loro_common::{ContainerID, LoroValue, TreeID}; + +use crate::{container::idx::ContainerIdx, version::Frontiers}; + +#[derive(Debug, Clone)] +pub struct ContainerDiff { + pub id: ContainerID, + pub path: Vec<(ContainerID, Index)>, + pub(crate) idx: ContainerIdx, + pub is_unknown: bool, + pub diff: Diff, +} + +/// The kind of the event trigger. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum EventTriggerKind { + /// The event is triggered by a local transaction. + Local, + /// The event is triggered by importing + Import, + /// The event is triggered by checkout + Checkout, +} + +impl std::fmt::Display for EventTriggerKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EventTriggerKind::Local => write!(f, "local"), + EventTriggerKind::Import => write!(f, "import"), + EventTriggerKind::Checkout => write!(f, "checkout"), + } + } +} + +impl EventTriggerKind { + #[inline] + pub fn is_local(&self) -> bool { + matches!(self, EventTriggerKind::Local) + } + + #[inline] + pub fn is_import(&self) -> bool { + matches!(self, EventTriggerKind::Import) + } + + #[inline] + pub fn is_checkout(&self) -> bool { + matches!(self, EventTriggerKind::Checkout) + } +} + +#[derive(Debug, Clone)] +pub struct DiffEvent<'a> { + /// The receiver of the event. + pub current_target: Option, + /// A list of events that should be received by the current target. + pub events: &'a [&'a ContainerDiff], + pub event_meta: &'a DocDiff, +} + +/// It's the exposed event type. +/// It's exposed to the user. The user can use this to apply the diff to their local state. +/// +/// [DocDiff] may include the diff that calculated from several transactions and imports. +/// They all should have the same origin and local flag. +#[derive(Debug, Clone)] +pub struct DocDiff { + pub from: Frontiers, + pub to: Frontiers, + pub origin: InternalString, + pub by: EventTriggerKind, + pub diff: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct InternalContainerDiff { + pub(crate) idx: ContainerIdx, + /// Indicates whether this event was generated by a container that has been resurrected during a checkout operation. + pub(crate) bring_back: bool, + pub(crate) diff: DiffVariant, + /// This mode decides how should we apply the diff. + pub(crate) diff_mode: DiffMode, +} + +#[derive(Default, Debug, Clone, EnumAsInner)] +pub(crate) enum DiffVariant { + #[default] + None, + Internal(InternalDiff), + External(Diff), +} + +/// It's used for transmitting and recording the diff internally. +/// +/// It can be convert into a [DocDiff]. +// Internally, we need to batch the diff then calculate the event. Because +// we need to sort the diff by containers' created time, to make sure the +// the path to each container is up-to-date. +#[derive(Debug, Clone)] +pub(crate) struct InternalDocDiff<'a> { + pub(crate) origin: InternalString, + pub(crate) by: EventTriggerKind, + /// The values inside this array is in random order + pub(crate) diff: Cow<'a, [InternalContainerDiff]>, + pub(crate) new_version: Cow<'a, Frontiers>, +} + +impl InternalDocDiff<'_> { + pub fn into_owned(self) -> InternalDocDiff<'static> { + InternalDocDiff { + origin: self.origin, + by: self.by, + diff: Cow::Owned((*self.diff).to_owned()), + new_version: Cow::Owned((*self.new_version).to_owned()), + } + } + + pub fn can_merge(&self, other: &Self) -> bool { + self.by == other.by + } +} + +pub type Path = SmallVec<[Index; 4]>; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize, enum_as_inner::EnumAsInner)] +pub enum Index { + Key(InternalString), + Seq(usize), + Node(TreeID), +} + +impl std::fmt::Debug for Index { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Key(arg0) => write!(f, "Index::Key(\"{}\")", arg0), + Self::Seq(arg0) => write!(f, "Index::Seq({})", arg0), + Self::Node(arg0) => write!(f, "Index::Node({})", arg0), + } + } +} + +impl std::fmt::Display for Index { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Index::Key(key) => write!(f, "{}", key), + Index::Seq(s) => write!(f, "{}", s), + Index::Node(id) => write!(f, "{}@{}", id.peer, id.counter), + } + } +} + +impl From for Index { + fn from(s: usize) -> Self { + Index::Seq(s) + } +} + +impl TryFrom<&str> for Index { + type Error = &'static str; + fn try_from(s: &str) -> Result { + if s.is_empty() { + return Ok(Index::Key(InternalString::default())); + } + + let c = s.chars().next().unwrap(); + if c.is_ascii_digit() { + if let Ok(seq) = s.parse::() { + Ok(Index::Seq(seq)) + } else if let Ok(id) = s.try_into() { + Ok(Index::Node(id)) + } else { + Ok(Index::Key(InternalString::from(s))) + } + } else { + Ok(Index::Key(InternalString::from(s))) + } + } +} + +impl DiffVariant { + pub fn compose(self, other: Self) -> Result { + match (self, other) { + (DiffVariant::Internal(a), DiffVariant::Internal(b)) => { + Ok(DiffVariant::Internal(a.compose(b)?)) + } + (DiffVariant::External(a), DiffVariant::External(b)) => { + Ok(DiffVariant::External(a.compose(b)?)) + } + (a, _) => Err(a), + } + } + + pub fn is_empty(&self) -> bool { + match self { + DiffVariant::Internal(diff) => diff.is_empty(), + DiffVariant::External(diff) => diff.is_empty(), + DiffVariant::None => true, + } + } +} + +#[non_exhaustive] +#[derive(Clone, Debug, EnumAsInner)] +pub(crate) enum InternalDiff { + ListRaw(Delta), + /// This always uses entity indexes. + RichtextRaw(DeltaRope), + Map(MapDelta), + Tree(TreeDelta), + MovableList(MovableListInnerDelta), + #[cfg(feature = "counter")] + Counter(f64), + Unknown, +} + +impl From for DiffVariant { + fn from(diff: InternalDiff) -> Self { + DiffVariant::Internal(diff) + } +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct ListDeltaMeta { + /// Whether the content of the insert is moved from + /// a deletion in the same delta and **the value is not changed**. + /// + /// If true, this op must be a move op under the hood. + /// But an insert created by a move op doesn't necessarily + /// have this flag, because the insert content may not + /// be moved from a deletion in the same delta. + pub from_move: bool, +} + +impl Meta for ListDeltaMeta { + fn is_empty(&self) -> bool { + !self.from_move + } + + fn compose( + &mut self, + other: &Self, + type_pair: (crate::delta::DeltaType, crate::delta::DeltaType), + ) { + // We can't have two Some because we don't have `move_from` for Retain. + // And this function is only called when composing a insert/retain with a retain. + if let (crate::delta::DeltaType::Insert, crate::delta::DeltaType::Insert) = type_pair { + unreachable!() + } + + self.from_move = self.from_move || other.from_move; + } + + fn is_mergeable(&self, other: &Self) -> bool { + self.from_move == other.from_move + } + + fn merge(&mut self, _other: &Self) {} +} + +impl DeltaAttr for ListDeltaMeta { + fn compose(&mut self, other: &Self) { + self.from_move = self.from_move || other.from_move; + } + + fn attr_is_empty(&self) -> bool { + !self.from_move + } +} + +pub type ListDiffInsertItem = ArrayVec; +pub type ListDiffItem = DeltaItem; +pub type ListDiff = DeltaRope; + +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct TextMeta(pub FxHashMap); + +impl From for FxHashMap { + fn from(value: TextMeta) -> Self { + value.0 + } +} + +impl From> for TextMeta { + fn from(value: FxHashMap) -> Self { + TextMeta(value) + } +} + +pub type TextDiffItem = DeltaItem; +pub type TextDiff = DeltaRope; + +impl DeltaAttr for TextMeta { + fn compose(&mut self, other: &Self) { + for (key, value) in other.0.iter() { + self.0.insert(key.clone(), value.clone()); + } + } + + fn attr_is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// Diff is the diff between two versions of a container. +/// It's used to describe the change of a container and the events. +/// +/// # Internal +/// +/// Text index variants: +/// +/// - When `wasm` is enabled, it should use utf16 indexes. +/// - When `wasm` is disabled, it should use unicode indexes. +#[non_exhaustive] +#[derive(Clone, Debug, EnumAsInner)] +pub enum Diff { + List(ListDiff), + /// - When feature `wasm` is enabled, it should use utf16 indexes. + /// - When feature `wasm` is disabled, it should use unicode indexes. + Text(TextDiff), + Map(ResolvedMapDelta), + Tree(TreeDiff), + #[cfg(feature = "counter")] + Counter(f64), + Unknown, +} + +impl From for DiffVariant { + fn from(diff: Diff) -> Self { + DiffVariant::External(diff) + } +} + +impl InternalDiff { + pub(crate) fn is_empty(&self) -> bool { + match self { + InternalDiff::ListRaw(s) => s.is_empty(), + InternalDiff::RichtextRaw(t) => t.is_empty(), + InternalDiff::Map(m) => m.updated.is_empty(), + InternalDiff::Tree(t) => t.is_empty(), + InternalDiff::MovableList(t) => t.is_empty(), + #[cfg(feature = "counter")] + InternalDiff::Counter(c) => c.abs() < f64::EPSILON, + InternalDiff::Unknown => true, + } + } + + pub(crate) fn compose(self, diff: InternalDiff) -> Result { + // PERF: avoid clone + match (self, diff) { + (InternalDiff::ListRaw(a), InternalDiff::ListRaw(b)) => { + Ok(InternalDiff::ListRaw(a.compose(b))) + } + (InternalDiff::RichtextRaw(a), InternalDiff::RichtextRaw(b)) => { + let mut ans = a.clone(); + ans.compose(&b); + Ok(InternalDiff::RichtextRaw(ans)) + } + (InternalDiff::Map(a), InternalDiff::Map(b)) => Ok(InternalDiff::Map(a.compose(b))), + (InternalDiff::Tree(a), InternalDiff::Tree(b)) => Ok(InternalDiff::Tree(a.compose(b))), + (a, _) => Err(a), + } + } +} + +impl Diff { + pub fn compose_ref(&mut self, diff: &Diff) { + // PERF: avoid clone + match (self, diff) { + (Diff::List(a), Diff::List(b)) => { + a.compose(b); + } + (Diff::Text(a), Diff::Text(b)) => { + a.compose(b); + } + (Diff::Map(a), Diff::Map(b)) => { + *a = a.clone().compose(b.clone()); + } + (Diff::Tree(a), Diff::Tree(b)) => { + *a = a.clone().compose(b.clone()); + } + #[cfg(feature = "counter")] + (Diff::Counter(a), Diff::Counter(b)) => *a += b, + (_, _) => unreachable!(), + } + } + + pub fn compose(self, diff: Diff) -> Result { + // PERF: avoid clone + match (self, diff) { + (Diff::List(mut a), Diff::List(b)) => { + a.compose(&b); + Ok(Diff::List(a)) + } + (Diff::Text(mut a), Diff::Text(b)) => { + a.compose(&b); + Ok(Diff::Text(a)) + } + (Diff::Map(a), Diff::Map(b)) => Ok(Diff::Map(a.compose(b))), + + (Diff::Tree(a), Diff::Tree(b)) => Ok(Diff::Tree(a.compose(b))), + #[cfg(feature = "counter")] + (Diff::Counter(a), Diff::Counter(b)) => Ok(Diff::Counter(a + b)), + (a, _) => Err(a), + } + } + + // Transform this diff based on the other diff + pub fn transform(&mut self, other: &Self, left_prior: bool) { + match (self, other) { + (Diff::List(a), Diff::List(b)) => a.transform_(b, left_prior), + (Diff::Text(a), Diff::Text(b)) => a.transform_(b, left_prior), + (Diff::Map(a), Diff::Map(b)) => a.transform(b, left_prior), + (Diff::Tree(a), Diff::Tree(b)) => a.transform(b, left_prior), + #[cfg(feature = "counter")] + (Diff::Counter(_a), Diff::Counter(_b)) => { + // Counter operations commute; no transformation is needed. + } + _ => {} + } + } + + pub fn is_empty(&self) -> bool { + match self { + Diff::List(s) => s.is_empty(), + Diff::Text(t) => t.is_empty(), + Diff::Map(m) => m.updated.is_empty(), + Diff::Tree(t) => t.diff.is_empty(), + #[cfg(feature = "counter")] + Diff::Counter(c) => c.abs() < f64::EPSILON, + Diff::Unknown => true, + } + } + + #[allow(unused)] + pub(crate) fn concat(self, diff: Diff) -> Diff { + match (self, diff) { + (Diff::List(mut a), Diff::List(b)) => { + a.compose(&b); + Diff::List(a) + } + (Diff::Text(mut a), Diff::Text(b)) => { + a.compose(&b); + Diff::Text(a) + } + (Diff::Map(a), Diff::Map(b)) => { + let mut a = a; + for (k, v) in b.updated { + a = a.with_entry(k, v); + } + Diff::Map(a) + } + + (Diff::Tree(a), Diff::Tree(b)) => Diff::Tree(a.extend(b.diff)), + #[cfg(feature = "counter")] + (Diff::Counter(a), Diff::Counter(b)) => Diff::Counter(a + b), + _ => unreachable!(), + } + } + + /// Transform the cursor based on this diff + pub(crate) fn transform_cursor(&self, pos: usize, left_prior: bool) -> usize { + match self { + Diff::List(list) => list.transform_pos(pos, left_prior), + Diff::Text(text) => text.transform_pos(pos, left_prior), + _ => pos, + } + } +} + +pub fn str_to_path(s: &str) -> Option> { + s.split('/').map(|x| x.try_into()).try_collect().ok() +} + +pub fn path_to_str(path: &[Index]) -> String { + path.iter().map(|x| x.to_string()).join("/") +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use itertools::Itertools; + use loro_common::LoroValue; + + use crate::{cursor::PosType, ApplyDiff, LoroDoc}; + + #[test] + fn test_text_event() { + let loro = LoroDoc::new(); + let _g = loro.subscribe_root(Arc::new(|event| { + let mut value = LoroValue::String(Default::default()); + value.apply_diff(&event.events.iter().map(|x| x.diff.clone()).collect_vec()); + assert_eq!(value, "h223ello".into()); + })); + let mut txn = loro.txn().unwrap(); + let text = loro.get_text("id"); + text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + text.insert_with_txn(&mut txn, 1, "223", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } +} diff --git a/vendor/loro-internal/src/fork.rs b/vendor/loro-internal/src/fork.rs new file mode 100644 index 00000000..371a82c0 --- /dev/null +++ b/vendor/loro-internal/src/fork.rs @@ -0,0 +1,52 @@ +//! The fork module provides functionality to create a new LoroDoc instance at a specified version +//! (Frontiers) with minimal overhead. +//! +//! # Implementation Overview +//! +//! The `fork_at` function in this module allows for the creation of a new document that reflects +//! the state of the original document at a given version. The function achieves this by: +//! +//! ## Exporting Necessary Data: +//! +//! - **Change Store Data**: Collects all changes up to the specified version from the change +//! store's key-value (kv) data store. It includes the version vector and frontiers for accurate +//! identification of the version. +//! +//! - **Container Store Data**: Exports the container store's kv data representing the document's +//! state at the specified version. This involves checking out to the desired version, exporting +//! the state, and efficiently checking back to the latest version. +//! +//! - **GC Store Data**: If applicable, exports the gc store's kv data, ensuring that version +//! identifiers are included. +//! +//! ## Reconstructing the New Document: +//! +//! Imports the exported data into a new LoroDoc instance using optimized import mechanisms +//! similar to those used in fast snapshot imports. +//! +//! By focusing on exporting only the necessary data and optimizing state transitions during +//! version checkout, the `fork_at` function minimizes overhead and efficiently creates new +//! document instances representing past versions. +//! +use std::borrow::Cow; + +use crate::{version::Frontiers, LoroDoc}; +use loro_common::{LoroError, LoroResult}; + +impl LoroDoc { + /// Creates a new LoroDoc at a specified version (Frontiers) + pub fn fork_at(&self, frontiers: &Frontiers) -> LoroResult { + let bytes = self + .export(crate::loro::ExportMode::SnapshotAt { + version: Cow::Borrowed(frontiers), + }) + .map_err(LoroError::from)?; + let doc = LoroDoc::new(); + doc.set_config(&self.config); + if self.auto_commit.load(std::sync::atomic::Ordering::Relaxed) { + doc.start_auto_commit(); + } + doc.import(&bytes)?; + Ok(doc) + } +} diff --git a/vendor/loro-internal/src/fuzz.rs b/vendor/loro-internal/src/fuzz.rs new file mode 100644 index 00000000..6c22cddd --- /dev/null +++ b/vendor/loro-internal/src/fuzz.rs @@ -0,0 +1,89 @@ +use std::{fmt::Debug, time::Instant}; + +pub fn minify_error(site_num: u8, actions: Vec, f: F, normalize: N) +where + F: Fn(u8, &mut [T]), + N: Fn(u8, &mut [T]) -> Vec, + T: Clone + Debug, +{ + std::panic::set_hook(Box::new(|_info| { + // ignore panic output + // println!("{:?}", _info); + })); + + let f_ref: *const _ = &f; + let f_ref: usize = f_ref as usize; + #[allow(clippy::redundant_clone)] + let mut actions_clone = actions.clone(); + let action_ref: usize = (&mut actions_clone) as *mut _ as usize; + #[allow(clippy::blocks_in_conditions)] + if std::panic::catch_unwind(|| { + // SAFETY: test + let f = unsafe { &*(f_ref as *const F) }; + // SAFETY: test + let actions_ref = unsafe { &mut *(action_ref as *mut Vec) }; + f(site_num, actions_ref); + }) + .is_ok() + { + println!("No Error Found"); + return; + } + + let mut minified = actions.clone(); + let mut candidates = Vec::new(); + for i in 0..actions.len() { + let mut new = actions.clone(); + new.remove(i); + candidates.push(new); + } + + println!("Minifying..."); + let start = Instant::now(); + while let Some(candidate) = candidates.pop() { + let f_ref: *const _ = &f; + let f_ref: usize = f_ref as usize; + let mut actions_clone = candidate.clone(); + let action_ref: usize = (&mut actions_clone) as *mut _ as usize; + #[allow(clippy::blocks_in_conditions)] + if std::panic::catch_unwind(|| { + // SAFETY: test + let f = unsafe { &*(f_ref as *const F) }; + // SAFETY: test + let actions_ref = unsafe { &mut *(action_ref as *mut Vec) }; + f(site_num, actions_ref); + }) + .is_err() + { + for i in 0..candidate.len() { + let mut new = candidate.clone(); + new.remove(i); + candidates.push(new); + } + if candidate.len() < minified.len() { + minified = candidate; + println!("New min len={}", minified.len()); + } + if candidates.len() > 40 { + candidates.drain(0..30); + } + } + if start.elapsed().as_secs() > 10 && minified.len() <= 4 { + break; + } + if start.elapsed().as_secs() > 60 { + break; + } + } + + let minified = normalize(site_num, &mut minified); + println!( + "Old Length {}, New Length {}", + actions.len(), + minified.len() + ); + dbg!(&minified); + if actions.len() > minified.len() { + minify_error(site_num, minified, f, normalize); + } +} diff --git a/vendor/loro-internal/src/handler.rs b/vendor/loro-internal/src/handler.rs new file mode 100644 index 00000000..66acc5ff --- /dev/null +++ b/vendor/loro-internal/src/handler.rs @@ -0,0 +1,5821 @@ +use super::{state::DocState, txn::Transaction}; +use crate::sync::Mutex; +use crate::{ + container::{ + idx::ContainerIdx, + list::list_op::{DeleteSpan, DeleteSpanWithId, ListOp}, + richtext::{richtext_state::PosType, RichtextState, StyleKey, StyleOp, TextStyleInfoFlag}, + }, + cursor::{Cursor, Side}, + delta::{DeltaItem, Meta, StyleMeta, TreeExternalDiff}, + diff::{diff, diff_impl::UpdateTimeoutError, OperateProxy}, + event::{Diff, TextDiff, TextDiffItem, TextMeta}, + op::ListSlice, + state::{IndexType, State, TreeParentId}, + txn::EventHint, + utils::{string_slice::StringSlice, utf16::count_utf16_len}, + LoroDoc, LoroDocInner, +}; +use append_only_bytes::BytesSlice; +use enum_as_inner::EnumAsInner; +use generic_btree::rle::HasLength; +use loro_common::{ + ContainerID, ContainerType, IdFull, InternalString, LoroError, LoroResult, LoroValue, PeerID, + TreeID, ID, +}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, cmp::Reverse, collections::BinaryHeap, fmt::Debug, ops::Deref, sync::Arc}; +use tracing::{error, instrument}; + +pub use crate::diff::diff_impl::UpdateOptions; +pub use tree::TreeHandler; +mod movable_list_apply_delta; +mod tree; + +const REGULAR_CONTAINER_VALUE_ARG_ERROR: &str = + "Cannot use a LoroValue::Container as a regular value. To create child container, use insert_container or set_container"; + +mod text_update; + +fn ensure_no_regular_container_value(value: &LoroValue) -> LoroResult<()> { + let mut stack = vec![value]; + while let Some(value) = stack.pop() { + match value { + LoroValue::Container(_) => { + return Err(LoroError::ArgErr( + REGULAR_CONTAINER_VALUE_ARG_ERROR + .to_string() + .into_boxed_str(), + )); + } + LoroValue::List(list) => { + stack.extend(list.iter()); + } + LoroValue::Map(map) => { + stack.extend(map.values()); + } + LoroValue::Null + | LoroValue::Bool(_) + | LoroValue::Double(_) + | LoroValue::I64(_) + | LoroValue::Binary(_) + | LoroValue::String(_) => {} + } + } + + Ok(()) +} + +fn checked_range_end( + pos: usize, + len: usize, + container_len: usize, + info: Box, +) -> LoroResult { + let end = pos.checked_add(len).ok_or_else(|| LoroError::OutOfBound { + pos: usize::MAX, + len: container_len, + info: info.clone(), + })?; + if end > container_len { + return Err(LoroError::OutOfBound { + pos: end, + len: container_len, + info, + }); + } + + Ok(end) +} + +fn checked_delta_index_end(pos: usize, len: usize, container_len: usize) -> LoroResult { + pos.checked_add(len).ok_or_else(|| LoroError::OutOfBound { + pos: usize::MAX, + len: container_len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }) +} + +pub trait HandlerTrait: Clone + Sized { + fn is_attached(&self) -> bool; + fn attached_handler(&self) -> Option<&BasicHandler>; + fn get_value(&self) -> LoroValue; + fn get_deep_value(&self) -> LoroValue; + fn kind(&self) -> ContainerType; + fn to_handler(&self) -> Handler; + fn from_handler(h: Handler) -> Option; + fn doc(&self) -> Option; + /// This method returns an attached handler. + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult; + /// If a detached container is attached, this method will return its corresponding attached handler. + fn get_attached(&self) -> Option; + + fn parent(&self) -> Option { + self.attached_handler().and_then(|x| x.parent()) + } + + fn idx(&self) -> ContainerIdx { + self.attached_handler() + .map(|x| x.container_idx) + .unwrap_or_else(|| { + ContainerIdx::from_index_and_type(ContainerIdx::INDEX_MASK, self.kind()) + }) + } + + fn id(&self) -> ContainerID { + self.attached_handler() + .map(|x| x.id.clone()) + .unwrap_or_else(|| ContainerID::new_normal(ID::NONE_ID, self.kind())) + } + + fn with_state(&self, f: impl FnOnce(&mut State) -> LoroResult) -> LoroResult { + let inner = self + .attached_handler() + .ok_or(LoroError::MisuseDetachedContainer { + method: "with_state", + })?; + let state = inner.doc.state.clone(); + let mut guard = state.lock(); + guard.with_state_mut(inner.container_idx, f) + } +} + +fn create_handler(inner: &BasicHandler, id: ContainerID) -> Handler { + Handler::new_attached(id, inner.doc.clone()) +} + +fn value_to_value_or_handler(inner: &BasicHandler, value: LoroValue) -> ValueOrHandler { + match value { + LoroValue::Container(container_id) => { + ValueOrHandler::Handler(create_handler(inner, container_id)) + } + value => ValueOrHandler::Value(value), + } +} + +/// Flatten attributes that allow overlap +#[derive(Clone, Debug)] +pub struct BasicHandler { + id: ContainerID, + container_idx: ContainerIdx, + doc: LoroDoc, +} + +struct DetachedInner { + value: T, + /// If the handler attached later, this field will be filled. + attached: Option, +} + +impl DetachedInner { + fn new(v: T) -> Self { + Self { + value: v, + attached: None, + } + } +} + +enum MaybeDetached { + Detached(Arc>>), + Attached(BasicHandler), +} + +impl Clone for MaybeDetached { + fn clone(&self) -> Self { + match self { + MaybeDetached::Detached(a) => MaybeDetached::Detached(Arc::clone(a)), + MaybeDetached::Attached(a) => MaybeDetached::Attached(a.clone()), + } + } +} + +impl MaybeDetached { + fn new_detached(v: T) -> Self { + MaybeDetached::Detached(Arc::new(Mutex::new(DetachedInner::new(v)))) + } + + fn is_attached(&self) -> bool { + match self { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(_) => true, + } + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match self { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a), + } + } + + fn try_attached_state(&self) -> LoroResult<&BasicHandler> { + match self { + MaybeDetached::Detached(_) => Err(LoroError::MisuseDetachedContainer { + method: "inner_state", + }), + MaybeDetached::Attached(a) => Ok(a), + } + } +} + +impl From for MaybeDetached { + fn from(a: BasicHandler) -> Self { + MaybeDetached::Attached(a) + } +} + +impl BasicHandler { + pub(crate) fn doc(&self) -> LoroDoc { + self.doc.clone() + } + + #[inline] + fn with_doc_state(&self, f: impl FnOnce(&mut DocState) -> R) -> R { + let state = self.doc.state.clone(); + let mut guard = state.lock(); + f(&mut guard) + } + + fn with_txn( + &self, + f: impl FnOnce(&mut Transaction) -> Result, + ) -> Result { + with_txn(&self.doc, f) + } + + fn get_parent(&self) -> Option { + let parent_idx = self.doc.arena.get_parent(self.container_idx)?; + let parent_id = self.doc.arena.get_container_id(parent_idx).unwrap(); + { + let kind = parent_id.container_type(); + let handler = BasicHandler { + container_idx: parent_idx, + id: parent_id, + doc: self.doc.clone(), + }; + + Some(match kind { + ContainerType::Map => Handler::Map(MapHandler { + inner: handler.into(), + }), + ContainerType::List => Handler::List(ListHandler { + inner: handler.into(), + }), + ContainerType::Tree => Handler::Tree(TreeHandler { + inner: handler.into(), + }), + ContainerType::Text => Handler::Text(TextHandler { + inner: handler.into(), + }), + ContainerType::MovableList => Handler::MovableList(MovableListHandler { + inner: handler.into(), + }), + #[cfg(feature = "counter")] + ContainerType::Counter => Handler::Counter(counter::CounterHandler { + inner: handler.into(), + }), + ContainerType::Unknown(_) => unreachable!(), + }) + } + } + + pub fn get_value(&self) -> LoroValue { + self.doc.state.lock().get_value_by_idx(self.container_idx) + } + + pub fn get_deep_value(&self) -> LoroValue { + self.doc + .state + .lock() + .get_container_deep_value(self.container_idx) + } + + fn with_state(&self, f: impl FnOnce(&mut State) -> R) -> R { + let mut guard = self.doc.state.lock(); + guard.with_state_mut(self.container_idx, f) + } + + pub fn parent(&self) -> Option { + self.get_parent() + } + + fn is_deleted(&self) -> bool { + self.doc.state.lock().is_deleted(self.container_idx) + } + + fn has_decoded_state(&self) -> bool { + self.with_doc_state(|state| state.has_decoded_container_state(self.container_idx)) + } +} + +/// Flatten attributes that allow overlap +#[derive(Clone)] +pub struct TextHandler { + inner: MaybeDetached, +} + +impl HandlerTrait for TextHandler { + fn to_handler(&self) -> Handler { + Handler::Text(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let inner = create_handler(parent, self_id); + let text = inner.into_text().unwrap(); + let mut delta: Vec = Vec::new(); + for span in t.value.iter() { + delta.push(TextDelta::Insert { + insert: span.text.to_string(), + attributes: span.attributes.to_option_map(), + }); + } + + text.apply_delta_with_txn(txn, &delta)?; + t.attached = text.attached_handler().cloned(); + Ok(text) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_text().unwrap(); + + let delta = self.get_delta(); + ans.apply_delta_with_txn(txn, &delta)?; + Ok(ans) + } + } + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + LoroValue::String((t.value.to_string()).into()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + self.get_value() + } + + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn kind(&self) -> ContainerType { + ContainerType::Text + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Text(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for TextHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => { + write!(f, "TextHandler(Unattached)") + } + MaybeDetached::Attached(a) => { + write!(f, "TextHandler({:?})", &a.id) + } + } + } +} + +#[derive(Debug, Clone, EnumAsInner, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum TextDelta { + Retain { + retain: usize, + attributes: Option>, + }, + Insert { + insert: String, + attributes: Option>, + }, + Delete { + delete: usize, + }, +} + +impl TextDelta { + pub fn from_text_diff<'a>(diff: impl Iterator) -> Vec { + let mut ans = Vec::with_capacity(diff.size_hint().0); + for iter in diff { + match iter { + loro_delta::DeltaItem::Retain { len, attr } => { + ans.push(TextDelta::Retain { + retain: *len, + attributes: if attr.0.is_empty() { + None + } else { + Some(attr.0.clone()) + }, + }); + } + loro_delta::DeltaItem::Replace { + value, + attr, + delete, + } => { + if value.rle_len() > 0 { + ans.push(TextDelta::Insert { + insert: value.to_string(), + attributes: if attr.0.is_empty() { + None + } else { + Some(attr.0.clone()) + }, + }); + } + if *delete > 0 { + ans.push(TextDelta::Delete { delete: *delete }); + } + } + } + } + + ans + } + + pub fn into_text_diff(vec: impl Iterator) -> TextDiff { + let mut delta = TextDiff::new(); + for item in vec { + match item { + TextDelta::Retain { retain, attributes } => { + delta.push_retain(retain, TextMeta(attributes.unwrap_or_default().clone())); + } + TextDelta::Insert { insert, attributes } => { + delta.push_insert( + StringSlice::from(insert.as_str()), + TextMeta(attributes.unwrap_or_default()), + ); + } + TextDelta::Delete { delete } => { + delta.push_delete(delete); + } + } + } + + delta + } +} + +impl From<&DeltaItem> for TextDelta { + fn from(value: &DeltaItem) -> Self { + match value { + crate::delta::DeltaItem::Retain { retain, attributes } => TextDelta::Retain { + retain: *retain, + attributes: attributes.to_option_map(), + }, + crate::delta::DeltaItem::Insert { insert, attributes } => TextDelta::Insert { + insert: insert.to_string(), + attributes: attributes.to_option_map(), + }, + crate::delta::DeltaItem::Delete { + delete, + attributes: _, + } => TextDelta::Delete { delete: *delete }, + } + } +} + +#[derive(Clone)] +pub struct MapHandler { + inner: MaybeDetached>, +} + +impl HandlerTrait for MapHandler { + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a), + } + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + let mut map = FxHashMap::default(); + for (k, v) in m.value.iter() { + map.insert(k.to_string(), v.to_value()); + } + LoroValue::Map(map.into()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + let mut map = FxHashMap::default(); + for (k, v) in m.value.iter() { + map.insert(k.to_string(), v.to_deep_value()); + } + LoroValue::Map(map.into()) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::Map + } + + fn to_handler(&self) -> Handler { + Handler::Map(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let inner = create_handler(parent, self_id); + let map = inner.into_map().unwrap(); + for (k, v) in m.value.iter() { + match v { + ValueOrHandler::Value(v) => { + map.insert_with_txn(txn, k, v.clone())?; + } + ValueOrHandler::Handler(h) => { + map.insert_container_with_txn(txn, k, h.clone())?; + } + } + } + m.attached = map.attached_handler().cloned(); + Ok(map) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_map().unwrap(); + + for (k, v) in self.get_value().into_map().unwrap().iter() { + if let LoroValue::Container(id) = v { + ans.insert_container_with_txn(txn, k, create_handler(a, id.clone()))?; + } else { + ans.insert_with_txn(txn, k, v.clone())?; + } + } + + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Map(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for MapHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "MapHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "MapHandler {}", a.id), + } + } +} + +#[derive(Clone)] +pub struct ListHandler { + inner: MaybeDetached>, +} + +#[derive(Clone)] +pub struct MovableListHandler { + inner: MaybeDetached>, +} + +impl HandlerTrait for MovableListHandler { + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a), + } + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_deep_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::MovableList + } + + fn to_handler(&self) -> Handler { + Handler::MovableList(self.clone()) + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::MovableList(x) => Some(x), + _ => None, + } + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut l = l.lock(); + let inner = create_handler(parent, self_id); + let list = inner.into_movable_list().unwrap(); + for (index, v) in l.value.iter().enumerate() { + match v { + ValueOrHandler::Value(v) => { + list.insert_with_txn(txn, index, v.clone())?; + } + ValueOrHandler::Handler(h) => { + list.insert_container_with_txn(txn, index, h.clone())?; + } + } + } + l.attached = list.attached_handler().cloned(); + Ok(list) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_movable_list().unwrap(); + + for (i, v) in self.get_value().into_list().unwrap().iter().enumerate() { + if let LoroValue::Container(id) = v { + ans.insert_container_with_txn(txn, i, create_handler(a, id.clone()))?; + } else { + ans.insert_with_txn(txn, i, v.clone())?; + } + } + + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for MovableListHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "MovableListHandler {}", self.id()) + } +} + +impl std::fmt::Debug for ListHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "ListHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "ListHandler {}", a.id), + } + } +} + +impl HandlerTrait for ListHandler { + fn is_attached(&self) -> bool { + self.inner.is_attached() + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(a) => { + let a = a.lock(); + LoroValue::List(a.value.iter().map(|v| v.to_deep_value()).collect()) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::List + } + + fn to_handler(&self) -> Handler { + Handler::List(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut l = l.lock(); + let inner = create_handler(parent, self_id); + let list = inner.into_list().unwrap(); + for (index, v) in l.value.iter().enumerate() { + match v { + ValueOrHandler::Value(v) => { + list.insert_with_txn(txn, index, v.clone())?; + } + ValueOrHandler::Handler(h) => { + list.insert_container_with_txn(txn, index, h.clone())?; + } + } + } + l.attached = list.attached_handler().cloned(); + Ok(list) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_list().unwrap(); + + for (i, v) in self.get_value().into_list().unwrap().iter().enumerate() { + if let LoroValue::Container(id) = v { + ans.insert_container_with_txn(txn, i, create_handler(a, id.clone()))?; + } else { + ans.insert_with_txn(txn, i, v.clone())?; + } + } + + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::List(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +#[derive(Clone)] +pub struct UnknownHandler { + inner: BasicHandler, +} + +impl Debug for UnknownHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "UnknownHandler") + } +} + +impl UnknownHandler { + pub fn is_deleted(&self) -> bool { + self.inner.is_deleted() + } +} + +impl HandlerTrait for UnknownHandler { + fn is_attached(&self) -> bool { + true + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + Some(&self.inner) + } + + fn get_value(&self) -> LoroValue { + todo!() + } + + fn get_deep_value(&self) -> LoroValue { + todo!() + } + + fn kind(&self) -> ContainerType { + self.inner.id.container_type() + } + + fn to_handler(&self) -> Handler { + Handler::Unknown(self.clone()) + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Unknown(x) => Some(x), + _ => None, + } + } + + fn attach( + &self, + _txn: &mut Transaction, + _parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + let new_inner = create_handler(&self.inner, self_id); + let ans = new_inner.into_unknown().unwrap(); + Ok(ans) + } + + fn get_attached(&self) -> Option { + Some(self.clone()) + } + + fn doc(&self) -> Option { + Some(self.inner.doc()) + } +} + +#[derive(Clone, EnumAsInner, Debug)] +pub enum Handler { + Text(TextHandler), + Map(MapHandler), + List(ListHandler), + MovableList(MovableListHandler), + Tree(TreeHandler), + #[cfg(feature = "counter")] + Counter(counter::CounterHandler), + Unknown(UnknownHandler), +} + +impl HandlerTrait for Handler { + fn is_attached(&self) -> bool { + match self { + Self::Text(x) => x.is_attached(), + Self::Map(x) => x.is_attached(), + Self::List(x) => x.is_attached(), + Self::Tree(x) => x.is_attached(), + Self::MovableList(x) => x.is_attached(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.is_attached(), + Self::Unknown(x) => x.is_attached(), + } + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + match self { + Self::Text(x) => x.attached_handler(), + Self::Map(x) => x.attached_handler(), + Self::List(x) => x.attached_handler(), + Self::MovableList(x) => x.attached_handler(), + Self::Tree(x) => x.attached_handler(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.attached_handler(), + Self::Unknown(x) => x.attached_handler(), + } + } + + fn get_value(&self) -> LoroValue { + match self { + Self::Text(x) => x.get_value(), + Self::Map(x) => x.get_value(), + Self::List(x) => x.get_value(), + Self::MovableList(x) => x.get_value(), + Self::Tree(x) => x.get_value(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_value(), + Self::Unknown(x) => x.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match self { + Self::Text(x) => x.get_deep_value(), + Self::Map(x) => x.get_deep_value(), + Self::List(x) => x.get_deep_value(), + Self::MovableList(x) => x.get_deep_value(), + Self::Tree(x) => x.get_deep_value(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_deep_value(), + Self::Unknown(x) => x.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + match self { + Self::Text(x) => x.kind(), + Self::Map(x) => x.kind(), + Self::List(x) => x.kind(), + Self::MovableList(x) => x.kind(), + Self::Tree(x) => x.kind(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.kind(), + Self::Unknown(x) => x.kind(), + } + } + + fn to_handler(&self) -> Handler { + match self { + Self::Text(x) => x.to_handler(), + Self::Map(x) => x.to_handler(), + Self::List(x) => x.to_handler(), + Self::MovableList(x) => x.to_handler(), + Self::Tree(x) => x.to_handler(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.to_handler(), + Self::Unknown(x) => x.to_handler(), + } + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match self { + Self::Text(x) => Ok(Handler::Text(x.attach(txn, parent, self_id)?)), + Self::Map(x) => Ok(Handler::Map(x.attach(txn, parent, self_id)?)), + Self::List(x) => Ok(Handler::List(x.attach(txn, parent, self_id)?)), + Self::MovableList(x) => Ok(Handler::MovableList(x.attach(txn, parent, self_id)?)), + Self::Tree(x) => Ok(Handler::Tree(x.attach(txn, parent, self_id)?)), + #[cfg(feature = "counter")] + Self::Counter(x) => Ok(Handler::Counter(x.attach(txn, parent, self_id)?)), + Self::Unknown(x) => Ok(Handler::Unknown(x.attach(txn, parent, self_id)?)), + } + } + + fn get_attached(&self) -> Option { + match self { + Self::Text(x) => x.get_attached().map(Handler::Text), + Self::Map(x) => x.get_attached().map(Handler::Map), + Self::List(x) => x.get_attached().map(Handler::List), + Self::MovableList(x) => x.get_attached().map(Handler::MovableList), + Self::Tree(x) => x.get_attached().map(Handler::Tree), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_attached().map(Handler::Counter), + Self::Unknown(x) => x.get_attached().map(Handler::Unknown), + } + } + + fn from_handler(h: Handler) -> Option { + Some(h) + } + + fn doc(&self) -> Option { + match self { + Self::Text(x) => x.doc(), + Self::Map(x) => x.doc(), + Self::List(x) => x.doc(), + Self::MovableList(x) => x.doc(), + Self::Tree(x) => x.doc(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.doc(), + Self::Unknown(x) => x.doc(), + } + } +} + +impl Handler { + pub(crate) fn new_attached(id: ContainerID, doc: LoroDoc) -> Self { + let kind = id.container_type(); + let handler = BasicHandler { + container_idx: doc.arena.register_container(&id), + id, + doc, + }; + + match kind { + ContainerType::Map => Self::Map(MapHandler { + inner: handler.into(), + }), + ContainerType::List => Self::List(ListHandler { + inner: handler.into(), + }), + ContainerType::Tree => Self::Tree(TreeHandler { + inner: handler.into(), + }), + ContainerType::Text => Self::Text(TextHandler { + inner: handler.into(), + }), + ContainerType::MovableList => Self::MovableList(MovableListHandler { + inner: handler.into(), + }), + #[cfg(feature = "counter")] + ContainerType::Counter => Self::Counter(counter::CounterHandler { + inner: handler.into(), + }), + ContainerType::Unknown(_) => Self::Unknown(UnknownHandler { inner: handler }), + } + } + + #[allow(unused)] + pub(crate) fn new_unattached(kind: ContainerType) -> Self { + match kind { + ContainerType::Text => Self::Text(TextHandler::new_detached()), + ContainerType::Map => Self::Map(MapHandler::new_detached()), + ContainerType::List => Self::List(ListHandler::new_detached()), + ContainerType::Tree => Self::Tree(TreeHandler::new_detached()), + ContainerType::MovableList => Self::MovableList(MovableListHandler::new_detached()), + #[cfg(feature = "counter")] + ContainerType::Counter => Self::Counter(counter::CounterHandler::new_detached()), + ContainerType::Unknown(_) => unreachable!(), + } + } + + pub fn id(&self) -> ContainerID { + match self { + Self::Map(x) => x.id(), + Self::List(x) => x.id(), + Self::Text(x) => x.id(), + Self::Tree(x) => x.id(), + Self::MovableList(x) => x.id(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.id(), + Self::Unknown(x) => x.id(), + } + } + + pub(crate) fn container_idx(&self) -> ContainerIdx { + match self { + Self::Map(x) => x.idx(), + Self::List(x) => x.idx(), + Self::Text(x) => x.idx(), + Self::Tree(x) => x.idx(), + Self::MovableList(x) => x.idx(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.idx(), + Self::Unknown(x) => x.idx(), + } + } + + pub fn c_type(&self) -> ContainerType { + match self { + Self::Map(_) => ContainerType::Map, + Self::List(_) => ContainerType::List, + Self::Text(_) => ContainerType::Text, + Self::Tree(_) => ContainerType::Tree, + Self::MovableList(_) => ContainerType::MovableList, + #[cfg(feature = "counter")] + Self::Counter(_) => ContainerType::Counter, + Self::Unknown(x) => x.id().container_type(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match self { + Self::Map(x) => x.get_deep_value(), + Self::List(x) => x.get_deep_value(), + Self::MovableList(x) => x.get_deep_value(), + Self::Text(x) => x.get_deep_value(), + Self::Tree(x) => x.get_deep_value(), + #[cfg(feature = "counter")] + Self::Counter(x) => x.get_deep_value(), + Self::Unknown(x) => x.get_deep_value(), + } + } + + pub(crate) fn apply_diff( + &self, + diff: Diff, + container_remap: &mut FxHashMap, + ) -> LoroResult<()> { + // In this method we will not clone the values of the containers if + // they are remapped. It's the caller's duty to do so + let on_container_remap = &mut |old_id, new_id| { + if old_id != new_id { + container_remap.insert(old_id, new_id); + } + }; + match self { + Self::Map(x) => { + let diff = match diff { + crate::event::Diff::Map(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for map container".into(), + )); + } + }; + for (key, value) in diff.updated.into_iter() { + match value.value { + Some(ValueOrHandler::Handler(h)) => { + let old_id = h.id(); + let new_h = x.insert_container( + &key, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_h.id(); + on_container_remap(old_id, new_id); + } + Some(ValueOrHandler::Value(LoroValue::Container(old_id))) => { + let new_h = x.insert_container( + &key, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_h.id(); + on_container_remap(old_id, new_id); + } + Some(ValueOrHandler::Value(v)) => { + x.insert_without_skipping(&key, v)?; + } + None => { + x.delete(&key)?; + } + } + } + } + Self::Text(x) => { + let delta = match diff { + crate::event::Diff::Text(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for text container".into(), + )); + } + }; + x.apply_delta(&TextDelta::from_text_diff(delta.iter()))?; + } + Self::List(x) => { + let delta = match diff { + crate::event::Diff::List(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for list container".into(), + )); + } + }; + x.apply_delta(delta, on_container_remap)?; + } + Self::MovableList(x) => { + let delta = match diff { + crate::event::Diff::List(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for movable list container".into(), + )); + } + }; + x.apply_delta(delta, container_remap)?; + } + Self::Tree(x) => { + fn remap_tree_id( + id: &mut TreeID, + container_remap: &FxHashMap, + ) { + let mut remapped = false; + let mut map_id = id.associated_meta_container(); + while let Some(rid) = container_remap.get(&map_id) { + remapped = true; + map_id = rid.clone(); + } + if remapped { + *id = TreeID::new( + *map_id.as_normal().unwrap().0, + *map_id.as_normal().unwrap().1, + ) + } + } + let tree_diff = match diff { + crate::event::Diff::Tree(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for tree container".into(), + )); + } + }; + for diff in tree_diff.diff { + let mut target = diff.target; + match diff.action { + TreeExternalDiff::Create { + mut parent, + index: _, + position, + } => { + if let TreeParentId::Node(p) = &mut parent { + remap_tree_id(p, container_remap) + } + remap_tree_id(&mut target, container_remap); + if !x.is_node_unexist(&target) && !x.is_node_deleted(&target)? { + // 1@0 is the parent of 2@1 + // ┌────┐ ┌───────────────┐ + // │xxxx│◀───│Move 2@1 to 0@0◀┐ + // └────┘ └───────────────┘│ + // ┌───────┐ │ ┌────────┐ + // │Del 1@0│◀─────────────────┴─│Meta 2@1│ ◀─── undo 2 ops redo 2 ops + // └───────┘ └────────┘ + // + // When we undo the delete operation, we should not create a new tree node and its child. + // However, the concurrent operation has moved the child to another parent. It's still alive. + // So when we redo the delete operation, we should check if the target is still alive. + // If it's alive, we should move it back instead of creating new one. + x.move_at_with_target_for_apply_diff(parent, position, target)?; + } else { + let new_target = x.__internal__next_tree_id(); + if x.create_at_with_target_for_apply_diff( + parent, position, new_target, + )? { + container_remap.insert( + target.associated_meta_container(), + new_target.associated_meta_container(), + ); + } + } + } + TreeExternalDiff::Move { + mut parent, + index: _, + position, + old_parent: _, + old_index: _, + } => { + if let TreeParentId::Node(p) = &mut parent { + remap_tree_id(p, container_remap) + } + remap_tree_id(&mut target, container_remap); + // determine if the target is deleted + if x.is_node_unexist(&target) || x.is_node_deleted(&target)? { + // create the target node, we should use the new target id + let new_target = x.__internal__next_tree_id(); + if x.create_at_with_target_for_apply_diff( + parent, position, new_target, + )? { + container_remap.insert( + target.associated_meta_container(), + new_target.associated_meta_container(), + ); + } + } else { + x.move_at_with_target_for_apply_diff(parent, position, target)?; + } + } + TreeExternalDiff::Delete { .. } => { + remap_tree_id(&mut target, container_remap); + if !x.is_node_deleted(&target)? { + x.delete(target)?; + } + } + } + } + } + #[cfg(feature = "counter")] + Self::Counter(x) => { + let delta = match diff { + crate::event::Diff::Counter(d) => d, + _ => { + return Err(LoroError::DecodeError( + "Invalid diff type for counter container".into(), + )); + } + }; + x.increment(delta)?; + } + Self::Unknown(_) => { + // do nothing + } + } + + Ok(()) + } + + pub fn clear(&self) -> LoroResult<()> { + match self { + Handler::Text(text_handler) => text_handler.clear(), + Handler::Map(map_handler) => map_handler.clear(), + Handler::List(list_handler) => list_handler.clear(), + Handler::MovableList(movable_list_handler) => movable_list_handler.clear(), + Handler::Tree(tree_handler) => tree_handler.clear(), + #[cfg(feature = "counter")] + Handler::Counter(counter_handler) => counter_handler.clear(), + Handler::Unknown(_unknown_handler) => Ok(()), + } + } +} + +#[derive(Clone, EnumAsInner, Debug)] +pub enum ValueOrHandler { + Value(LoroValue), + Handler(Handler), +} + +impl ValueOrHandler { + pub(crate) fn from_value(value: LoroValue, doc: &Arc) -> Self { + if let LoroValue::Container(c) = value { + ValueOrHandler::Handler(Handler::new_attached(c, LoroDoc::from_inner(doc.clone()))) + } else { + ValueOrHandler::Value(value) + } + } + + pub(crate) fn to_value(&self) -> LoroValue { + match self { + Self::Value(v) => v.clone(), + Self::Handler(h) => LoroValue::Container(h.id().clone()), + } + } + + pub(crate) fn to_deep_value(&self) -> LoroValue { + match self { + Self::Value(v) => v.clone(), + Self::Handler(h) => h.get_deep_value(), + } + } +} + +impl From for ValueOrHandler { + fn from(value: LoroValue) -> Self { + ValueOrHandler::Value(value) + } +} + +impl TextHandler { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(RichtextState::default()), + } + } + + /// Get the version id of the richtext + /// + /// This can be used to detect whether the richtext is changed + pub fn version_id(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + Some(a.with_state(|state| state.as_richtext_state_mut().unwrap().get_version_id())) + } + } + } + + pub fn get_richtext_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_richtext_value() + } + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().get_richtext_value()) + } + } + } + + /// §act-eleven A11.10 (flowstate vendor patch): the streaming twin of + /// [`Self::get_richtext_value`] — visits every text span as + /// `(text, styles)` without allocating the intermediate delta + /// `Vec` (or, downstream in the `loro` facade, a second + /// `Vec` copy of every string). Spans arrive UNMERGED, one per + /// text chunk; delta-shaped consumers merge adjacent spans whose visible + /// attributes are equal ([`StyleMeta::visible_eq`]) — the exact merge law + /// `get_richtext_value` applies. + pub fn for_each_richtext_span(&self, f: &mut dyn FnMut(&str, &crate::delta::StyleMeta)) { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.for_each_richtext_span(f); + } + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_richtext_state_mut() + .unwrap() + .for_each_richtext_span(f) + }), + } + } + + /// FLOWSTATE vendor patch #26 (§A13.4.2): batch cursor resolution — ONE + /// state acquisition for N `(event_index, side)` queries instead of one + /// lock + tree setup per call (a 5k-paragraph search reindex made ~21k + /// `get_cursor` calls; the per-call overhead dominated). Per-query + /// semantics are identical to `get_cursor` (event-index flavor). + pub fn get_cursors_batch(&self, queries: &[(usize, Side)]) -> Vec> { + match &self.inner { + MaybeDetached::Detached(_) => queries.iter().map(|_| None).collect(), + MaybeDetached::Attached(a) => { + let container = self.id(); + a.with_state(|s| { + let s = s.as_richtext_state_mut().unwrap(); + let len = s.len_event(); + queries + .iter() + .map(|&(index, side)| { + if len == 0 { + return Some(Cursor { + id: None, + container: container.clone(), + side: if side == Side::Middle { Side::Left } else { side }, + origin_pos: 0, + }); + } + if len <= index { + return Some(Cursor { + id: None, + container: container.clone(), + side: Side::Right, + origin_pos: len, + }); + } + let id = s.get_stable_position(index, true)?; + let origin_pos = s.event_index_to_unicode_index(index); + Some(Cursor { + id: Some(id), + container: container.clone(), + side, + origin_pos, + }) + }) + .collect() + }) + } + } + } + + pub fn is_empty(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.is_empty(), + MaybeDetached::Attached(a) if a.has_decoded_state() => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().is_empty()) + } + MaybeDetached::Attached(a) => a.get_value().as_string().unwrap().is_empty(), + } + } + + pub fn len_utf8(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.len_utf8() + } + MaybeDetached::Attached(a) if a.has_decoded_state() => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().len_utf8()) + } + MaybeDetached::Attached(a) => a.get_value().as_string().unwrap().len(), + } + } + + pub fn len_utf16(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.len_utf16() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_text_utf16_len(a.container_idx)) + } + } + } + + pub fn len_unicode(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.len_unicode() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_text_unicode_len(a.container_idx)) + } + } + } + + /// if `wasm` feature is enabled, it is a UTF-16 length + /// otherwise, it is a Unicode length + pub fn len_event(&self) -> usize { + if cfg!(feature = "wasm") { + self.len_utf16() + } else { + self.len_unicode() + } + } + + fn len(&self, pos_type: PosType) -> usize { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.len(pos_type), + MaybeDetached::Attached(a) if a.has_decoded_state() || pos_type == PosType::Entity => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().len(pos_type)) + } + MaybeDetached::Attached(a) => match pos_type { + PosType::Bytes => a.get_value().as_string().unwrap().len(), + PosType::Unicode => { + a.with_doc_state(|state| state.get_text_unicode_len(a.container_idx)) + } + PosType::Utf16 => { + a.with_doc_state(|state| state.get_text_utf16_len(a.container_idx)) + } + PosType::Event if cfg!(feature = "wasm") => { + a.with_doc_state(|state| state.get_text_utf16_len(a.container_idx)) + } + PosType::Event => { + a.with_doc_state(|state| state.get_text_unicode_len(a.container_idx)) + } + PosType::Entity => unreachable!("entity length is handled by the state path"), + }, + } + } + + fn validate_text_boundary(&self, pos: usize, pos_type: PosType) -> LoroResult<()> { + let err = match pos_type { + PosType::Bytes => Some(LoroError::UTF8InUnicodeCodePoint { pos }), + PosType::Utf16 => Some(LoroError::UTF16InUnicodeCodePoint { pos }), + PosType::Event if cfg!(feature = "wasm") => { + Some(LoroError::UTF16InUnicodeCodePoint { pos }) + } + _ => None, + }; + + let Some(err) = err else { + return Ok(()); + }; + + if pos > self.len(pos_type) { + return Ok(()); + } + + let Some(unicode_pos) = self.convert_pos(pos, pos_type, PosType::Unicode) else { + return Err(err); + }; + if self.convert_pos(unicode_pos, PosType::Unicode, pos_type) != Some(pos) { + return Err(err); + } + + Ok(()) + } + + pub fn diagnose(&self) { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.diagnose(); + } + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().diagnose()); + } + } + } + + pub fn iter(&self, mut callback: impl FnMut(&str) -> bool) { + // Do not call user callbacks while holding the state lock; callbacks may re-enter Loro. + let spans: Vec = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value + .iter() + .map(|span| span.text.as_str().to_owned()) + .collect() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let mut spans = Vec::new(); + state.as_richtext_state_mut().unwrap().iter(|span| { + spans.push(span.to_owned()); + true + }); + spans + }), + }; + + for span in spans { + if !callback(span.as_str()) { + return; + } + } + } + + /// Get a character at `pos` in the coordinate system specified by `pos_type`. + pub fn char_at(&self, pos: usize, pos_type: PosType) -> LoroResult { + let len = self.len(pos_type); + if pos >= len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + if let Ok(c) = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + let event_pos = match pos_type { + PosType::Event => pos, + _ => t.value.index_to_event_index(pos, pos_type), + }; + t.value.get_char_by_event_index(event_pos) + } + MaybeDetached::Attached(a) if a.has_decoded_state() || pos_type == PosType::Entity => a + .with_state(|state| { + // §perf-heaven T1: `char_at_pos` answers a unicode `pos` from + // the Src loader without forcing the `Src -> Dst` build, so the + // projection's object-anchor validation keeps object flows on + // the cold-decode fast path (Dst path is the exact prior code). + state.as_richtext_state_mut().unwrap().char_at_pos(pos, pos_type) + }), + MaybeDetached::Attached(a) => { + return text_char_at(a.get_value().as_string().unwrap(), pos, pos_type); + } + } { + Ok(c) + } else { + Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }) + } + } + + /// `start_index` and `end_index` are Event Index: + /// + /// - if feature="wasm", pos is a UTF-16 index + /// - if feature!="wasm", pos is a Unicode index + /// + pub fn slice( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult { + self.slice_with_pos_type(start_index, end_index, pos_type) + } + + pub fn slice_utf16(&self, start_index: usize, end_index: usize) -> LoroResult { + self.slice(start_index, end_index, PosType::Utf16) + } + + fn slice_with_pos_type( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult { + if end_index < start_index { + return Err(LoroError::EndIndexLessThanStartIndex { + start: start_index, + end: end_index, + }); + } + if start_index == end_index { + return Ok(String::new()); + } + + let info = || format!("Position: {}:{}", file!(), line!()).into_boxed_str(); + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + let len = t.value.len(pos_type); + if end_index > len { + return Err(LoroError::OutOfBound { + pos: end_index, + len, + info: info(), + }); + } + let (start, end) = match pos_type { + PosType::Event => (start_index, end_index), + _ => ( + t.value.index_to_event_index(start_index, pos_type), + t.value.index_to_event_index(end_index, pos_type), + ), + }; + t.value.get_text_slice_by_event_index(start, end - start) + } + MaybeDetached::Attached(a) if a.has_decoded_state() || pos_type == PosType::Entity => a + .with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + let len = state.len(pos_type); + if end_index > len { + return Err(LoroError::OutOfBound { + pos: end_index, + len, + info: info(), + }); + } + let (start, end) = match pos_type { + PosType::Event => (start_index, end_index), + _ => ( + state.index_to_event_index(start_index, pos_type), + state.index_to_event_index(end_index, pos_type), + ), + }; + state.get_text_slice_by_event_index(start, end - start) + }), + MaybeDetached::Attached(a) => text_slice( + a.get_value().as_string().unwrap(), + start_index, + end_index, + pos_type, + ) + .map_err(|err| match err { + LoroError::OutOfBound { pos, len, .. } => LoroError::OutOfBound { + pos, + len, + info: info(), + }, + err => err, + }), + } + } + + pub fn slice_delta( + &self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult> { + if end_index < start_index { + return Err(LoroError::EndIndexLessThanStartIndex { + start: start_index, + end: end_index, + }); + } + if start_index == end_index { + return Ok(Vec::new()); + } + + let len = self.len(pos_type); + if end_index > len { + return Err(LoroError::OutOfBound { + pos: end_index, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + self.validate_text_boundary(start_index, pos_type)?; + self.validate_text_boundary(end_index, pos_type)?; + + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + let ans = t.value.slice_delta(start_index, end_index, pos_type)?; + Ok(ans + .into_iter() + .map(|(s, a)| TextDelta::Insert { + insert: s, + attributes: a.to_option_map_without_null_value(), + }) + .collect()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let ans = state.as_richtext_state_mut().unwrap().slice_delta( + start_index, + end_index, + pos_type, + )?; + Ok(ans + .into_iter() + .map(|(s, a)| TextDelta::Insert { + insert: s, + attributes: a.to_option_map_without_null_value(), + }) + .collect()) + }), + } + } + + /// `pos` is a Event Index: + /// + /// - if feature="wasm", pos is a UTF-16 index + /// - if feature!="wasm", pos is a Unicode index + /// + /// This method requires auto_commit to be enabled. + pub fn splice(&self, pos: usize, len: usize, s: &str, pos_type: PosType) -> LoroResult { + let end = checked_range_end( + pos, + len, + self.len(pos_type), + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + let x = self.slice(pos, end, pos_type)?; + self.delete(pos, len, pos_type)?; + self.insert(pos, s, pos_type)?; + Ok(x) + } + + pub fn splice_utf8(&self, pos: usize, len: usize, s: &str) -> LoroResult<()> { + // let x = self.slice(pos, pos + len)?; + self.delete_utf8(pos, len)?; + self.insert_utf8(pos, s)?; + Ok(()) + } + + pub fn splice_utf16(&self, pos: usize, len: usize, s: &str) -> LoroResult<()> { + self.delete(pos, len, PosType::Utf16)?; + self.insert(pos, s, PosType::Utf16)?; + Ok(()) + } + + /// Insert text at `pos` using the given `pos_type` coordinate system. + /// + /// This method requires auto_commit to be enabled. + pub fn insert(&self, pos: usize, s: &str, pos_type: PosType) -> LoroResult<()> { + let len = self.len(pos_type); + if pos > len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + self.validate_text_boundary(pos, pos_type)?; + + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let (index, _) = t + .value + .get_entity_index_for_text_insert(pos, pos_type) + .unwrap(); + t.value.insert_at_entity_index( + index, + BytesSlice::from_bytes(s.as_bytes()), + IdFull::NONE_ID, + ); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, pos, s, pos_type)) + } + } + } + + pub fn insert_utf8(&self, pos: usize, s: &str) -> LoroResult<()> { + self.insert(pos, s, PosType::Bytes) + } + + pub fn insert_utf16(&self, pos: usize, s: &str) -> LoroResult<()> { + self.insert(pos, s, PosType::Utf16) + } + + pub fn insert_unicode(&self, pos: usize, s: &str) -> LoroResult<()> { + self.insert(pos, s, PosType::Unicode) + } + + /// Insert text within an existing transaction using the provided `pos_type`. + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + s: &str, + pos_type: PosType, + ) -> LoroResult<()> { + self.insert_with_txn_and_attr(txn, pos, s, None, pos_type)?; + Ok(()) + } + + pub fn insert_with_txn_utf8( + &self, + txn: &mut Transaction, + pos: usize, + s: &str, + ) -> LoroResult<()> { + self.insert_with_txn(txn, pos, s, PosType::Bytes) + } + + /// Delete a span using the coordinate system described by `pos_type`. + /// + /// This method requires auto_commit to be enabled. + pub fn delete(&self, pos: usize, len: usize, pos_type: PosType) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let text_len = self.len(pos_type); + let end = checked_range_end( + pos, + len, + text_len, + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + self.validate_text_boundary(pos, pos_type)?; + self.validate_text_boundary(end, pos_type)?; + + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let ranges = t.value.get_text_entity_ranges(pos, len, pos_type)?; + for range in ranges.iter().rev() { + t.value + .drain_by_entity_index(range.entity_start, range.entity_len(), None); + } + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.delete_with_txn(txn, pos, len, pos_type)) + } + } + } + + pub fn delete_utf8(&self, pos: usize, len: usize) -> LoroResult<()> { + self.delete(pos, len, PosType::Bytes) + } + + pub fn delete_utf16(&self, pos: usize, len: usize) -> LoroResult<()> { + self.delete(pos, len, PosType::Utf16) + } + + pub fn delete_unicode(&self, pos: usize, len: usize) -> LoroResult<()> { + self.delete(pos, len, PosType::Unicode) + } + + /// If attr is specified, it will be used as the attribute of the inserted text. + /// It will override the existing attribute of the text. + fn insert_with_txn_and_attr( + &self, + txn: &mut Transaction, + pos: usize, + s: &str, + attr: Option<&FxHashMap>, + pos_type: PosType, + ) -> Result, LoroError> { + if s.is_empty() { + return Ok(Vec::new()); + } + + match pos_type { + PosType::Event => { + if pos > self.len_event() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_event(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + PosType::Bytes => { + if pos > self.len_utf8() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_utf8(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + PosType::Unicode => { + if pos > self.len_unicode() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_unicode(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + PosType::Entity => {} + PosType::Utf16 => { + if pos > self.len_utf16() { + return Err(LoroError::OutOfBound { + pos, + len: self.len_utf16(), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + } + } + self.validate_text_boundary(pos, pos_type)?; + + let inner = self.inner.try_attached_state()?; + let (entity_index, event_index, styles) = inner.with_state(|state| { + let richtext_state = state.as_richtext_state_mut().unwrap(); + let ret = richtext_state.get_entity_index_for_text_insert(pos, pos_type); + let (entity_index, cursor) = match ret { + Err(_) => match pos_type { + PosType::Bytes => { + return ( + Err(LoroError::UTF8InUnicodeCodePoint { pos }), + 0, + StyleMeta::empty(), + ); + } + PosType::Utf16 | PosType::Event => { + return ( + Err(LoroError::UTF16InUnicodeCodePoint { pos }), + 0, + StyleMeta::empty(), + ); + } + _ => unreachable!(), + }, + Ok(x) => x, + }; + let event_index = if let Some(cursor) = cursor { + if pos_type == PosType::Event { + debug_assert_eq!( + richtext_state.get_event_index_by_cursor(cursor), + pos, + "pos={} cursor={:?} state={:#?}", + pos, + cursor, + &richtext_state + ); + pos + } else { + richtext_state.get_event_index_by_cursor(cursor) + } + } else { + assert_eq!(entity_index, 0); + 0 + }; + let styles = richtext_state.get_styles_at_entity_index(entity_index); + (Ok(entity_index), event_index, styles) + }); + + let entity_index = match entity_index { + Err(x) => return Err(x), + _ => entity_index.unwrap(), + }; + + let mut override_styles = Vec::new(); + if let Some(attr) = attr { + // current styles + let map: FxHashMap<_, _> = styles.iter().map(|x| (x.0.clone(), x.1.data)).collect(); + for (key, style) in map.iter() { + match attr.get(key.deref()) { + Some(v) if v == style => {} + new_style_value => { + // need to override + let new_style_value = new_style_value.cloned().unwrap_or(LoroValue::Null); + override_styles.push((key.clone(), new_style_value)); + } + } + } + + for (key, style) in attr.iter() { + let key = key.as_str().into(); + if !map.contains_key(&key) { + override_styles.push((key, style.clone())); + } + } + } + + let unicode_len = s.chars().count(); + let event_len = if cfg!(feature = "wasm") { + count_utf16_len(s.as_bytes()) + } else { + unicode_len + }; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawStr { + str: Cow::Borrowed(s), + unicode_len, + }, + pos: entity_index, + }), + EventHint::InsertText { + pos: event_index as u32, + styles, + unicode_len: unicode_len as u32, + event_len: event_len as u32, + }, + &inner.doc, + )?; + + Ok(override_styles) + } + + /// Delete text within a transaction using the specified `pos_type`. + pub fn delete_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + len: usize, + pos_type: PosType, + ) -> LoroResult<()> { + self.delete_with_txn_inline(txn, pos, len, pos_type) + } + + fn delete_with_txn_inline( + &self, + txn: &mut Transaction, + pos: usize, + len: usize, + pos_type: PosType, + ) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let text_len = self.len(pos_type); + let end = checked_range_end( + pos, + len, + text_len, + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + ) + .inspect_err(|_| error!("pos={} len={} len={}", pos, len, text_len))?; + self.validate_text_boundary(pos, pos_type)?; + self.validate_text_boundary(end, pos_type)?; + + let inner = self.inner.try_attached_state()?; + let s = tracing::span!(tracing::Level::INFO, "delete", "pos={} len={}", pos, len); + let _e = s.enter(); + let mut event_pos = 0; + let mut event_len = 0; + let ranges = inner.with_state(|state| { + let richtext_state = state.as_richtext_state_mut().unwrap(); + event_pos = richtext_state.index_to_event_index(pos, pos_type); + let event_end = richtext_state.index_to_event_index(end, pos_type); + event_len = event_end - event_pos; + + richtext_state.get_text_entity_ranges_in_event_index_range(event_pos, event_len) + })?; + + //debug_assert_eq!(ranges.iter().map(|x| x.event_len).sum::(), len); + let delete_timing = std::env::var_os("LORO_DELETE_TIMING").is_some(); + let apply_started = std::time::Instant::now(); + let range_count = ranges.len(); + let pos = event_pos as isize; + let len = event_len as isize; + let mut event_end = pos + len; + // §perf (act three D.2): a delete spanning many style anchors fans out + // into one op per text fragment; batch the per-op transaction ceremony + // for large fan-outs (see `Transaction::apply_local_ops_batch`). + const DELETE_BATCH_THRESHOLD: usize = 8; + if range_count > DELETE_BATCH_THRESHOLD { + let mut ops = Vec::with_capacity(range_count); + for range in ranges.iter().rev() { + let event_start = event_end - range.event_len as isize; + ops.push(( + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + range.id_start, + range.entity_start as isize, + range.entity_len() as isize, + ))), + EventHint::DeleteText { + span: DeleteSpan { + pos: event_start, + signed_len: range.event_len as isize, + }, + unicode_len: range.entity_len(), + }, + )); + event_end = event_start; + } + txn.apply_local_ops_batch(inner.container_idx, ops, &inner.doc)?; + } else { + for range in ranges.iter().rev() { + let event_start = event_end - range.event_len as isize; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + range.id_start, + range.entity_start as isize, + range.entity_len() as isize, + ))), + EventHint::DeleteText { + span: DeleteSpan { + pos: event_start, + signed_len: range.event_len as isize, + }, + unicode_len: range.entity_len(), + }, + &inner.doc, + )?; + event_end = event_start; + } + } + if delete_timing { + eprintln!( + "[loro-delete-timing] ranges={range_count} event_len={event_len} apply_loop={:?}", + apply_started.elapsed() + ); + } + + Ok(()) + } + + /// `start` and `end` are interpreted using `pos_type`. + /// + /// This method requires auto_commit to be enabled. + pub fn mark( + &self, + start: usize, + end: usize, + key: impl Into, + value: LoroValue, + pos_type: PosType, + ) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut g = t.lock(); + self.mark_for_detached(&mut g.value, key, &value, start, end, pos_type) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.mark_with_txn(txn, start, end, key, value, pos_type)) + } + } + } + + fn mark_for_detached( + &self, + state: &mut RichtextState, + key: impl Into, + value: &LoroValue, + start: usize, + end: usize, + pos_type: PosType, + ) -> Result<(), LoroError> { + let key: InternalString = key.into(); + let is_delete = matches!(value, &LoroValue::Null); + if start >= end { + return Err(loro_common::LoroError::ArgErr( + "Start must be less than end".to_string().into_boxed_str(), + )); + } + ensure_no_regular_container_value(value)?; + + let len = state.len(pos_type); + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + let (entity_range, styles) = + state.get_entity_range_and_text_styles_at_range(start..end, pos_type); + if let Some(styles) = styles { + if styles.has_key_value(&key, value) { + return Ok(()); + } + } + + let has_target_style = + state.range_has_style_key(entity_range.clone(), &StyleKey::Key(key.clone())); + let missing_style_key = is_delete && !has_target_style; + + if missing_style_key { + return Ok(()); + } + + let style_op = Arc::new(StyleOp { + lamport: 0, + peer: 0, + cnt: 0, + key, + value: value.clone(), + // TODO: describe this behavior in the document + info: if is_delete { + TextStyleInfoFlag::BOLD.to_delete() + } else { + TextStyleInfoFlag::BOLD + }, + }); + state.mark_with_entity_index(entity_range, style_op); + Ok(()) + } + + /// `start` and `end` are interpreted using `pos_type`. + pub fn unmark( + &self, + start: usize, + end: usize, + key: impl Into, + pos_type: PosType, + ) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => self.mark_for_detached( + &mut t.lock().value, + key, + &LoroValue::Null, + start, + end, + pos_type, + ), + MaybeDetached::Attached(a) => a.with_txn(|txn| { + self.mark_with_txn(txn, start, end, key, LoroValue::Null, pos_type) + }), + } + } + + /// `start` and `end` are interpreted using `pos_type`. + pub fn mark_with_txn( + &self, + txn: &mut Transaction, + start: usize, + end: usize, + key: impl Into, + value: LoroValue, + pos_type: PosType, + ) -> LoroResult<()> { + if start >= end { + return Err(loro_common::LoroError::ArgErr( + "Start must be less than end".to_string().into_boxed_str(), + )); + } + ensure_no_regular_container_value(&value)?; + + let inner = self.inner.try_attached_state()?; + let key: InternalString = key.into(); + let is_delete = matches!(&value, &LoroValue::Null); + + let mut doc_state = inner.doc.state.lock(); + let len = doc_state.with_state_mut(inner.container_idx, |state| { + state.as_richtext_state_mut().unwrap().len(pos_type) + }); + + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let (entity_range, skip, missing_style_key, event_start, event_end) = doc_state + .with_state_mut(inner.container_idx, |state| { + let state = state.as_richtext_state_mut().unwrap(); + let event_start = state.index_to_event_index(start, pos_type); + let event_end = state.index_to_event_index(end, pos_type); + let (entity_range, styles) = + state.get_entity_range_and_styles_at_range(start..end, pos_type); + + let skip = styles + .as_ref() + .map(|styles| styles.has_key_value(&key, &value)) + .unwrap_or(false); + let has_target_style = state.has_style_key_in_entity_range( + entity_range.clone(), + &StyleKey::Key(key.clone()), + ); + let missing_style_key = is_delete && !has_target_style; + + ( + entity_range, + skip, + missing_style_key, + event_start, + event_end, + ) + }); + + if skip || missing_style_key { + return Ok(()); + } + + let entity_start = entity_range.start; + let entity_end = entity_range.end; + let style_config = doc_state.config.text_style_config.read(); + let flag = if is_delete { + style_config + .get_style_flag_for_unmark(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + } else { + style_config + .get_style_flag(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + }; + + drop(style_config); + drop(doc_state); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::StyleStart { + start: entity_start as u32, + end: entity_end as u32, + key: key.clone(), + value: value.clone(), + info: flag, + }), + EventHint::Mark { + start: event_start as u32, + end: event_end as u32, + style: crate::container::richtext::Style { key, data: value }, + }, + &inner.doc, + )?; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::StyleEnd), + EventHint::MarkEnd, + &inner.doc, + )?; + + Ok(()) + } + + /// §perf (flowstate vendor patch, act three D.1): apply MANY marks in one + /// batched pass — one transaction ceremony for the whole set instead of + /// two full `apply_local_op` rounds per mark. Positions are `pos_type` + /// indices into the CURRENT text (marks do not move text positions, so + /// caller-supplied positions stay valid across the batch). `Null` values + /// unmark, mirroring `unmark`. Only supported on attached containers. + pub fn mark_batch( + &self, + marks: Vec<(usize, usize, InternalString, LoroValue)>, + pos_type: PosType, + ) -> LoroResult<()> { + if marks.is_empty() { + return Ok(()); + } + let inner = self.inner.try_attached_state()?; + let len = self.len(pos_type); + let mut prepared = Vec::with_capacity(marks.len()); + { + let doc_state = inner.doc.state.lock(); + let style_config = doc_state.config.text_style_config.read(); + for (start, end, key, value) in marks { + if start >= end { + return Err(loro_common::LoroError::ArgErr( + "Start must be less than end".to_string().into_boxed_str(), + )); + } + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + ensure_no_regular_container_value(&value)?; + let flag = if matches!(value, LoroValue::Null) { + style_config + .get_style_flag_for_unmark(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + } else { + style_config + .get_style_flag(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + }; + prepared.push((start, end, key, value, flag)); + } + } + match &self.inner { + MaybeDetached::Detached(_) => Err(LoroError::NotImplemented( + "`mark_batch` on a detached text container", + )), + MaybeDetached::Attached(a) => a.with_txn(|txn| { + txn.apply_local_text_mark_batch(inner.container_idx, prepared, pos_type, &inner.doc) + }), + } + } + + pub fn check(&self) { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.check_consistency_between_content_and_style_ranges(); + } + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_richtext_state_mut() + .unwrap() + .check_consistency_between_content_and_style_ranges(); + }), + } + } + + pub fn apply_delta(&self, delta: &[TextDelta]) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let _t = t.lock(); + // TODO: implement + Err(LoroError::NotImplemented( + "`apply_delta` on a detached text container", + )) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.apply_delta_with_txn(txn, delta)), + } + } + + pub fn apply_delta_with_txn( + &self, + txn: &mut Transaction, + delta: &[TextDelta], + ) -> LoroResult<()> { + // §perf (flowstate vendor patch, act three D.1): a many-span + // Insert/Retain delta (an undo restoring a mass deletion arrives as + // one span per style run — tens of thousands) pays the full per-op + // transaction ceremony per span. Route large well-formed deltas + // through the batched insert + batched mark path; op emission is + // byte-identical, so convergence and replay are untouched. + const APPLY_DELTA_BATCH_THRESHOLD: usize = 64; + if !cfg!(feature = "wasm") + && delta.len() > APPLY_DELTA_BATCH_THRESHOLD + && delta + .iter() + .all(|d| matches!(d, TextDelta::Insert { .. } | TextDelta::Retain { .. })) + { + return self.apply_delta_with_txn_batched(txn, delta); + } + let mut index = 0; + struct PendingMark { + start: usize, + end: usize, + attributes: FxHashMap, + } + let mut marks: Vec = Vec::new(); + for d in delta { + match d { + TextDelta::Insert { insert, attributes } => { + let insert_len = event_len(insert.as_str()); + if insert_len == 0 { + continue; + } + + let mut empty_attr = None; + let attr_ref = attributes.as_ref().unwrap_or_else(|| { + empty_attr = Some(FxHashMap::default()); + empty_attr.as_ref().unwrap() + }); + + let end = checked_delta_index_end(index, insert_len, self.len_event())?; + let override_styles = self.insert_with_txn_and_attr( + txn, + index, + insert.as_str(), + Some(attr_ref), + PosType::Event, + )?; + + let mut pending_mark = PendingMark { + start: index, + end, + attributes: FxHashMap::default(), + }; + for (key, value) in override_styles { + pending_mark.attributes.insert(key, value); + } + marks.push(pending_mark); + index = end; + } + TextDelta::Delete { delete } => { + self.delete_with_txn(txn, index, *delete, PosType::Event)?; + } + TextDelta::Retain { attributes, retain } => { + let end = checked_delta_index_end(index, *retain, self.len_event())?; + match attributes { + Some(attr) if !attr.is_empty() => { + let mut pending_mark = PendingMark { + start: index, + end, + attributes: FxHashMap::default(), + }; + for (key, value) in attr { + pending_mark + .attributes + .insert(key.deref().into(), value.clone()); + } + marks.push(pending_mark); + } + _ => {} + } + index = end; + } + } + } + + let mut len = match &self.inner { + MaybeDetached::Detached(_) => self.len_event(), + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().len(PosType::Event)) + } + }; + for pending_mark in marks { + if pending_mark.start >= len { + self.insert_with_txn( + txn, + len, + &"\n".repeat(pending_mark.start - len + 1), + PosType::Event, + )?; + len = pending_mark.start; + } + + for (key, value) in pending_mark.attributes { + self.mark_with_txn( + txn, + pending_mark.start, + pending_mark.end, + key.deref(), + value, + PosType::Event, + )?; + } + } + + Ok(()) + } + + /// §perf (act three D.1): the batched arm of [`Self::apply_delta_with_txn`] + /// for Insert/Retain-only deltas. Inserts flow through the batched insert + /// path; override + retain marks apply afterwards in delta-walk order + /// through the batched mark path — matching the per-span path's op + /// emission exactly. + fn apply_delta_with_txn_batched( + &self, + txn: &mut Transaction, + delta: &[TextDelta], + ) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + let mut index = 0usize; + let mut seq = 0usize; + let mut inserts: Vec<(usize, usize, String, Option>)> = + Vec::new(); + // (seq, start, end, key, value) + let mut retain_marks: Vec<(usize, usize, usize, InternalString, LoroValue)> = Vec::new(); + for d in delta { + match d { + TextDelta::Insert { insert, attributes } => { + let insert_len = event_len(insert.as_str()); + if insert_len == 0 { + continue; + } + inserts.push((seq, index, insert.clone(), attributes.clone())); + seq += 1; + index += insert_len; + } + TextDelta::Retain { retain, attributes } => { + if let Some(attr) = attributes { + if !attr.is_empty() { + for (key, value) in attr { + retain_marks.push(( + seq, + index, + index + *retain, + key.deref().into(), + value.clone(), + )); + } + seq += 1; + } + } + index += retain; + } + TextDelta::Delete { .. } => unreachable!("gated by the caller"), + } + } + + let mut marks = txn.apply_local_text_insert_batch(inner.container_idx, inserts, &inner.doc)?; + marks.append(&mut retain_marks); + marks.sort_by_key(|(seq, ..)| *seq); + + let len = match &self.inner { + MaybeDetached::Detached(_) => self.len_event(), + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_richtext_state_mut().unwrap().len(PosType::Event)) + } + }; + if marks.iter().any(|(_, start, ..)| *start >= len) { + // Rare malformed-delta shape (marks past the end require newline + // padding) — replicate the per-span path exactly for the whole + // mark set. + let mut len = len; + for (_, start, end, key, value) in marks { + if start >= len { + self.insert_with_txn(txn, len, &"\n".repeat(start - len + 1), PosType::Event)?; + len = start; + } + self.mark_with_txn(txn, start, end, key.deref(), value, PosType::Event)?; + } + return Ok(()); + } + + let mut prepared = Vec::with_capacity(marks.len()); + { + let doc_state = inner.doc.state.lock(); + let style_config = doc_state.config.text_style_config.read(); + for (_, start, end, key, value) in marks { + if start >= end { + continue; + } + let flag = if matches!(value, LoroValue::Null) { + style_config + .get_style_flag_for_unmark(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + } else { + style_config + .get_style_flag(&key) + .ok_or_else(|| LoroError::StyleConfigMissing(key.clone()))? + }; + prepared.push((start, end, key, value, flag)); + } + } + txn.apply_local_text_mark_batch(inner.container_idx, prepared, PosType::Event, &inner.doc) + } + + pub fn update(&self, text: &str, options: UpdateOptions) -> Result<(), UpdateTimeoutError> { + let old_str = self.to_string(); + let new = text.chars().map(|x| x as u32).collect::>(); + let old = old_str.chars().map(|x| x as u32).collect::>(); + diff( + &mut OperateProxy::new(text_update::DiffHook::new(self, &new)), + options, + &old, + &new, + )?; + Ok(()) + } + + pub fn update_by_line( + &self, + text: &str, + options: UpdateOptions, + ) -> Result<(), UpdateTimeoutError> { + let hook = text_update::DiffHookForLine::new(self, text); + let old_lines = hook.get_old_arr().to_vec(); + let new_lines = hook.get_new_arr().to_vec(); + diff( + &mut OperateProxy::new(hook), + options, + &old_lines, + &new_lines, + ) + } + + #[allow(clippy::inherent_to_string)] + pub fn to_string(&self) -> String { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.to_string(), + MaybeDetached::Attached(a) => a.get_value().into_string().unwrap().unwrap(), + } + } + + pub fn get_cursor(&self, event_index: usize, side: Side) -> Option { + self.get_cursor_internal(event_index, side, true) + } + + /// Get the stable position representation for the target pos + pub(crate) fn get_cursor_internal( + &self, + index: usize, + side: Side, + get_by_event_index: bool, + ) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + let (id, len, origin_pos) = a.with_state(|s| { + let s = s.as_richtext_state_mut().unwrap(); + ( + s.get_stable_position(index, get_by_event_index), + if get_by_event_index { + s.len_event() + } else { + s.len_unicode() + }, + if get_by_event_index { + s.event_index_to_unicode_index(index) + } else { + index + }, + ) + }); + + if len == 0 { + return Some(Cursor { + id: None, + container: self.id(), + side: if side == Side::Middle { + Side::Left + } else { + side + }, + origin_pos: 0, + }); + } + + if len <= index { + return Some(Cursor { + id: None, + container: self.id(), + side: Side::Right, + origin_pos: len, + }); + } + + let id = id?; + Some(Cursor { + id: Some(id), + container: self.id(), + side, + origin_pos, + }) + } + } + } + + pub(crate) fn convert_entity_index_to_event_index(&self, entity_index: usize) -> usize { + match &self.inner { + MaybeDetached::Detached(s) => s.lock().value.entity_index_to_event_index(entity_index), + MaybeDetached::Attached(a) => { + let mut pos = 0; + a.with_state(|s| { + let s = s.as_richtext_state_mut().unwrap(); + pos = s.entity_index_to_event_index(entity_index); + }); + pos + } + } + } + + pub fn get_delta(&self) -> Vec { + match &self.inner { + MaybeDetached::Detached(s) => { + let mut delta = Vec::new(); + for span in s.lock().value.iter() { + if span.text.as_str().is_empty() { + continue; + } + + let next_attr = span.attributes.to_option_map(); + match delta.last_mut() { + Some(TextDelta::Insert { insert, attributes }) + if &next_attr == attributes => + { + insert.push_str(span.text.as_str()); + continue; + } + _ => {} + } + + delta.push(TextDelta::Insert { + insert: span.text.as_str().to_string(), + attributes: next_attr, + }) + } + delta + } + MaybeDetached::Attached(_a) => self + .with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + Ok(state.get_delta()) + }) + .unwrap(), + } + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn push_str(&self, s: &str) -> LoroResult<()> { + self.insert_utf8(self.len_utf8(), s) + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(mutex) => { + let mut t = mutex.lock(); + let len = t.value.len_unicode(); + let ranges = t.value.get_text_entity_ranges(0, len, PosType::Unicode)?; + for range in ranges.iter().rev() { + t.value + .drain_by_entity_index(range.entity_start, range.entity_len(), None); + } + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| { + let len = a.with_state(|s| s.as_richtext_state_mut().unwrap().len_unicode()); + self.delete_with_txn_inline(txn, 0, len, PosType::Unicode) + }), + } + } + + /// Convert a position `index` from one coordinate system to another. + /// + /// Supported `PosType` conversions: `Event`, `Unicode`, `Utf16`, and `Bytes`. + /// Returns `None` if the index is out of bounds or the conversion is unsupported. + pub fn convert_pos(&self, index: usize, from: PosType, to: PosType) -> Option { + if from == to { + return Some(index); + } + + if matches!(from, PosType::Entity) || matches!(to, PosType::Entity) { + return None; + } + + // Normalize to event + unicode indices for the given position. + let (event_index, unicode_index) = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + if index > t.value.len(from) { + return None; + } + let event_index = if from == PosType::Event { + index + } else { + t.value.index_to_event_index(index, from) + }; + let unicode_index = if from == PosType::Unicode { + index + } else { + t.value.event_index_to_unicode_index(event_index) + }; + (event_index, unicode_index) + } + MaybeDetached::Attached(a) if a.has_decoded_state() => { + let res: Option<(usize, usize)> = a.with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + if index > state.len(from) { + return None; + } + + let event_index = if from == PosType::Event { + index + } else { + state.index_to_event_index(index, from) + }; + let unicode_index = if from == PosType::Unicode { + index + } else { + state.event_index_to_unicode_index(event_index) + }; + Some((event_index, unicode_index)) + }); + + res? + } + MaybeDetached::Attached(a) => { + let value = a.get_value(); + let s = value.as_string().unwrap(); + let unicode_index = text_pos_to_unicode(s, index, from)?; + let event_index = unicode_to_text_pos(s, unicode_index, PosType::Event)?; + (event_index, unicode_index) + } + }; + + let result = match to { + PosType::Unicode => Some(unicode_index), + PosType::Event => Some(event_index), + PosType::Bytes | PosType::Utf16 => { + // Use the prefix text to compute target offset. + let prefix = match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + if event_index > t.value.len_event() { + return None; + } + t.value.get_text_slice_by_event_index(0, event_index).ok()? + } + MaybeDetached::Attached(a) if a.has_decoded_state() => { + let res: Result = a.with_state(|state| { + let state = state.as_richtext_state_mut().unwrap(); + if event_index > state.len_event() { + return Err(()); + } + state + .get_text_slice_by_event_index(0, event_index) + .map_err(|_| ()) + }); + + match res { + Ok(v) => v, + Err(_) => return None, + } + } + MaybeDetached::Attached(a) => { + let value = a.get_value(); + let s = value.as_string().unwrap(); + return unicode_to_text_pos(s, unicode_index, to); + } + }; + + Some(match to { + PosType::Bytes => prefix.len(), + PosType::Utf16 => count_utf16_len(prefix.as_bytes()), + _ => unreachable!(), + }) + } + PosType::Entity => None, + }; + result + } +} + +fn event_len(s: &str) -> usize { + if cfg!(feature = "wasm") { + count_utf16_len(s.as_bytes()) + } else { + s.chars().count() + } +} + +fn text_len(s: &str, pos_type: PosType) -> Option { + Some(match pos_type { + PosType::Bytes => s.len(), + PosType::Unicode => s.chars().count(), + PosType::Utf16 => count_utf16_len(s.as_bytes()), + PosType::Event => event_len(s), + PosType::Entity => return None, + }) +} + +fn text_pos_to_unicode(s: &str, index: usize, pos_type: PosType) -> Option { + match pos_type { + PosType::Unicode => (index <= s.chars().count()).then_some(index), + PosType::Bytes => { + if index > s.len() { + None + } else { + Some( + s.char_indices() + .take_while(|(pos, c)| *pos + c.len_utf8() <= index) + .count(), + ) + } + } + PosType::Utf16 => utf16_to_unicode_pos(s, index), + PosType::Event if cfg!(feature = "wasm") => utf16_to_unicode_pos(s, index), + PosType::Event => (index <= s.chars().count()).then_some(index), + PosType::Entity => None, + } +} + +fn unicode_to_text_pos(s: &str, index: usize, pos_type: PosType) -> Option { + match pos_type { + PosType::Unicode => (index <= s.chars().count()).then_some(index), + PosType::Bytes => unicode_to_byte_pos(s, index), + PosType::Utf16 => unicode_to_utf16_pos(s, index), + PosType::Event if cfg!(feature = "wasm") => unicode_to_utf16_pos(s, index), + PosType::Event => (index <= s.chars().count()).then_some(index), + PosType::Entity => None, + } +} + +fn unicode_to_byte_pos(s: &str, index: usize) -> Option { + if index == 0 { + return Some(0); + } + + let mut unicode_pos = 0; + for (byte_pos, _) in s.char_indices() { + if unicode_pos == index { + return Some(byte_pos); + } + unicode_pos += 1; + } + + (unicode_pos == index).then_some(s.len()) +} + +fn unicode_to_utf16_pos(s: &str, index: usize) -> Option { + let mut unicode_pos = 0; + let mut utf16_pos = 0; + if index == 0 { + return Some(0); + } + + for c in s.chars() { + unicode_pos += 1; + utf16_pos += c.len_utf16(); + if unicode_pos == index { + return Some(utf16_pos); + } + } + + (unicode_pos == index).then_some(utf16_pos) +} + +fn utf16_to_unicode_pos(s: &str, index: usize) -> Option { + let mut unicode_pos = 0; + let mut utf16_pos = 0; + if index == 0 { + return Some(0); + } + + for c in s.chars() { + let next_utf16_pos = utf16_pos + c.len_utf16(); + if index < next_utf16_pos { + return Some(unicode_pos); + } + if index == next_utf16_pos { + return Some(unicode_pos + 1); + } + utf16_pos = next_utf16_pos; + unicode_pos += 1; + } + + (index == utf16_pos).then_some(unicode_pos) +} + +fn text_boundary_error(pos: usize, pos_type: PosType) -> LoroError { + match pos_type { + PosType::Bytes => LoroError::UTF8InUnicodeCodePoint { pos }, + PosType::Utf16 => LoroError::UTF16InUnicodeCodePoint { pos }, + PosType::Event if cfg!(feature = "wasm") => LoroError::UTF16InUnicodeCodePoint { pos }, + _ => LoroError::OutOfBound { + pos, + len: 0, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }, + } +} + +fn text_char_at(s: &str, pos: usize, pos_type: PosType) -> LoroResult { + let len = text_len(s, pos_type).unwrap_or(0); + if pos >= len { + return Err(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let unicode_pos = + text_pos_to_unicode(s, pos, pos_type).ok_or_else(|| text_boundary_error(pos, pos_type))?; + s.chars().nth(unicode_pos).ok_or(LoroError::OutOfBound { + pos, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }) +} + +fn text_slice(s: &str, start: usize, end: usize, pos_type: PosType) -> LoroResult { + if end < start { + return Err(LoroError::EndIndexLessThanStartIndex { start, end }); + } + if start == end { + return Ok(String::new()); + } + + let len = text_len(s, pos_type).unwrap_or(0); + if end > len { + return Err(LoroError::OutOfBound { + pos: end, + len, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + + let start = text_pos_to_unicode(s, start, pos_type) + .ok_or_else(|| text_boundary_error(start, pos_type))?; + let end = + text_pos_to_unicode(s, end, pos_type).ok_or_else(|| text_boundary_error(end, pos_type))?; + let start = unicode_to_byte_pos(s, start).expect("unicode index must map to a byte boundary"); + let end = unicode_to_byte_pos(s, end).expect("unicode index must map to a byte boundary"); + Ok(s[start..end].to_string()) +} + +impl ListHandler { + /// Create a new container that is detached from the document. + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(Vec::new()), + } + } + + pub fn insert(&self, pos: usize, v: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + let len = list.value.len(); + if pos > len { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len, + }); + } + let value = v.into(); + ensure_no_regular_container_value(&value)?; + list.value.insert(pos, ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, pos, v.into())) + } + } + } + + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + v: LoroValue, + ) -> LoroResult<()> { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let inner = self.inner.try_attached_state()?; + ensure_no_regular_container_value(&v)?; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + ) + } + + pub fn push(&self, v: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + let value = v.into(); + ensure_no_regular_container_value(&value)?; + list.value.push(ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.push_with_txn(txn, v.into())), + } + } + + pub fn push_with_txn(&self, txn: &mut Transaction, v: LoroValue) -> LoroResult<()> { + let pos = self.len(); + self.insert_with_txn(txn, pos, v) + } + + pub fn pop(&self) -> LoroResult> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + Ok(list.value.pop().map(|v| v.to_value())) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.pop_with_txn(txn)), + } + } + + pub fn pop_with_txn(&self, txn: &mut Transaction) -> LoroResult> { + let len = self.len(); + if len == 0 { + return Ok(None); + } + + let v = self.get(len - 1); + self.delete_with_txn(txn, len - 1, 1)?; + Ok(v) + } + + pub fn insert_container(&self, pos: usize, child: H) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + if pos > list.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: list.value.len(), + }); + } + list.value + .insert(pos, ValueOrHandler::Handler(child.to_handler())); + Ok(child) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_container_with_txn(txn, pos, child)) + } + } + } + + pub fn push_container(&self, child: H) -> LoroResult { + self.insert_container(self.len(), child) + } + + pub fn insert_container_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + child: H, + ) -> LoroResult { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let inner = self.inner.try_attached_state()?; + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + let v = LoroValue::Container(container_id.clone()); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + )?; + let ans = child.attach(txn, inner, container_id)?; + Ok(ans) + } + + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut list = l.lock(); + let end = checked_range_end( + pos, + len, + list.value.len(), + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + list.value.drain(pos..end); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, pos, len)), + } + } + + pub fn delete_with_txn(&self, txn: &mut Transaction, pos: usize, len: usize) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let list_len = self.len(); + let end = checked_range_end( + pos, + len, + list_len, + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + + let inner = self.inner.try_attached_state()?; + let ids: Vec<_> = inner.with_state(|state| { + let list = state.as_list_state().unwrap(); + (pos..end).map(|i| list.get_id_at(i).unwrap()).collect() + }); + + for id in ids.into_iter() { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + id.id(), + pos as isize, + 1, + ))), + EventHint::DeleteList(DeleteSpan::new(pos as isize, 1)), + &inner.doc, + )?; + } + + Ok(()) + } + + pub fn get_child_handler(&self, index: usize) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let list = l.lock(); + let value = list.value.get(index).ok_or(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: list.value.len(), + })?; + match value { + ValueOrHandler::Handler(h) => Ok(h.clone()), + _ => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + MaybeDetached::Attached(_) => { + let Some(value) = self.get_(index) else { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + }; + match value { + ValueOrHandler::Handler(handler) => Ok(handler), + ValueOrHandler::Value(value) => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + } + } + + pub fn len(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(l) => l.lock().value.len(), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_len(a.container_idx)) + } + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get_deep_value_with_id(&self) -> LoroResult { + let inner = self.inner.try_attached_state()?; + Ok(inner.with_doc_state(|state| { + state.get_container_deep_value_with_id(inner.container_idx, None) + })) + } + + pub fn get(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(l) => l.lock().value.get(index).map(|x| x.to_value()), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_value_at(a.container_idx, index)) + } + } + } + + /// Get value at given index, if it's a container, return a handler to the container + pub fn get_(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(l) => { + let l = l.lock(); + l.value.get(index).cloned() + } + MaybeDetached::Attached(inner) => { + let value = inner + .with_doc_state(|state| state.get_list_value_at(inner.container_idx, index)); + value.map(|value| value_to_value_or_handler(inner, value)) + } + } + } + + pub fn for_each(&self, mut f: I) + where + I: FnMut(ValueOrHandler), + { + match &self.inner { + MaybeDetached::Detached(l) => { + let l = l.lock(); + for v in l.value.iter() { + f(v.clone()) + } + } + MaybeDetached::Attached(inner) => { + let temp = inner.with_doc_state(|state| { + state + .get_list_values(inner.container_idx) + .into_iter() + .map(|value| value_to_value_or_handler(inner, value)) + .collect::>() + }); + for v in temp.into_iter() { + f(v); + } + } + } + } + + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + let (id, len) = a.with_state(|s| { + let l = s.as_list_state().unwrap(); + (l.get_id_at(pos), l.len()) + }); + + if len == 0 { + return Some(Cursor { + id: None, + container: self.id(), + side: if side == Side::Middle { + Side::Left + } else { + side + }, + origin_pos: 0, + }); + } + + if len <= pos { + return Some(Cursor { + id: None, + container: self.id(), + side: Side::Right, + origin_pos: len, + }); + } + + let id = id?; + Some(Cursor { + id: Some(id.id()), + container: self.id(), + side, + origin_pos: pos, + }) + } + } + } + + fn apply_delta( + &self, + delta: loro_delta::DeltaRope< + loro_delta::array_vec::ArrayVec, + crate::event::ListDeltaMeta, + >, + on_container_remap: &mut dyn FnMut(ContainerID, ContainerID), + ) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(_) => unimplemented!(), + MaybeDetached::Attached(_) => { + let mut index = 0; + for item in delta.iter() { + match item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + self.delete(index, *delete)?; + } + + for v in value.iter() { + match v { + ValueOrHandler::Value(LoroValue::Container(old_id)) => { + let new_h = self.insert_container( + index, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_h.id(); + on_container_remap(old_id.clone(), new_id); + } + ValueOrHandler::Handler(h) => { + let old_id = h.id(); + let new_h = self.insert_container( + index, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_h.id(); + on_container_remap(old_id, new_id); + } + ValueOrHandler::Value(v) => { + self.insert(index, v.clone())?; + } + } + + index += 1; + } + } + } + } + } + } + + Ok(()) + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(l) => { + let mut l = l.lock(); + l.value.clear(); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.clear_with_txn(txn)), + } + } + + pub fn clear_with_txn(&self, txn: &mut Transaction) -> LoroResult<()> { + self.delete_with_txn(txn, 0, self.len()) + } + + pub fn get_id_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_list_state() + .unwrap() + .get_id_at(pos) + .map(|x| x.id()) + }), + } + } +} + +impl MovableListHandler { + pub fn insert(&self, pos: usize, v: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if pos > d.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + let value = v.into(); + ensure_no_regular_container_value(&value)?; + d.value.insert(pos, ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, pos, v.into())) + } + } + } + + #[instrument(skip_all)] + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + v: LoroValue, + ) -> LoroResult<()> { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + ensure_no_regular_container_value(&v)?; + + let op_index = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list + .convert_index(pos, IndexType::ForUser, IndexType::ForOp) + .unwrap()) + })?; + + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos: op_index, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + ) + } + + #[inline] + pub fn mov(&self, from: usize, to: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if from >= d.value.len() { + return Err(LoroError::OutOfBound { + pos: from, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + if to >= d.value.len() { + return Err(LoroError::OutOfBound { + pos: to, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + let v = d.value.remove(from); + d.value.insert(to, v); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.move_with_txn(txn, from, to)), + } + } + + /// Move element from `from` to `to`. After this op, elem will be at pos `to`. + #[instrument(skip_all)] + pub fn move_with_txn(&self, txn: &mut Transaction, from: usize, to: usize) -> LoroResult<()> { + if from == to { + return Ok(()); + } + + if from >= self.len() { + return Err(LoroError::OutOfBound { + pos: from, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + if to >= self.len() { + return Err(LoroError::OutOfBound { + pos: to, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let (op_from, op_to, elem_id, value) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + let (elem_id, elem) = list + .get_elem_at_given_pos(from, IndexType::ForUser) + .unwrap(); + Ok(( + list.convert_index(from, IndexType::ForUser, IndexType::ForOp) + .unwrap(), + list.convert_index(to, IndexType::ForUser, IndexType::ForOp) + .unwrap(), + elem_id, + elem.value().clone(), + )) + })?; + + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Move { + from: op_from as u32, + to: op_to as u32, + elem_id: elem_id.to_id(), + }), + EventHint::Move { + value, + from: from as u32, + to: to as u32, + }, + &inner.doc, + ) + } + + pub fn push(&self, v: LoroValue) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + d.value.push(v.into()); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.push_with_txn(txn, v)), + } + } + + pub fn push_with_txn(&self, txn: &mut Transaction, v: LoroValue) -> LoroResult<()> { + let pos = self.len(); + self.insert_with_txn(txn, pos, v) + } + + pub fn pop_(&self) -> LoroResult> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + Ok(d.value.pop()) + } + MaybeDetached::Attached(a) => { + if self.is_empty() { + return Ok(None); + } + let last = self.len() - 1; + let ans = self.get_(last); + a.with_txn(|txn| self.pop_with_txn(txn))?; + Ok(ans) + } + } + } + + pub fn pop(&self) -> LoroResult> { + match &self.inner { + MaybeDetached::Detached(a) => { + let mut a = a.lock(); + Ok(a.value.pop().map(|x| x.to_value())) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.pop_with_txn(txn)), + } + } + + pub fn pop_with_txn(&self, txn: &mut Transaction) -> LoroResult> { + let len = self.len(); + if len == 0 { + return Ok(None); + } + + let v = self.get(len - 1); + self.delete_with_txn(txn, len - 1, 1)?; + Ok(v) + } + + pub fn insert_container(&self, pos: usize, child: H) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if pos > d.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + d.value + .insert(pos, ValueOrHandler::Handler(child.to_handler())); + Ok(child) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_container_with_txn(txn, pos, child)) + } + } + } + + pub fn push_container(&self, child: H) -> LoroResult { + self.insert_container(self.len(), child) + } + + pub fn insert_container_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + child: H, + ) -> LoroResult { + if pos > self.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let op_index = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list + .convert_index(pos, IndexType::ForUser, IndexType::ForOp) + .unwrap()) + })?; + + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + let v = LoroValue::Container(container_id.clone()); + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![v.clone()])), + pos: op_index, + }), + EventHint::InsertList { len: 1, pos }, + &inner.doc, + )?; + child.attach(txn, inner, container_id) + } + + pub fn set(&self, index: usize, value: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if index >= d.value.len() { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + let value = value.into(); + ensure_no_regular_container_value(&value)?; + d.value[index] = ValueOrHandler::Value(value); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.set_with_txn(txn, index, value.into())) + } + } + } + + pub fn set_with_txn( + &self, + txn: &mut Transaction, + index: usize, + value: LoroValue, + ) -> LoroResult<()> { + if index >= self.len() { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + } + + let inner = self.inner.try_attached_state()?; + let Some(elem_id) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list.get_elem_id_at(index, IndexType::ForUser)) + })? + else { + unreachable!() + }; + ensure_no_regular_container_value(&value)?; + + let op = crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Set { + elem_id: elem_id.to_id(), + value: value.clone(), + }); + + let hint = EventHint::SetList { index, value }; + txn.apply_local_op(inner.container_idx, op, hint, &inner.doc) + } + + pub fn set_container(&self, pos: usize, child: H) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + if pos >= d.value.len() { + return Err(LoroError::OutOfBound { + pos, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: d.value.len(), + }); + } + d.value[pos] = ValueOrHandler::Handler(child.to_handler()); + Ok(child) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.set_container_with_txn(txn, pos, child)) + } + } + } + + pub fn set_container_with_txn( + &self, + txn: &mut Transaction, + pos: usize, + child: H, + ) -> LoroResult { + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + let v = LoroValue::Container(container_id.clone()); + let Some(elem_id) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + Ok(list.get_elem_id_at(pos, IndexType::ForUser)) + })? + else { + let len = self.len(); + if pos >= len { + return Err(LoroError::OutOfBound { + pos, + len, + info: "".into(), + }); + } else { + unreachable!() + } + }; + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(crate::container::list::list_op::ListOp::Set { + elem_id: elem_id.to_id(), + value: v.clone(), + }), + EventHint::SetList { + index: pos, + value: v, + }, + &inner.doc, + )?; + + child.attach(txn, inner, container_id) + } + + pub fn delete(&self, pos: usize, len: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + let end = checked_range_end( + pos, + len, + d.value.len(), + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + d.value.drain(pos..end); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, pos, len)), + } + } + + #[instrument(skip_all)] + pub fn delete_with_txn(&self, txn: &mut Transaction, pos: usize, len: usize) -> LoroResult<()> { + if len == 0 { + return Ok(()); + } + + let list_len = self.len(); + let end = checked_range_end( + pos, + len, + list_len, + format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + )?; + + let (ids, new_poses) = self.with_state(|state| { + let list = state.as_movable_list_state().unwrap(); + let ids: Vec<_> = (pos..end) + .map(|i| list.get_list_id_at(i, IndexType::ForUser).unwrap()) + .collect(); + let poses: Vec<_> = (pos..end) + // need to -i because we delete the previous ones + .map(|user_index| { + let op_index = list + .convert_index(user_index, IndexType::ForUser, IndexType::ForOp) + .unwrap(); + assert!(op_index >= user_index); + op_index - (user_index - pos) + }) + .collect(); + Ok((ids, poses)) + })?; + + loro_common::info!(?pos, ?len, ?ids, ?new_poses, "delete_with_txn"); + let user_pos = pos; + let inner = self.inner.try_attached_state()?; + for (id, op_pos) in ids.into_iter().zip(new_poses.into_iter()) { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new( + id, + op_pos as isize, + 1, + ))), + EventHint::DeleteList(DeleteSpan::new(user_pos as isize, 1)), + &inner.doc, + )?; + } + + Ok(()) + } + + pub fn get_child_handler(&self, index: usize) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(l) => { + let list = l.lock(); + let value = list.value.get(index).ok_or(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: list.value.len(), + })?; + match value { + ValueOrHandler::Handler(h) => Ok(h.clone()), + _ => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + MaybeDetached::Attached(_) => { + let Some(value) = self.get_(index) else { + return Err(LoroError::OutOfBound { + pos: index, + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + len: self.len(), + }); + }; + match value { + ValueOrHandler::Handler(handler) => Ok(handler), + ValueOrHandler::Value(value) => Err(LoroError::ArgErr( + format!( + "Expected container at index {}, but found {:?}", + index, value + ) + .into_boxed_str(), + )), + } + } + } + } + + pub fn len(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value.len() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_len(a.container_idx)) + } + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn get_deep_value_with_id(&self) -> LoroValue { + let inner = self.inner.try_attached_state().unwrap(); + inner + .doc + .state + .lock() + .get_container_deep_value_with_id(inner.container_idx, None) + } + + pub fn get(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value.get(index).map(|v| v.to_value()) + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_list_value_at(a.container_idx, index)) + } + } + } + + /// Get value at given index, if it's a container, return a handler to the container + pub fn get_(&self, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value.get(index).cloned() + } + MaybeDetached::Attached(m) => { + let value = + m.with_doc_state(|state| state.get_list_value_at(m.container_idx, index)); + value.map(|value| value_to_value_or_handler(m, value)) + } + } + } + + pub fn for_each(&self, mut f: I) + where + I: FnMut(ValueOrHandler), + { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + for v in d.value.iter() { + f(v.clone()); + } + } + MaybeDetached::Attached(m) => { + let temp = m.with_doc_state(|state| { + state + .get_list_values(m.container_idx) + .into_iter() + .map(|value| value_to_value_or_handler(m, value)) + .collect::>() + }); + + for v in temp.into_iter() { + f(v); + } + } + } + } + + pub fn log_internal_state(&self) -> String { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + format!("{:#?}", &d.value) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_movable_list_state().unwrap(); + format!("{a:#?}") + }), + } + } + + pub fn new_detached() -> MovableListHandler { + MovableListHandler { + inner: MaybeDetached::new_detached(Default::default()), + } + } + + pub fn get_cursor(&self, pos: usize, side: Side) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(inner) => { + let (id, len) = inner.with_state(|s| { + let l = s.as_movable_list_state().unwrap(); + (l.get_list_item_id_at(pos), l.len()) + }); + + if len == 0 { + return Some(Cursor { + id: None, + container: self.id(), + side: if side == Side::Middle { + Side::Left + } else { + side + }, + origin_pos: 0, + }); + } + + if len <= pos { + return Some(Cursor { + id: None, + container: self.id(), + side: Side::Right, + origin_pos: len, + }); + } + + let id = id?; + Some(Cursor { + id: Some(id.id()), + container: self.id(), + side, + origin_pos: pos, + }) + } + } + } + + pub(crate) fn op_pos_to_user_pos(&self, new_pos: usize) -> usize { + match &self.inner { + MaybeDetached::Detached(_) => new_pos, + MaybeDetached::Attached(inner) => { + let mut pos = new_pos; + inner.with_state(|s| { + let l = s.as_movable_list_state().unwrap(); + pos = l + .convert_index(new_pos, IndexType::ForOp, IndexType::ForUser) + .unwrap_or(l.len()); + }); + pos + } + } + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let mut d = d.lock(); + d.value.clear(); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.clear_with_txn(txn)), + } + } + + pub fn clear_with_txn(&self, txn: &mut Transaction) -> LoroResult<()> { + self.delete_with_txn(txn, 0, self.len()) + } + + pub fn get_creator_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => { + a.with_state(|state| state.as_movable_list_state().unwrap().get_creator_at(pos)) + } + } + } + + pub fn get_last_mover_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_movable_list_state() + .unwrap() + .get_last_mover_at(pos) + }), + } + } + + pub fn get_last_editor_at(&self, pos: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + state + .as_movable_list_state() + .unwrap() + .get_last_editor_at(pos) + }), + } + } +} + +impl MapHandler { + /// Create a new container that is detached from the document. + /// The edits on a detached container will not be persisted. + /// To attach the container to the document, please insert it into an attached container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(Default::default()), + } + } + + pub fn insert(&self, key: &str, value: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let value = value.into(); + ensure_no_regular_container_value(&value)?; + m.value.insert(key.into(), ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_with_txn(txn, key, value.into())) + } + } + } + + /// This method will insert the value even if the same value is already in the given entry. + fn insert_without_skipping(&self, key: &str, value: impl Into) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let value = value.into(); + ensure_no_regular_container_value(&value)?; + m.value.insert(key.into(), ValueOrHandler::Value(value)); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| { + let this = &self; + let value = value.into(); + ensure_no_regular_container_value(&value)?; + + let inner = this.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: Some(value.clone()), + }), + EventHint::Map { + key: key.into(), + value: Some(value.clone()), + }, + &inner.doc, + ) + }), + } + } + + pub fn insert_with_txn( + &self, + txn: &mut Transaction, + key: &str, + value: LoroValue, + ) -> LoroResult<()> { + ensure_no_regular_container_value(&value)?; + + if self.get(key).map(|x| x == value).unwrap_or(false) { + // skip if the value is already set + return Ok(()); + } + + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: Some(value.clone()), + }), + EventHint::Map { + key: key.into(), + value: Some(value.clone()), + }, + &inner.doc, + ) + } + + pub fn insert_container(&self, key: &str, handler: T) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + let to_insert = handler.to_handler(); + m.value + .insert(key.into(), ValueOrHandler::Handler(to_insert.clone())); + Ok(handler) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.insert_container_with_txn(txn, key, handler)) + } + } + } + + pub fn insert_container_with_txn( + &self, + txn: &mut Transaction, + key: &str, + child: H, + ) -> LoroResult { + let inner = self.inner.try_attached_state()?; + let id = txn.next_id(); + let container_id = ContainerID::new_normal(id, child.kind()); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: Some(LoroValue::Container(container_id.clone())), + }), + EventHint::Map { + key: key.into(), + value: Some(LoroValue::Container(container_id.clone())), + }, + &inner.doc, + )?; + + child.attach(txn, inner, container_id) + } + + pub fn delete(&self, key: &str) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + m.value.remove(key); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, key)), + } + } + + pub fn delete_with_txn(&self, txn: &mut Transaction, key: &str) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Map(crate::container::map::MapSet { + key: key.into(), + value: None, + }), + EventHint::Map { + key: key.into(), + value: None, + }, + &inner.doc, + ) + } + + pub fn for_each(&self, mut f: I) + where + I: FnMut(&str, ValueOrHandler), + { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + for (k, v) in m.value.iter() { + f(k, v.clone()); + } + } + MaybeDetached::Attached(inner) => { + let temp = inner.with_doc_state(|state| { + state + .get_map_entries(inner.container_idx) + .into_iter() + .map(|(key, value)| { + let translated = + loro_common::translate_mergeable_marker_value(&inner.id, key.as_ref(), value); + (key.to_string(), value_to_value_or_handler(inner, translated)) + }) + .collect::>() + }); + + for (k, v) in temp.into_iter() { + f(&k, v.clone()); + } + } + } + } + + pub fn get_child_handler(&self, key: &str) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + let value = m.value.get(key).unwrap(); + match value { + ValueOrHandler::Value(v) => Err(LoroError::ArgErr( + format!("Expected Handler but found {:?}", v).into_boxed_str(), + )), + ValueOrHandler::Handler(h) => Ok(h.clone()), + } + } + MaybeDetached::Attached(_) => { + let Some(value) = self.get_(key) else { + return Err(LoroError::ArgErr( + format!("Key {key} does not exist").into_boxed_str(), + )); + }; + match value { + ValueOrHandler::Handler(handler) => Ok(handler), + ValueOrHandler::Value(value) => Err(LoroError::ArgErr( + format!("Expected Handler but found {:?}", value).into_boxed_str(), + )), + } + } + } + } + + pub fn get_deep_value_with_id(&self) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(_) => Err(LoroError::MisuseDetachedContainer { + method: "get_deep_value_with_id", + }), + MaybeDetached::Attached(inner) => Ok(inner.with_doc_state(|state| { + state.get_container_deep_value_with_id(inner.container_idx, None) + })), + } + } + + pub fn get(&self, key: &str) -> Option { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.get(key).map(|v| v.to_value()) + } + MaybeDetached::Attached(inner) => { + let value = inner + .with_doc_state(|state| state.get_map_value_by_key(inner.container_idx, key))?; + Some(loro_common::translate_mergeable_marker_value(&inner.id, key, value)) + } + } + } + + /// Get the value at given key, if value is a container, return a handler to the container + pub fn get_(&self, key: &str) -> Option { + match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.get(key).cloned() + } + MaybeDetached::Attached(inner) => { + let value = inner + .with_doc_state(|state| state.get_map_value_by_key(inner.container_idx, key))?; + let value = loro_common::translate_mergeable_marker_value(&inner.id, key, value); + Some(value_to_value_or_handler(inner, value)) + } + } + } + + pub fn get_or_create_container(&self, key: &str, child: C) -> LoroResult { + if let Some(ans) = self.get_(key) { + if let ValueOrHandler::Handler(h) = ans { + let kind = h.kind(); + return C::from_handler(h).ok_or_else(move || { + LoroError::ArgErr( + format!("Expected value type {} but found {:?}", child.kind(), kind) + .into_boxed_str(), + ) + }); + } else if let ValueOrHandler::Value(LoroValue::Null) = ans { + // do nothing + } else { + return Err(LoroError::ArgErr( + format!("Expected value type {} but found {:?}", child.kind(), ans) + .into_boxed_str(), + )); + } + } + + self.insert_container(key, child) + } + + /// Shared implementation for all `ensure_mergeable_*` methods. + /// + /// Computes a deterministic [`ContainerID::Root`] in the mergeable namespace from + /// `(parent.id, key, child.kind())` and constructs the handler from it. Two peers calling this + /// with the same `(parent, key, kind)` receive handlers with identical container ids, which is + /// what makes the child container mergeable on concurrent first-write. + /// + /// # Errors + /// + /// Returns [`LoroError::MisuseDetachedContainer`] when called on a detached handler. The + /// deterministic cid is computed from the parent's cid, which a detached parent does not have + /// yet; falling back to a non-deterministic regular child would silently drop the mergeable + /// guarantee at attach time. Detached callers must attach the parent first. + /// + /// Returns [`LoroError::ArgErr`] if the parent slot already holds a non-mergeable value, or if + /// `C::from_handler` rejects the handler built from the deterministic cid (unreachable by + /// construction; guards against future drift between `from_handler` and `kind`). + fn ensure_mergeable_container(&self, key: &str, child: C) -> LoroResult { + let MaybeDetached::Attached(parent) = &self.inner else { + return Err(LoroError::MisuseDetachedContainer { + method: "ensure_mergeable_container", + }); + }; + + // Compare against the raw marker bytes (skipping `MapHandler::get`'s user-facing + // marker → Container translation) so the non-mergeable-occupant guard sees the real + // slot value and the same-kind idempotent-skip can match. + let existing_raw = parent.with_doc_state(|state| { + state.get_map_value_by_key(parent.container_idx, key) + }); + + // A non-mergeable occupant (scalar, arbitrary binary, regular child container) would be + // silently clobbered by the marker write, so reject rather than overwrite under a + // `get_`-named API. Only the exact binary marker for this `(parent, key, kind)` is + // accepted as an existing mergeable occupant. + if let Some(existing) = &existing_raw { + if !matches!(existing, LoroValue::Null) + && loro_common::parse_mergeable_marker(&parent.id, key, existing).is_none() + { + return Err(LoroError::ArgErr( + format!( + "Cannot create a mergeable {} at key {key:?}: the key already holds a non-mergeable value", + child.kind() + ) + .into_boxed_str(), + )); + } + } + + let cid = ContainerID::new_mergeable(&parent.id, key, child.kind()); + let marker = loro_common::mergeable_marker(&parent.id, key, child.kind()); + + // Idempotent-skip on same marker: `MapHandler::get` translates markers to Container, so + // `insert_with_txn`'s equality check can't see this collision — do it directly. + // A different-kind marker is a deliberate kind change; let the insert through. + if existing_raw.as_ref() != Some(&marker) { + self.insert(key, marker)?; + } + + C::from_handler(create_handler(parent, cid.clone())).ok_or_else(|| { + LoroError::ArgErr( + format!( + "Expected value type {} but found {}", + child.kind(), + cid.container_type() + ) + .into_boxed_str(), + ) + }) + } + + #[cfg(feature = "counter")] + pub fn ensure_mergeable_counter(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, counter::CounterHandler::new_detached()) + } + + /// Ensure a mergeable Map child exists under `key` and return its handler. + /// + /// Prefer to avoid very deep mergeable-map chains: mergeable cids encode their flattened + /// logical path, so cid size still grows with depth and rides through every op/snapshot + /// reference to it. See [`MERGEABLE_NAMESPACE_PREFIX`](loro_common::MERGEABLE_NAMESPACE_PREFIX). + pub fn ensure_mergeable_map(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, MapHandler::new_detached()) + } + + pub fn ensure_mergeable_list(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, ListHandler::new_detached()) + } + + pub fn ensure_mergeable_movable_list(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, MovableListHandler::new_detached()) + } + + pub fn ensure_mergeable_text(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, TextHandler::new_detached()) + } + + pub fn ensure_mergeable_tree(&self, key: &str) -> LoroResult { + self.ensure_mergeable_container(key, TreeHandler::new_detached()) + } + + pub fn contains_key(&self, key: &str) -> bool { + self.get(key).is_some() + } + + pub fn len(&self) -> usize { + match &self.inner { + MaybeDetached::Detached(m) => m.lock().value.len(), + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_map_len(a.container_idx)) + } + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(m) => { + let mut m = m.lock(); + m.value.clear(); + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.clear_with_txn(txn)), + } + } + + pub fn clear_with_txn(&self, txn: &mut Transaction) -> LoroResult<()> { + let keys: Vec = self.inner.try_attached_state()?.with_state(|state| { + state + .as_map_state() + .unwrap() + .iter() + .map(|(k, _)| k.clone()) + .collect() + }); + + for key in keys { + self.delete_with_txn(txn, &key)?; + } + + Ok(()) + } + + pub fn keys(&self) -> impl Iterator + '_ { + let keys: Vec = match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.keys().map(|x| x.as_str().into()).collect() + } + MaybeDetached::Attached(a) => { + a.with_doc_state(|state| state.get_map_keys(a.container_idx)) + } + }; + + keys.into_iter() + } + + pub fn values(&self) -> impl Iterator + '_ { + let values: Vec = match &self.inner { + MaybeDetached::Detached(m) => { + let m = m.lock(); + m.value.values().cloned().collect() + } + MaybeDetached::Attached(a) => a.with_doc_state(|state| { + // A mergeable child's marker lives in the parent map's value table; iterate + // entries (key + value) so the user-boundary translation can resolve each marker + // to the deterministic child cid before wrapping into a handler. + state + .get_map_entries(a.container_idx) + .into_iter() + .map(|(key, value)| { + let translated = + loro_common::translate_mergeable_marker_value(&a.id, key.as_ref(), value); + value_to_value_or_handler(a, translated) + }) + .collect() + }), + }; + + values.into_iter() + } + + pub fn get_last_editor(&self, key: &str) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + let m = state.as_map_state().unwrap(); + m.get_last_edit_peer(key) + }), + } + } +} + +fn with_txn(doc: &LoroDoc, f: impl FnOnce(&mut Transaction) -> LoroResult) -> LoroResult { + let txn = &doc.txn; + let mut txn = txn.lock(); + loop { + if let Some(txn) = &mut *txn { + return f(txn); + } else if cfg!(target_arch = "wasm32") || !doc.can_edit() { + return Err(LoroError::AutoCommitNotStarted); + } else { + drop(txn); + #[cfg(loom)] + loom::thread::yield_now(); + doc.start_auto_commit(); + txn = doc.txn.lock(); + } + } +} + +#[cfg(feature = "counter")] +pub mod counter { + + use loro_common::LoroResult; + + use crate::{ + txn::{EventHint, Transaction}, + HandlerTrait, + }; + + use super::{create_handler, Handler, MaybeDetached}; + + #[derive(Clone)] + pub struct CounterHandler { + pub(super) inner: MaybeDetached, + } + + impl CounterHandler { + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(0.), + } + } + + pub fn increment(&self, n: f64) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = &mut d.lock().value; + *d += n; + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.increment_with_txn(txn, n)), + } + } + + pub fn decrement(&self, n: f64) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = &mut d.lock().value; + *d -= n; + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.increment_with_txn(txn, -n)), + } + } + + fn increment_with_txn(&self, txn: &mut Transaction, n: f64) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Counter(n), + EventHint::Counter(n), + &inner.doc, + ) + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn clear(&self) -> LoroResult<()> { + self.decrement(self.get_value().into_double().unwrap()) + } + } + + impl std::fmt::Debug for CounterHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "CounterHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "CounterHandler {}", a.id), + } + } + } + + impl HandlerTrait for CounterHandler { + fn is_attached(&self) -> bool { + matches!(&self.inner, MaybeDetached::Attached(..)) + } + + fn attached_handler(&self) -> Option<&crate::BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> loro_common::LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.into() + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> loro_common::LoroValue { + self.get_value() + } + + fn kind(&self) -> loro_common::ContainerType { + loro_common::ContainerType::Counter + } + + fn to_handler(&self) -> super::Handler { + Handler::Counter(self.clone()) + } + + fn from_handler(h: super::Handler) -> Option { + match h { + Handler::Counter(x) => Some(x), + _ => None, + } + } + + fn attach( + &self, + txn: &mut crate::txn::Transaction, + parent: &crate::BasicHandler, + self_id: loro_common::ContainerID, + ) -> loro_common::LoroResult { + match &self.inner { + MaybeDetached::Detached(v) => { + let mut v = v.lock(); + let inner = create_handler(parent, self_id); + let c = inner.into_counter().unwrap(); + + c.increment_with_txn(txn, v.value)?; + + v.attached = c.attached_handler().cloned(); + Ok(c) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_counter().unwrap(); + let delta = *self.get_value().as_double().unwrap(); + ans.increment_with_txn(txn, delta)?; + Ok(ans) + } + } + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Attached(a) => Some(Self { + inner: MaybeDetached::Attached(a.clone()), + }), + MaybeDetached::Detached(v) => v.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } + } +} + +#[cfg(test)] +mod test { + use std::borrow::Cow; + + use super::{ + Handler, HandlerTrait, ListHandler, MapHandler, MovableListHandler, TextDelta, TextHandler, + ValueOrHandler, + }; + use crate::container::list::list_op::ListOp; + use crate::cursor::PosType; + use crate::loro::ExportMode; + use crate::op::ListSlice; + use crate::state::TreeParentId; + use crate::txn::EventHint; + use crate::version::Frontiers; + use crate::LoroDoc; + use crate::{fx_map, ToJson}; + use loro_common::{ContainerID, ContainerType, LoroError, LoroValue, ID}; + use serde_json::json; + + fn recheck_fast_blob(mut bytes: Vec) -> Vec { + let checksum = xxhash_rust::xxh32::xxh32(&bytes[20..], u32::from_le_bytes(*b"LORO")); + bytes[16..20].copy_from_slice(&checksum.to_le_bytes()); + bytes + } + + fn replace_fast_snapshot_state_bytes(mut snapshot: Vec, state_bytes: &[u8]) -> Vec { + let mut body = &snapshot[22..]; + let oplog_len = u32::from_le_bytes(body[..4].try_into().unwrap()) as usize; + body = &body[4 + oplog_len..]; + let old_state_len = u32::from_le_bytes(body[..4].try_into().unwrap()) as usize; + let state_len_pos = 22 + 4 + oplog_len; + let state_start = state_len_pos + 4; + let state_end = state_start + old_state_len; + snapshot[state_len_pos..state_start] + .copy_from_slice(&(state_bytes.len() as u32).to_le_bytes()); + snapshot.splice(state_start..state_end, state_bytes.iter().copied()); + recheck_fast_blob(snapshot) + } + + fn insert_many_with_single_list_op( + txn: &mut crate::txn::Transaction, + list: &crate::handler::ListHandler, + pos: usize, + values: Vec, + ) { + let len = values.len(); + let inner = list.inner.try_attached_state().unwrap(); + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::List(ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(values)), + pos, + }), + EventHint::InsertList { + len: len as u32, + pos, + }, + &inner.doc, + ) + .unwrap(); + } + + #[test] + fn richtext_handler() { + let loro = LoroDoc::new(); + loro.set_peer_id(1).unwrap(); + let loro2 = LoroDoc::new(); + loro2.set_peer_id(2).unwrap(); + + let mut txn = loro.txn().unwrap(); + let text = txn.get_text("hello"); + text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let exported = loro.export(ExportMode::all_updates()).unwrap(); + + loro2.import(&exported).unwrap(); + let mut txn = loro2.txn().unwrap(); + let text = txn.get_text("hello"); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello"); + text.insert_with_txn(&mut txn, 5, " world", PosType::Unicode) + .unwrap(); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello world"); + txn.commit().unwrap(); + + loro.import(&loro2.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let txn = loro.txn().unwrap(); + let text = txn.get_text("hello"); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello world"); + txn.commit().unwrap(); + + // test checkout + loro.checkout(&Frontiers::from_id(ID::new(2, 1))).unwrap(); + assert_eq!(&**text.get_value().as_string().unwrap(), "hello w"); + } + + #[test] + fn richtext_handler_concurrent() { + let loro = LoroDoc::new(); + let mut txn = loro.txn().unwrap(); + let handler = loro.get_text("richtext"); + handler + .insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + for i in 0..100 { + let new_loro = LoroDoc::new(); + new_loro + .import(&loro.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let mut txn = new_loro.txn().unwrap(); + let handler = new_loro.get_text("richtext"); + handler + .insert_with_txn(&mut txn, i % 5, &i.to_string(), PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + loro.import( + &new_loro + .export(ExportMode::updates(&loro.oplog_vv())) + .unwrap(), + ) + .unwrap(); + } + } + + #[test] + fn list_import_batch_stays_consistent_after_repeated_tail_splits() { + let doc_a = LoroDoc::new(); + doc_a.set_peer_id(1).unwrap(); + let mut txn = doc_a.txn().unwrap(); + let list_a = txn.get_list("list"); + insert_many_with_single_list_op( + &mut txn, + &list_a, + 0, + (0..300).map(|i| LoroValue::I64(i)).collect(), + ); + txn.commit().unwrap(); + + let doc_b = LoroDoc::new(); + doc_b.set_peer_id(2).unwrap(); + doc_b + .import(&doc_a.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + + let list_b = doc_b.get_list("list"); + let mut vv = doc_a.oplog_vv(); + let mut updates = Vec::new(); + for (i, pos) in [100, 201, 252, 278].into_iter().enumerate() { + list_b.insert(pos, 1000 + i as i64).unwrap(); + updates.push(doc_b.export(ExportMode::updates(&vv)).unwrap()); + vv = doc_b.oplog_vv(); + } + + doc_a.import_batch(&updates).unwrap(); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); + assert_eq!(doc_a.get_deep_value(), doc_b.get_deep_value()); + } + + #[test] + fn richtext_handler_mark() { + let loro = LoroDoc::new_auto_commit(); + let handler = loro.get_text("richtext"); + handler.insert(0, "hello world", PosType::Unicode).unwrap(); + handler + .mark(0, 5, "bold", true.into(), PosType::Event) + .unwrap(); + loro.commit_then_renew(); + + // assert has bold + let value = handler.get_richtext_value(); + assert_eq!(value[0]["insert"], "hello".into()); + let meta = value[0]["attributes"].as_map().unwrap(); + assert_eq!(meta.len(), 1); + meta.get("bold").unwrap(); + + let loro2 = LoroDoc::new_auto_commit(); + loro2 + .import(&loro.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + let handler2 = loro2.get_text("richtext"); + assert_eq!(&**handler2.get_value().as_string().unwrap(), "hello world"); + + // assert has bold + let value = handler2.get_richtext_value(); + assert_eq!(value[0]["insert"], "hello".into()); + let meta = value[0]["attributes"].as_map().unwrap(); + assert_eq!(meta.len(), 1); + meta.get("bold").unwrap(); + + // insert after bold should be bold + { + handler2.insert(5, " new", PosType::Unicode).unwrap(); + let value = handler2.get_richtext_value(); + assert_eq!( + value.to_json_value(), + serde_json::json!([ + {"insert": "hello new", "attributes": {"bold": true}}, + {"insert": " world"} + ]) + ); + } + } + + #[test] + fn richtext_snapshot() { + let loro = LoroDoc::new(); + let mut txn = loro.txn().unwrap(); + let handler = loro.get_text("richtext"); + handler + .insert_with_txn(&mut txn, 0, "hello world", PosType::Unicode) + .unwrap(); + handler + .mark_with_txn(&mut txn, 0, 5, "bold", true.into(), PosType::Event) + .unwrap(); + txn.commit().unwrap(); + + let loro2 = LoroDoc::new(); + loro2 + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let handler2 = loro2.get_text("richtext"); + assert_eq!( + handler2.get_richtext_value().to_json_value(), + serde_json::json!([ + {"insert": "hello", "attributes": {"bold": true}}, + {"insert": " world"} + ]) + ); + } + + #[test] + fn text_snapshot_string_queries_do_not_decode_state() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "a😀文", PosType::Unicode).unwrap(); + text.mark(1, 3, "bold", true.into(), PosType::Unicode) + .unwrap(); + + let restored = LoroDoc::new(); + restored + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let text = restored.get_text("text"); + assert!(!text.attached_handler().unwrap().has_decoded_state()); + + assert_eq!(text.len_unicode(), 3); + assert_eq!(text.len_utf16(), 4); + assert_eq!(text.len_utf8(), "a😀文".len()); + assert_eq!(text.char_at(1, PosType::Unicode).unwrap(), '😀'); + assert_eq!(text.slice(1, 3, PosType::Unicode).unwrap(), "😀文"); + assert_eq!( + text.convert_pos(2, PosType::Unicode, PosType::Utf16), + Some(3) + ); + assert!(matches!( + text.delete_utf16(2, 1), + Err(LoroError::UTF16InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.delete_utf8(2, 1), + Err(LoroError::UTF8InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.slice_delta(2, 3, PosType::Utf16), + Err(LoroError::UTF16InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.slice_delta(2, 3, PosType::Bytes), + Err(LoroError::UTF8InUnicodeCodePoint { pos: 2 }) + )); + assert!(!text.attached_handler().unwrap().has_decoded_state()); + + assert_eq!(text.get_delta().len(), 2); + assert!(text.attached_handler().unwrap().has_decoded_state()); + } + + #[test] + fn text_lazy_event_queries_match_decoded_state() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "ab😀cd", PosType::Unicode).unwrap(); + text.mark(1, 4, "bold", true.into(), PosType::Unicode) + .unwrap(); + text.mark(2, 3, "link", "x".into(), PosType::Unicode) + .unwrap(); + + let lazy_doc = LoroDoc::new(); + lazy_doc + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let lazy_text = lazy_doc.get_text("text"); + + let decoded_doc = LoroDoc::new(); + decoded_doc + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let decoded_text = decoded_doc.get_text("text"); + decoded_text.get_delta(); + + assert!(!lazy_text.attached_handler().unwrap().has_decoded_state()); + assert!(decoded_text.attached_handler().unwrap().has_decoded_state()); + + for pos_type in [ + PosType::Event, + PosType::Unicode, + PosType::Utf16, + PosType::Bytes, + ] { + assert_eq!(lazy_text.len(pos_type), decoded_text.len(pos_type)); + for pos in 0..=decoded_text.len(pos_type) { + assert_eq!( + lazy_text.convert_pos(pos, pos_type, PosType::Unicode), + decoded_text.convert_pos(pos, pos_type, PosType::Unicode), + "convert {pos_type:?} pos {pos} to unicode" + ); + assert_eq!( + lazy_text.convert_pos(pos, pos_type, PosType::Event), + decoded_text.convert_pos(pos, pos_type, PosType::Event), + "convert {pos_type:?} pos {pos} to event" + ); + if pos < decoded_text.len(pos_type) { + assert_eq!( + lazy_text.char_at(pos, pos_type), + decoded_text.char_at(pos, pos_type), + "char_at {pos_type:?} pos {pos}" + ); + } + for end in pos..=decoded_text.len(pos_type) { + assert_eq!( + lazy_text.slice(pos, end, pos_type), + decoded_text.slice(pos, end, pos_type), + "slice {pos_type:?} {pos}..{end}" + ); + } + } + } + } + + #[test] + fn deep_value_with_id_uses_lazy_values_for_snapshot_roots() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "hello", PosType::Unicode).unwrap(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let list = loro.get_list("list"); + list.push("item").unwrap(); + + let restored = LoroDoc::new(); + restored + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let text = restored.get_text("text"); + let map = restored.get_map("map"); + let list = restored.get_list("list"); + + let value = restored.get_deep_value_with_id(); + assert_eq!(value["text"]["value"], "hello".into()); + assert_eq!(value["map"]["value"]["key"], "value".into()); + assert_eq!(value["list"]["value"][0], "item".into()); + assert!(!text.attached_handler().unwrap().has_decoded_state()); + assert!(!map.attached_handler().unwrap().has_decoded_state()); + assert!(!list.attached_handler().unwrap().has_decoded_state()); + } + + #[test] + fn lazy_value_reads_do_not_write_stale_snapshot_after_mutation() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "old").unwrap(); + let child = map + .insert_container("child", MapHandler::new_detached()) + .unwrap(); + child.insert("nested", "old").unwrap(); + let list = loro.get_list("list"); + list.push("old").unwrap(); + let child_list = list.push_container(ListHandler::new_detached()).unwrap(); + child_list.push("nested-old").unwrap(); + + let restored = LoroDoc::new(); + restored + .import(&loro.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + let map = restored.get_map("map"); + let list = restored.get_list("list"); + + assert_eq!(map.get("key").unwrap(), "old".into()); + assert_eq!(list.get(0).unwrap(), "old".into()); + let child = match map.get_("child").unwrap() { + ValueOrHandler::Handler(handler) => handler.into_map().unwrap(), + ValueOrHandler::Value(value) => panic!("expected child map, got {value:?}"), + }; + let child_list = match list.get_(1).unwrap() { + ValueOrHandler::Handler(handler) => handler.into_list().unwrap(), + ValueOrHandler::Value(value) => panic!("expected child list, got {value:?}"), + }; + + map.insert("key", "new").unwrap(); + child.insert("nested", "new").unwrap(); + list.delete(0, 1).unwrap(); + list.insert(0, "new").unwrap(); + child_list.delete(0, 1).unwrap(); + child_list.insert(0, "nested-new").unwrap(); + restored.commit_then_renew(); + + let roundtrip = LoroDoc::new(); + roundtrip + .import(&restored.export(ExportMode::snapshot()).unwrap()) + .unwrap(); + assert_eq!( + roundtrip.get_deep_value().to_json_value(), + serde_json::json!({ + "map": { "key": "new", "child": { "nested": "new" } }, + "list": ["new", ["nested-new"]] + }) + ); + } + + #[test] + fn fast_snapshot_with_trailing_bytes_is_rejected_on_import() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let mut snapshot = loro.export(ExportMode::snapshot()).unwrap(); + snapshot.push(0xff); + let corrupted = recheck_fast_blob(snapshot); + + let doc = LoroDoc::new(); + assert!(doc.import(&corrupted).is_err()); + } + + #[test] + fn fast_snapshot_with_trailing_bytes_is_rejected_by_meta_decoder() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let mut snapshot = loro.export(ExportMode::snapshot()).unwrap(); + snapshot.push(0xff); + let corrupted = recheck_fast_blob(snapshot); + + assert!(LoroDoc::decode_import_blob_meta(&corrupted, true).is_err()); + } + + #[test] + fn fast_snapshot_empty_sstable_meta_is_rejected_on_import() { + let loro = LoroDoc::new_auto_commit(); + let map = loro.get_map("map"); + map.insert("key", "value").unwrap(); + let snapshot = loro.export(ExportMode::snapshot()).unwrap(); + + let mut malformed_state = Vec::new(); + malformed_state.extend_from_slice(b"LORO"); + malformed_state.push(0); + malformed_state.extend_from_slice(&0u32.to_le_bytes()); + let checksum = xxhash_rust::xxh32::xxh32(&[], u32::from_le_bytes(*b"LORO")); + malformed_state.extend_from_slice(&checksum.to_le_bytes()); + malformed_state.extend_from_slice(&5u32.to_le_bytes()); + let corrupted = replace_fast_snapshot_state_bytes(snapshot, &malformed_state); + + let doc = LoroDoc::new(); + assert!(doc.import(&corrupted).is_err()); + } + + #[test] + fn tree_meta() { + let loro = LoroDoc::new_auto_commit(); + loro.set_peer_id(1).unwrap(); + let tree = loro.get_tree("root"); + let id = tree.create(TreeParentId::Root).unwrap(); + let meta = tree.get_meta(id).unwrap(); + meta.insert("a", 123).unwrap(); + loro.commit_then_renew(); + let meta = tree.get_meta(id).unwrap(); + assert_eq!(meta.get("a").unwrap(), 123.into()); + assert_eq!( + json!([{"parent":null,"meta":{"a":123},"id":"0@1","index":0,"children":[],"fractional_index":"80"}]), + tree.get_deep_value().to_json_value() + ); + let bytes = loro.export(ExportMode::snapshot()).unwrap(); + let loro2 = LoroDoc::new(); + loro2.import(&bytes).unwrap(); + } + + #[test] + fn tree_meta_event() { + use std::sync::Arc; + let loro = LoroDoc::new_auto_commit(); + let tree = loro.get_tree("root"); + let text = loro.get_text("text"); + + let id = tree.create(TreeParentId::Root).unwrap(); + let meta = tree.get_meta(id).unwrap(); + meta.insert("a", 1).unwrap(); + text.insert(0, "abc", PosType::Unicode).unwrap(); + let _id2 = tree.create(TreeParentId::Root).unwrap(); + meta.insert("b", 2).unwrap(); + + let loro2 = LoroDoc::new_auto_commit(); + let _g = loro2.subscribe_root(Arc::new(|e| { + println!("{} {:?} ", e.event_meta.by, e.event_meta.diff) + })); + loro2 + .import(&loro.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!(loro.get_deep_value(), loro2.get_deep_value()); + } + + #[test] + fn richtext_apply_delta() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.apply_delta(&[TextDelta::Insert { + insert: "Hello World!".into(), + attributes: None, + }]) + .unwrap(); + dbg!(text.get_richtext_value()); + text.apply_delta(&[ + TextDelta::Retain { + retain: 6, + attributes: Some(fx_map!("italic".into() => loro_common::LoroValue::Bool(true))), + }, + TextDelta::Insert { + insert: "New ".into(), + attributes: Some(fx_map!("bold".into() => loro_common::LoroValue::Bool(true))), + }, + ]) + .unwrap(); + dbg!(text.get_richtext_value()); + loro.commit_then_renew(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([ + {"insert": "Hello ", "attributes": {"italic": true}}, + {"insert": "New ", "attributes": {"bold": true}}, + {"insert": "World!"} + + ]) + ) + } + + #[test] + fn richtext_apply_delta_marks_without_growth() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "abc", PosType::Unicode).unwrap(); + + text.apply_delta(&[TextDelta::Retain { + retain: 3, + attributes: Some(fx_map!("bold".into() => LoroValue::Bool(true))), + }]) + .unwrap(); + loro.commit_then_renew(); + + assert_eq!(text.to_string(), "abc"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert": "abc", "attributes": {"bold": true}}]) + ); + } + + #[test] + fn richtext_apply_delta_grows_for_mark_gap() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + + text.apply_delta(&[TextDelta::Retain { + retain: 1, + attributes: Some(fx_map!("bold".into() => LoroValue::Bool(true))), + }]) + .unwrap(); + loro.commit_then_renew(); + + assert_eq!(text.to_string(), "\n"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert": "\n", "attributes": {"bold": true}}]) + ); + } + + #[test] + fn richtext_apply_delta_ignores_empty_inserts() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "seed", PosType::Unicode).unwrap(); + + text.apply_delta(&[TextDelta::Insert { + insert: "".into(), + attributes: Some(fx_map!("bold".into() => LoroValue::Bool(true))), + }]) + .unwrap(); + loro.commit_then_renew(); + + assert_eq!(text.to_string(), "seed"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert": "seed"}]) + ); + } + + #[test] + fn handler_trait_dispatch_reports_attached_container_identity() { + let loro = LoroDoc::new_auto_commit(); + let handlers = [ + (loro.get_text("text").to_handler(), ContainerType::Text), + (loro.get_map("map").to_handler(), ContainerType::Map), + (loro.get_list("list").to_handler(), ContainerType::List), + ( + loro.get_movable_list("movable").to_handler(), + ContainerType::MovableList, + ), + (loro.get_tree("tree").to_handler(), ContainerType::Tree), + ]; + + for (handler, expected_type) in handlers { + assert!(handler.is_attached()); + assert!(handler.attached_handler().is_some()); + assert!(handler.doc().is_some()); + assert!(handler.get_attached().is_some()); + assert_eq!(handler.kind(), expected_type); + assert_eq!(handler.c_type(), expected_type); + assert_eq!(handler.id().container_type(), expected_type); + assert_eq!( + Handler::from_handler(handler.clone()).unwrap().c_type(), + expected_type + ); + + handler.get_value(); + handler.get_deep_value(); + handler.clear().unwrap(); + } + } + + #[test] + fn handler_trait_dispatch_reports_detached_container_identity() { + let handlers = [ + ( + Handler::new_unattached(ContainerType::Text), + ContainerType::Text, + ), + ( + Handler::new_unattached(ContainerType::Map), + ContainerType::Map, + ), + ( + Handler::new_unattached(ContainerType::List), + ContainerType::List, + ), + ( + Handler::new_unattached(ContainerType::MovableList), + ContainerType::MovableList, + ), + ( + Handler::new_unattached(ContainerType::Tree), + ContainerType::Tree, + ), + ]; + + for (handler, expected_type) in handlers { + assert!(!handler.is_attached()); + assert!(handler.attached_handler().is_none()); + assert!(handler.doc().is_none()); + assert!(handler.get_attached().is_none()); + assert_eq!(handler.kind(), expected_type); + assert_eq!(handler.c_type(), expected_type); + assert_eq!(handler.id().container_type(), expected_type); + assert_eq!(handler.idx().get_type(), expected_type); + assert_eq!( + Handler::from_handler(handler.clone()).unwrap().c_type(), + expected_type + ); + } + } + + #[test] + fn attaching_detached_handlers_sets_parent_and_attached_back_reference() { + let loro = LoroDoc::new_auto_commit(); + + let map = loro.get_map("map"); + let detached_text = TextHandler::new_detached(); + detached_text + .insert(0, "detached", PosType::Unicode) + .unwrap(); + let attached_text = map.insert_container("text", detached_text.clone()).unwrap(); + assert!(attached_text.is_attached()); + assert_eq!(attached_text.to_string(), "detached"); + assert_eq!(attached_text.parent().unwrap().c_type(), ContainerType::Map); + assert_eq!( + detached_text.get_attached().unwrap().id(), + attached_text.id() + ); + + let list = loro.get_list("list"); + let detached_map = MapHandler::new_detached(); + detached_map.insert("k", 1_i64).unwrap(); + let attached_map = list.insert_container(0, detached_map.clone()).unwrap(); + assert!(attached_map.is_attached()); + assert_eq!(attached_map.parent().unwrap().c_type(), ContainerType::List); + assert_eq!(detached_map.get_attached().unwrap().id(), attached_map.id()); + + let movable = loro.get_movable_list("movable"); + let detached_list = ListHandler::new_detached(); + detached_list.push("item").unwrap(); + let attached_list = movable.insert_container(0, detached_list.clone()).unwrap(); + assert!(attached_list.is_attached()); + assert_eq!( + attached_list.parent().unwrap().c_type(), + ContainerType::MovableList + ); + assert_eq!( + detached_list.get_attached().unwrap().id(), + attached_list.id() + ); + + let nested = attached_map + .insert_container("movable", MovableListHandler::new_detached()) + .unwrap(); + assert_eq!(nested.parent().unwrap().id(), attached_map.id()); + } + + #[test] + fn unknown_handler_reports_identity_without_materializing_value() { + let loro = LoroDoc::new_auto_commit(); + let id = ContainerID::Root { + name: "unknown".into(), + container_type: ContainerType::Unknown(7), + }; + let handler = Handler::new_attached(id.clone(), loro.clone()); + let unknown = handler.as_unknown().unwrap(); + + assert!(unknown.is_attached()); + assert_eq!(unknown.kind(), ContainerType::Unknown(7)); + assert_eq!(unknown.id(), id); + assert_eq!(unknown.to_handler().c_type(), ContainerType::Unknown(7)); + assert!(unknown.doc().is_some()); + assert!(!unknown.is_deleted()); + assert_eq!(format!("{unknown:?}"), "UnknownHandler"); + assert!(unknown.get_attached().is_some()); + assert!(super::UnknownHandler::from_handler(handler).is_some()); + } +} diff --git a/vendor/loro-internal/src/handler/movable_list_apply_delta.rs b/vendor/loro-internal/src/handler/movable_list_apply_delta.rs new file mode 100644 index 00000000..5e678af8 --- /dev/null +++ b/vendor/loro-internal/src/handler/movable_list_apply_delta.rs @@ -0,0 +1,375 @@ +use super::*; + +#[derive(Debug)] +struct ReplacementContext<'a> { + index: &'a mut usize, + index_shift: &'a mut usize, + to_delete: &'a mut FxHashMap, + container_remap: &'a mut FxHashMap, + deleted_indices: &'a mut Vec, + next_deleted: &'a mut BinaryHeap>, +} + +impl MovableListHandler { + /// Applies a delta to the movable list handler. + /// + /// This function processes the given delta, performing the necessary insertions, + /// deletions, and moves to update the list accordingly. It handles container elements, + /// maintains a map for remapping container IDs, and ensures proper indexing throughout + /// the operation. + /// + /// # Arguments + /// + /// * `delta` - A delta representing the changes to apply. + /// * `container_remap` - A map used to remap container IDs during the operation. + /// + /// # Returns + /// + /// * `LoroResult<()>` - Returns `Ok(())` if successful, or an error if something goes wrong. + #[tracing::instrument(level = "debug", skip_all)] + pub fn apply_delta( + &self, + delta: loro_delta::DeltaRope< + loro_delta::array_vec::ArrayVec, + crate::event::ListDeltaMeta, + >, + container_remap: &mut FxHashMap, + ) -> LoroResult<()> { + { + // Test whether the delta is valid + let len = self.len(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += *len; + } + loro_delta::DeltaItem::Replace { delete, .. } => { + index += *delete; + } + } + + if index > len { + return Err(LoroError::OutOfBound { + pos: index, + len, + info: "apply_delta".into(), + }); + } + } + } + + match &self.inner { + MaybeDetached::Detached(_) => { + unimplemented!(); + } + MaybeDetached::Attached(_) => { + // use tracing::debug; + // debug!( + // "Movable list value before apply_delta: {:#?}", + // self.get_deep_value_with_id() + // ); + // debug!("Applying delta: {:#?}", &delta); + + // Preprocess deletions to build a map of containers to delete. + let mut to_delete = self.preprocess_deletions(&delta); + // Process insertions and moves. + let mut index = 0; + let mut index_shift = 0; + let mut deleted_indices = Vec::new(); + let mut next_deleted = BinaryHeap::new(); + // - positive values are retain + // - negative values are deletions + let mut delta_change: Vec = Vec::new(); + + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + delta_change.push(*len as isize); + } + loro_delta::DeltaItem::Replace { + value, + delete, + attr, + } => { + // Handle deletions in the current replace operation. + let old_index = index; + self.handle_deletions_in_replace( + *delete, + &mut index, + index_shift, + &mut next_deleted, + ); + delta_change.push(-((index - old_index) as isize)); + + // Process the insertions and moves. + let mut context = ReplacementContext { + index: &mut index, + index_shift: &mut index_shift, + to_delete: &mut to_delete, + container_remap, + deleted_indices: &mut deleted_indices, + next_deleted: &mut next_deleted, + }; + + self.process_replacements(value, attr, &mut context) + .unwrap(); + delta_change.push(value.len() as isize); + } + } + } + + // Apply any remaining deletions. + self.apply_remaining_deletions(delta_change, &mut deleted_indices)?; + Ok(()) + } + } + } + + /// Preprocess deletions to build a map of containers to delete. + /// + /// # Arguments + /// + /// * `delta` - The delta containing the deletions. + /// + /// # Returns + /// + /// * `FxHashMap` - A map of containers to their indices that need to be deleted. + fn preprocess_deletions( + &self, + delta: &loro_delta::DeltaRope< + loro_delta::array_vec::ArrayVec, + crate::event::ListDeltaMeta, + >, + ) -> FxHashMap { + let mut index = 0; + let mut to_delete = FxHashMap::default(); + + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + } + loro_delta::DeltaItem::Replace { delete, .. } => { + if *delete > 0 { + for i in index..index + *delete { + if let Some(LoroValue::Container(c)) = self.get(i) { + to_delete.insert(c, i); + } + } + index += *delete; + } + } + } + } + + to_delete + } + + /// Handles deletions' effect on the index within a replace operation. + /// It will not perform the deletions. + /// + /// # Arguments + /// + /// * `delete_len` - The number of deletions. + /// * `index` - The current index in the list. + /// * `index_shift` - The current index shift due to previous operations. + /// * `next_deleted` - A heap of indices scheduled for deletion. + fn handle_deletions_in_replace( + &self, + delete_len: usize, + index: &mut usize, + index_shift: usize, + next_deleted: &mut BinaryHeap>, + ) { + if delete_len > 0 { + let mut remaining_deletes = delete_len; + while let Some(Reverse(old_index)) = next_deleted.peek() { + if *old_index + index_shift < *index + remaining_deletes { + assert!(*index <= *old_index + index_shift); + assert!(remaining_deletes > 0); + next_deleted.pop(); + remaining_deletes -= 1; + } else { + break; + } + } + + // Increase the index by the number of deletions handled. + *index += remaining_deletes; + } + } + + /// Processes replacements, handling insertions and moves. + /// + /// # Arguments + /// + /// * `values` - The values to insert or move. + /// * `attr` - Additional attributes for the delta item. + /// * `context` - A context struct containing related parameters. + fn process_replacements( + &self, + values: &loro_delta::array_vec::ArrayVec, + attr: &crate::event::ListDeltaMeta, + context: &mut ReplacementContext, + ) -> LoroResult<()> { + for v in values.iter() { + match v { + ValueOrHandler::Value(LoroValue::Container(old_id)) => { + self.apply_insertion(attr, context, old_id.clone())?; + } + ValueOrHandler::Handler(handler) => { + let old_id = handler.id(); + self.apply_insertion(attr, context, old_id)?; + } + ValueOrHandler::Value(value) => { + self.insert(*context.index, value.clone())?; + Self::update_positions_on_insert(context.to_delete, *context.index, 1); + *context.index += 1; + *context.index_shift += 1; + } + } + } + + Ok(()) + } + + fn apply_insertion( + &self, + attr: &crate::event::ListDeltaMeta, + context: &mut ReplacementContext<'_>, + mut old_id: ContainerID, + ) -> Result<(), LoroError> { + if !context.to_delete.contains_key(&old_id) { + while let Some(new_id) = context.container_remap.get(&old_id) { + old_id = new_id.clone(); + if context.to_delete.contains_key(&old_id) { + break; + } + } + } + if let Some(old_index) = context.to_delete.remove(&old_id) { + if old_index > *context.index { + ensure_cov::notify_cov( + "loro_internal::handler::movable_list_apply_delta::process_replacements::mov_0", + ); + self.mov(old_index, *context.index)?; + context.next_deleted.push(Reverse(old_index)); + *context.index += 1; + *context.index_shift += 1; + } else { + ensure_cov::notify_cov( + "loro_internal::handler::movable_list_apply_delta::process_replacements::mov_1", + ); + self.mov(old_index, *context.index - 1)?; + } + context.deleted_indices.push(old_index); + Self::update_positions_on_delete(context.to_delete, old_index); + Self::update_positions_on_insert(context.to_delete, *context.index, 1); + } else if !attr.from_move || !self.contains_container(&old_id) { + // Insert a new container if not moved. + let new_handler = self.insert_container( + *context.index, + Handler::new_unattached(old_id.container_type()), + )?; + let new_id = new_handler.id(); + context.container_remap.insert(old_id, new_id); + Self::update_positions_on_insert(context.to_delete, *context.index, 1); + *context.index += 1; + *context.index_shift += 1; + } + Ok(()) + } + + fn contains_container(&self, id: &ContainerID) -> bool { + (0..self.len()).any(|index| { + matches!( + self.get(index), + Some(LoroValue::Container(container_id)) if container_id == *id + ) + }) + } + + /// Applies any remaining deletions after processing insertions and moves. + /// + /// # Arguments + /// + /// * `delta` - The delta containing the deletions. + /// * `deleted_indices` - A list of indices that have been deleted. + fn apply_remaining_deletions( + &self, + delta: Vec, + deleted_indices: &mut Vec, + ) -> LoroResult<()> { + // Sort deleted indices from largest to smallest. + deleted_indices.sort_by_key(|&x| std::cmp::Reverse(x)); + + let mut index: usize = 0; + for delta_item in delta.iter() { + match *delta_item { + x if x > 0 => { + index += x as usize; + } + neg_delete => { + let delete = neg_delete.unsigned_abs(); + let mut remaining_deletes = delete; + while let Some(&last) = deleted_indices.last() { + if last < index { + deleted_indices.pop(); + continue; + } + + if last < index + remaining_deletes { + deleted_indices.pop(); + remaining_deletes -= 1; + } else { + break; + } + } + + self.delete(index, remaining_deletes)?; + } + } + } + + Ok(()) + } + + /// Updates positions in the map after an insertion. + /// + /// Increments positions that are greater than or equal to the insertion index. + /// + /// # Arguments + /// + /// * `positions` - The map of positions to update. + /// * `index` - The index where the insertion occurred. + /// * `len` - The length of the insertion. + fn update_positions_on_insert( + positions: &mut FxHashMap, + index: usize, + len: usize, + ) { + for pos in positions.values_mut() { + if *pos >= index { + *pos += len; + } + } + } + + /// Updates positions in the map after a deletion. + /// + /// Decrements positions that are greater than or equal to the deletion index. + /// + /// # Arguments + /// + /// * `positions` - The map of positions to update. + /// * `index` - The index where the deletion occurred. + fn update_positions_on_delete(positions: &mut FxHashMap, index: usize) { + for pos in positions.values_mut() { + if *pos >= index { + *pos -= 1; + } + } + } +} diff --git a/vendor/loro-internal/src/handler/text_update.rs b/vendor/loro-internal/src/handler/text_update.rs new file mode 100644 index 00000000..6202262d --- /dev/null +++ b/vendor/loro-internal/src/handler/text_update.rs @@ -0,0 +1,149 @@ +use std::{char, sync::Arc}; + +use itertools::Itertools; +use rustc_hash::FxHashMap; + +use crate::diff::DiffHandler; + +use super::TextHandler; + +pub(super) struct DiffHook<'a> { + text: &'a TextHandler, + new: &'a [u32], + last_old_index: usize, + current_index: usize, +} + +impl<'a> DiffHook<'a> { + pub(crate) fn new(text: &'a TextHandler, new: &'a [u32]) -> Self { + Self { + text, + new, + last_old_index: 0, + current_index: 0, + } + } +} + +impl DiffHandler for DiffHook<'_> { + fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize) { + if old_index > self.last_old_index { + self.current_index += old_index - self.last_old_index; + self.last_old_index = old_index; + } + + self.text + .insert_unicode( + self.current_index, + &self.new[new_index..new_index + new_len] + .iter() + .map(|x| char::from_u32(*x).unwrap()) + .collect::(), + ) + .unwrap(); + self.current_index += new_len; + } + + fn delete(&mut self, old_index: usize, old_len: usize) { + self.current_index += old_index - self.last_old_index; + self.text + .delete_unicode(self.current_index, old_len) + .unwrap(); + self.last_old_index = old_index + old_len; + } +} + +pub(super) struct DiffHookForLine<'a> { + text: &'a TextHandler, + old: Vec, + new: Vec, + lines: Vec>, + lines_lookup: FxHashMap, usize>, + + last_old_index: usize, + current_index: usize, +} + +impl<'a> DiffHookForLine<'a> { + pub(crate) fn new(text: &'a TextHandler, new_str: &str) -> Self { + let mut this = Self { + text, + old: Vec::new(), + new: Vec::new(), + lines: Vec::new(), + lines_lookup: FxHashMap::default(), + last_old_index: 0, + current_index: 0, + }; + + let text_str = text.to_string(); + for line in text_str.split_inclusive('\n') { + let line: Arc = Arc::from(line); + let id = this.register_line(line); + this.old.push(id as u32); + } + + for line in new_str.split_inclusive('\n') { + let line: Arc = Arc::from(line); + let id = this.register_line(line); + this.new.push(id as u32); + } + + this + } + + fn register_line(&mut self, line: Arc) -> usize { + if let Some(&index) = self.lines_lookup.get(&line) { + return index; + } + + self.lines.push(line.clone()); + self.lines_lookup.insert(line, self.lines.len() - 1); + self.lines.len() - 1 + } + + pub fn get_old_arr(&self) -> &[u32] { + &self.old + } + + pub fn get_new_arr(&self) -> &[u32] { + &self.new + } +} + +impl DiffHandler for DiffHookForLine<'_> { + fn insert(&mut self, old_index: usize, new_index: usize, new_len: usize) { + if self.last_old_index < old_index { + assert!(self.last_old_index < old_index); + self.current_index += (self.last_old_index..old_index) + .map(|x| self.lines[self.old[x] as usize].chars().count()) + .sum::(); + self.last_old_index = old_index; + } + + let s = self.new[new_index..new_index + new_len] + .iter() + .map(|x| self.lines[*x as usize].clone()) + .join(""); + self.text.insert_unicode(self.current_index, &s).unwrap(); + self.current_index += s.chars().count(); + } + + fn delete(&mut self, old_index: usize, old_len: usize) { + if self.last_old_index != old_index { + assert!(self.last_old_index < old_index); + self.current_index += (self.last_old_index..old_index) + .map(|x| self.lines[self.old[x] as usize].chars().count()) + .sum::(); + } + + self.last_old_index = old_index + old_len; + let delete_len = (old_index..old_index + old_len) + .map(|x| self.lines[self.old[x] as usize].chars().count()) + .sum::(); + + self.text + .delete_unicode(self.current_index, delete_len) + .unwrap(); + } +} diff --git a/vendor/loro-internal/src/handler/tree.rs b/vendor/loro-internal/src/handler/tree.rs new file mode 100644 index 00000000..ca2853bf --- /dev/null +++ b/vendor/loro-internal/src/handler/tree.rs @@ -0,0 +1,1184 @@ +use std::{collections::VecDeque, sync::Arc}; + +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, ContainerType, Counter, IdFull, IdLp, LoroError, LoroResult, LoroTreeError, + LoroValue, PeerID, TreeID, ID, +}; +use rustc_hash::FxHashMap; +use smallvec::smallvec; + +use crate::{ + container::tree::tree_op::TreeOp, + delta::{TreeDiffItem, TreeExternalDiff}, + state::{ + FiIfNotConfigured, FractionalIndexGenResult, NodePosition, TreeNode, TreeNodeWithChildren, + TreeParentId, + }, + txn::{EventHint, Transaction}, + BasicHandler, HandlerTrait, MapHandler, +}; + +use super::{create_handler, Handler, MaybeDetached}; + +#[derive(Clone)] +pub struct TreeHandler { + pub(super) inner: MaybeDetached, +} + +#[derive(Clone)] +pub(super) struct TreeInner { + next_counter: Counter, + map: FxHashMap, + parent_links: FxHashMap>, + children_links: FxHashMap, Vec>, +} + +impl TreeInner { + fn new() -> Self { + TreeInner { + next_counter: 0, + map: FxHashMap::default(), + parent_links: FxHashMap::default(), + children_links: FxHashMap::default(), + } + } + + fn create(&mut self, parent: Option, index: usize) -> TreeID { + let id = TreeID::new(PeerID::MAX, self.next_counter); + self.next_counter += 1; + self.map.insert(id, MapHandler::new_detached()); + self.parent_links.insert(id, parent); + let children = self.children_links.entry(parent).or_default(); + children.insert(index, id); + id + } + + fn mov(&mut self, target: TreeID, new_parent: Option, index: usize) -> LoroResult<()> { + let old_parent = self + .parent_links + .get_mut(&target) + .ok_or(LoroTreeError::TreeNodeNotExist(target))?; + let children = self.children_links.get_mut(old_parent).unwrap(); + children.retain(|x| x != &target); + self.parent_links.insert(target, new_parent); + let children = self.children_links.entry(new_parent).or_default(); + children.insert(index, target); + Ok(()) + } + + fn delete(&mut self, id: TreeID) -> LoroResult<()> { + self.map.remove(&id); + let parent = self + .parent_links + .remove(&id) + .ok_or(LoroTreeError::TreeNodeNotExist(id))?; + let children = self.children_links.get_mut(&parent).unwrap(); + children.retain(|x| x != &id); + self.children_links.remove(&Some(id)); + Ok(()) + } + + fn get_id_by_index(&self, parent: &Option, index: usize) -> Option { + self.children_links + .get(parent) + .and_then(|x| x.get(index).cloned()) + } + + fn get_parent(&self, id: &TreeID) -> Option> { + self.parent_links.get(id).cloned() + } + + fn get_children(&self, parent: Option) -> Option> { + self.children_links.get(&parent).cloned() + } + + fn children_num(&self, parent: Option) -> Option { + self.children_links.get(&parent).map(|x| x.len()) + } + + fn is_parent(&self, target: &TreeID, parent: &Option) -> bool { + self.parent_links.get(target) == Some(parent) + } + + fn get_index_by_tree_id(&self, target: &TreeID) -> Option { + self.parent_links + .get(target) + .and_then(|parent| self.children_links.get(parent)) + .and_then(|children| children.iter().position(|x| x == target)) + } + + fn get_value(&self, deep: bool) -> LoroValue { + let mut ans = vec![]; + + let empty_vec = vec![]; + let mut q = VecDeque::from_iter( + self.children_links + .get(&None) + .unwrap_or(&empty_vec) + .iter() + .enumerate() + .zip(std::iter::repeat(None::)), + ); + + while let Some(((idx, target), parent)) = q.pop_front() { + let map = self.map.get(target).unwrap(); + let mut loro_map_value = FxHashMap::default(); + loro_map_value.insert("id".to_string(), target.to_string().into()); + let parent = parent + .map(|p| p.to_string().into()) + .unwrap_or(LoroValue::Null); + loro_map_value.insert("parent".to_string(), parent); + loro_map_value.insert( + "meta".to_string(), + if deep { + map.get_deep_value() + } else { + String::from("UnResolved").into() + }, + ); + loro_map_value.insert("index".to_string(), (idx as i64).into()); + ans.push(loro_map_value); + if let Some(children) = self.children_links.get(&Some(*target)) { + for (idx, child) in children.iter().enumerate() { + q.push_back(((idx, child), Some(*target))); + } + } + } + ans.into() + } + + fn get_nodes_under(&self, root: TreeParentId) -> Vec { + let root_id = root.tree_id(); + let mut ans = vec![]; + let empty_vec = vec![]; + let mut q = VecDeque::from_iter( + self.children_links + .get(&root_id) + .unwrap_or(&empty_vec) + .iter() + .enumerate() + .zip(std::iter::repeat(root)), + ); + + while let Some(((index, target), parent)) = q.pop_front() { + ans.push(TreeNode { + id: *target, + parent, + fractional_index: FractionalIndex::default(), + index, + last_move_op: IdFull { + peer: target.peer, + lamport: 0, + counter: target.counter, + }, + }); + if let Some(children) = self.children_links.get(&Some(*target)) { + let parent = TreeParentId::Node(*target); + q.extend( + children + .iter() + .enumerate() + .map(|(index, target)| ((index, target), parent)), + ); + } + } + + ans + } +} + +impl HandlerTrait for TreeHandler { + fn to_handler(&self) -> Handler { + Handler::Tree(self.clone()) + } + + fn attach( + &self, + txn: &mut Transaction, + parent: &BasicHandler, + self_id: ContainerID, + ) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + let inner = create_handler(parent, self_id); + let tree = inner.into_tree().unwrap(); + + let children = t.value.children_links.get(&None); + let mut q = children + .map(|c| { + VecDeque::from_iter( + c.iter() + .enumerate() + .zip(std::iter::repeat(TreeParentId::Root)), + ) + }) + .unwrap_or_default(); + while let Some(((idx, target), parent)) = q.pop_front() { + let real_id = + tree.create_with_txn(txn, parent, idx, FiIfNotConfigured::UseJitterZero)?; + let map = t.value.map.get(target).unwrap(); + map.attach( + txn, + tree.inner.try_attached_state()?, + real_id.associated_meta_container(), + )?; + + if let Some(children) = t.value.children_links.get(&Some(*target)) { + for (idx, child) in children.iter().enumerate() { + q.push_back(((idx, child), TreeParentId::Node(real_id))); + } + } + } + t.attached = tree.attached_handler().cloned(); + Ok(tree) + } + MaybeDetached::Attached(a) => { + let new_inner = create_handler(a, self_id); + let ans = new_inner.into_tree().unwrap(); + + fn attach_nodes( + source: &TreeHandler, + target: &TreeHandler, + txn: &mut Transaction, + parent: TreeParentId, + nodes: Vec, + ) -> LoroResult<()> { + for node in nodes { + let real_id = target.create_with_txn( + txn, + parent, + node.index, + FiIfNotConfigured::UseJitterZero, + )?; + source.get_meta(node.id)?.attach( + txn, + target.inner.try_attached_state()?, + real_id.associated_meta_container(), + )?; + attach_nodes( + source, + target, + txn, + TreeParentId::Node(real_id), + node.children, + )?; + } + + Ok(()) + } + + let tree_nodes = self.get_all_hierarchy_nodes_under(TreeParentId::Root); + attach_nodes(self, &ans, txn, TreeParentId::Root, tree_nodes)?; + Ok(ans) + } + } + } + + fn is_attached(&self) -> bool { + self.inner.is_attached() + } + + fn attached_handler(&self) -> Option<&BasicHandler> { + self.inner.attached_handler() + } + + fn get_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_value(false) + } + MaybeDetached::Attached(a) => a.get_value(), + } + } + + fn get_deep_value(&self) -> LoroValue { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_value(true) + } + MaybeDetached::Attached(a) => a.get_deep_value(), + } + } + + fn kind(&self) -> ContainerType { + ContainerType::Tree + } + + fn get_attached(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(d) => d.lock().attached.clone().map(|x| Self { + inner: MaybeDetached::Attached(x), + }), + MaybeDetached::Attached(_a) => Some(self.clone()), + } + } + + fn from_handler(h: Handler) -> Option { + match h { + Handler::Tree(x) => Some(x), + _ => None, + } + } + + fn doc(&self) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => Some(a.doc()), + } + } +} + +impl std::fmt::Debug for TreeHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.inner { + MaybeDetached::Detached(_) => write!(f, "TreeHandler Detached"), + MaybeDetached::Attached(a) => write!(f, "TreeHandler {}", a.id), + } + } +} + +impl TreeHandler { + /// Create a new container that is detached from the document. + /// + /// The edits on a detached container will not be persisted/synced. + /// To attach the container to the document, please insert it into an attached + /// container. + pub fn new_detached() -> Self { + Self { + inner: MaybeDetached::new_detached(TreeInner::new()), + } + } + + pub fn delete(&self, target: TreeID) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + t.value.delete(target)?; + Ok(()) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| self.delete_with_txn(txn, target)), + } + } + + pub(crate) fn delete_with_txn(&self, txn: &mut Transaction, target: TreeID) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + let index = match self.get_index_by_tree_id(&target) { + Some(i) => i, + None => { + return Err(LoroTreeError::TreeNodeDeletedOrNotExist(target).into()); + } + }; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Delete { target })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent: self + .get_node_parent(&target) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(target))?, + old_index: index + }, + }]), + &inner.doc, + ) + } + + pub fn create(&self, parent: TreeParentId) -> LoroResult { + match parent { + TreeParentId::Deleted | TreeParentId::Unexist => { + return Err(LoroTreeError::InvalidParent.into()); + } + _ => {} + } + let index: usize = self.children_num(&parent).unwrap_or(0); + match &self.inner { + MaybeDetached::Detached(t) => { + let t = &mut t.lock().value; + Ok(t.create(parent.tree_id(), index)) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.create_with_txn(txn, parent, index, FiIfNotConfigured::Zero)) + } + } + } + + pub fn create_at(&self, parent: TreeParentId, index: usize) -> LoroResult { + match parent { + TreeParentId::Deleted | TreeParentId::Unexist => { + return Err(LoroTreeError::InvalidParent.into()); + } + _ => {} + } + let children_len = self.children_num(&parent).unwrap_or(0); + if index > children_len { + return Err(LoroTreeError::IndexOutOfBound { + len: children_len, + index, + } + .into()); + } + match &self.inner { + MaybeDetached::Detached(t) => { + let t = &mut t.lock().value; + Ok(t.create(parent.tree_id(), index)) + } + MaybeDetached::Attached(a) => { + a.with_txn(|txn| self.create_with_txn(txn, parent, index, FiIfNotConfigured::Throw)) + } + } + } + + /// For undo/redo, Specify the TreeID of the created node + pub(crate) fn create_at_with_target_for_apply_diff( + &self, + parent: TreeParentId, + position: FractionalIndex, + target: TreeID, + ) -> LoroResult { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!(); + }; + + if let Some(p) = self.get_node_parent(&target) { + if p == parent { + return Ok(false); + // If parent is deleted, we need to create the node, so this op from move_apply_diff + } + match p { + TreeParentId::Node(p) => { + if !self.is_node_unexist(&target) && !self.is_node_deleted(&p)? { + return self.move_at_with_target_for_apply_diff(parent, position, target); + } + } + TreeParentId::Root => { + return self.move_at_with_target_for_apply_diff(parent, position, target); + } + TreeParentId::Deleted | TreeParentId::Unexist => {} + } + } + + let with_event = !parent + .tree_id() + .is_some_and(|p| self.is_node_deleted(&p).unwrap()); + if !with_event { + return Ok(false); + } + + // println!( + // "create_at_with_target_for_apply_diff: {:?} {:?}", + // target, parent + // ); + + let index = self + .get_index_by_fractional_index( + &parent, + &NodePosition { + position: position.clone(), + idlp: self.next_idlp(), + }, + ) + // TODO: parent has deleted？ + .unwrap_or(0); + + let children = a.with_txn(|txn| { + let inner = self.inner.try_attached_state()?; + + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Create { + target, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent, + index, + position: position.clone(), + }, + }]), + &inner.doc, + )?; + + Ok(self + .children(&TreeParentId::Node(target)) + .unwrap_or_default()) + })?; + for child in children { + let position = self.get_position_by_tree_id(&child).unwrap(); + self.create_at_with_target_for_apply_diff(TreeParentId::Node(target), position, child)?; + } + Ok(true) + } + + /// For undo/redo, Specify the TreeID of the created node + pub(crate) fn move_at_with_target_for_apply_diff( + &self, + parent: TreeParentId, + position: FractionalIndex, + target: TreeID, + ) -> LoroResult { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!(); + }; + + // // the move node does not exist, create it + // if self.is_node_unexist(&target) || self.is_node_deleted(&target).unwrap() { + // return self.create_at_with_target_for_apply_diff(parent, position, target); + // } + + if let (Some(p), Some(fi)) = ( + self.get_node_parent(&target), + self.get_position_by_tree_id(&target), + ) { + if p == parent && position == fi { + return Ok(false); + } + } + + let old_parent = self.get_node_parent(&target).unwrap(); + let old_index = self.get_index_by_tree_id(&target).unwrap(); + let mut index = self + .get_index_by_fractional_index( + &parent, + &NodePosition { + position: position.clone(), + idlp: self.next_idlp(), + }, + ) + .unwrap_or(0); + if old_parent == parent && old_index < index { + index -= 1; + } + let with_event = match parent.tree_id() { + Some(p) => !self.is_node_deleted(&p)?, + None => true, + }; + + if !with_event { + return Ok(false); + } + + // println!( + // "move_at_with_target_for_apply_diff: {:?} {:?}", + // target, parent + // ); + + a.with_txn(|txn| { + let inner = self.inner.try_attached_state()?; + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Move { + target, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent, + index, + position: position.clone(), + // the old parent should be exist, so we can unwrap + old_parent, + old_index, + }, + }]), + &inner.doc, + ) + })?; + Ok(true) + } + + pub(crate) fn create_with_txn( + &self, + txn: &mut Transaction, + parent: TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> LoroResult { + let inner = self.inner.try_attached_state()?; + let target = TreeID::from_id(txn.next_id()); + + match self.generate_position_at(&target, &parent, index, cfg) { + FractionalIndexGenResult::Ok(position) => { + self.create_with_position(inner, txn, target, parent, index, position) + } + FractionalIndexGenResult::Rearrange(ids) => { + for (i, (id, position)) in ids.into_iter().enumerate() { + if i == 0 { + self.create_with_position(inner, txn, id, parent, index, position)?; + continue; + } + self.mov_with_position(inner, txn, id, parent, index + i, position, index + i)?; + } + Ok(target) + } + FractionalIndexGenResult::NotConfigured => { + Err(LoroTreeError::FractionalIndexNotEnabled.into()) + } + } + } + + pub fn mov(&self, target: TreeID, parent: TreeParentId) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(_) => { + let mut index: usize = self.children_num(&parent).unwrap_or(0); + if self.is_parent(&target, &parent) { + index -= 1; + } + self.move_to(target, parent, index) + } + MaybeDetached::Attached(a) => { + let mut index: usize = self.children_num(&parent).unwrap_or(0); + if self.is_parent(&target, &parent) { + index -= 1; + } + a.with_txn(|txn| { + self.mov_with_txn(txn, target, parent, index, FiIfNotConfigured::Zero) + }) + } + } + } + + pub fn mov_after(&self, target: TreeID, other: TreeID) -> LoroResult<()> { + let parent = self + .get_node_parent(&other) + .ok_or(LoroTreeError::TreeNodeNotExist(other))?; + let mut index = self + .get_index_by_tree_id(&other) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(other))? + + 1; + if self.is_parent(&target, &parent) { + if let Some(target_index) = self.get_index_by_tree_id(&target) { + if target_index < index { + index -= 1; + } + } + } + self.move_to(target, parent, index) + } + + pub fn mov_before(&self, target: TreeID, other: TreeID) -> LoroResult<()> { + let parent = self + .get_node_parent(&other) + .ok_or(LoroTreeError::TreeNodeNotExist(other))?; + let mut index = self + .get_index_by_tree_id(&other) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(other))?; + if self.is_parent(&target, &parent) && index >= 1 { + if let Some(target_index) = self.get_index_by_tree_id(&target) { + if target_index < index { + index -= 1; + } + } + } + self.move_to(target, parent, index) + } + + pub fn move_to(&self, target: TreeID, parent: TreeParentId, index: usize) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + t.value.mov(target, parent.tree_id(), index) + } + MaybeDetached::Attached(a) => a.with_txn(|txn| { + self.mov_with_txn(txn, target, parent, index, FiIfNotConfigured::Throw) + }), + } + } + + pub(crate) fn mov_with_txn( + &self, + txn: &mut Transaction, + target: TreeID, + parent: TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> LoroResult<()> { + let inner = self.inner.try_attached_state()?; + let mut children_len = self.children_num(&parent).unwrap_or(0); + let mut already_in_parent = false; + // check the input is valid + if self.is_parent(&target, &parent) { + // If the position after moving is same as the current position , do nothing + if let Some(current_index) = self.get_index_by_tree_id(&target) { + if current_index == index { + return Ok(()); + } + // move out first, we cannot delete the position here + // If throw error, the tree will be in a inconsistent state + children_len -= 1; + already_in_parent = true; + } + }; + if index > children_len { + return Err(LoroTreeError::IndexOutOfBound { + len: children_len, + index, + } + .into()); + } + let Some(old_index) = self.get_index_by_tree_id(&target) else { + return Err(LoroError::TreeError( + LoroTreeError::TreeNodeDeletedOrNotExist(target), + )); + }; + + if already_in_parent { + self.delete_position(&parent, &target); + } + + match self.generate_position_at(&target, &parent, index, cfg) { + FractionalIndexGenResult::Ok(position) => { + self.mov_with_position(inner, txn, target, parent, index, position, old_index) + } + FractionalIndexGenResult::Rearrange(ids) => { + for (i, (id, position)) in ids.into_iter().enumerate() { + self.mov_with_position(inner, txn, id, parent, index + i, position, old_index)?; + } + Ok(()) + } + FractionalIndexGenResult::NotConfigured => { + Err(LoroTreeError::FractionalIndexNotEnabled.into()) + } + } + } + + #[allow(clippy::too_many_arguments)] + fn create_with_position( + &self, + inner: &BasicHandler, + txn: &mut Transaction, + tree_id: TreeID, + parent: TreeParentId, + index: usize, + position: FractionalIndex, + ) -> LoroResult { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Create { + target: tree_id, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target: tree_id, + action: TreeExternalDiff::Create { + parent, + index, + position, + }, + }]), + &inner.doc, + )?; + Ok(tree_id) + } + + #[allow(clippy::too_many_arguments)] + fn mov_with_position( + &self, + inner: &BasicHandler, + txn: &mut Transaction, + target: TreeID, + parent: TreeParentId, + index: usize, + position: FractionalIndex, + old_index: usize, + ) -> LoroResult<()> { + txn.apply_local_op( + inner.container_idx, + crate::op::RawOpContent::Tree(Arc::new(TreeOp::Move { + target, + parent: parent.tree_id(), + position: position.clone(), + })), + EventHint::Tree(smallvec![TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent, + index, + position, + old_parent: self + .get_node_parent(&target) + .ok_or(LoroTreeError::TreeNodeDeletedOrNotExist(target))?, + old_index, + }, + }]), + &inner.doc, + ) + } + + pub fn get_meta(&self, target: TreeID) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + d.value + .map + .get(&target) + .cloned() + .ok_or(LoroTreeError::TreeNodeNotExist(target).into()) + } + MaybeDetached::Attached(a) => { + if self.is_node_unexist(&target) { + return Err(LoroTreeError::TreeNodeNotExist(target).into()); + } + let map_container_id = target.associated_meta_container(); + let handler = create_handler(a, map_container_id); + handler.into_map().map_err(|_| { + LoroError::DecodeError( + "Tree node's associated meta container is not a map".into(), + ) + }) + } + } + } + + pub fn is_node_unexist(&self, target: &TreeID) -> bool { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + !d.value.map.contains_key(target) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_node_unexist(target) + }), + } + } + + pub fn is_node_deleted(&self, target: &TreeID) -> LoroResult { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value + .map + .get(target) + .and(Some(true)) + .ok_or(LoroTreeError::TreeNodeNotExist(*target).into()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_node_deleted(target) + .ok_or(LoroTreeError::TreeNodeNotExist(*target).into()) + }), + } + } + + /// Get the parent of the node, if the node does not exist, return None + pub fn get_node_parent(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_parent(target).map(TreeParentId::from) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.parent(target) + }), + } + } + + // TODO: iterator + pub fn children(&self, parent: &TreeParentId) -> Option> { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_children(parent.tree_id()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_children(parent).map(|x| x.collect()) + }), + } + } + + pub fn children_num(&self, parent: &TreeParentId) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.children_num(parent.tree_id()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.children_num(parent) + }), + } + } + + /// Check if the node is exist. include deleted node. + pub fn contains(&self, target: TreeID) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.map.contains_key(&target) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + !a.is_node_unexist(&target) + }), + } + } + + pub fn get_child_at(&self, parent: &TreeParentId, index: usize) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_id_by_index(&parent.tree_id(), index) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_id_by_index(parent, index) + }), + } + } + + pub fn is_parent(&self, target: &TreeID, parent: &TreeParentId) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.is_parent(target, &parent.tree_id()) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_parent(target, parent) + }), + } + } + + /// Get all nodes in the tree, including deleted nodes + pub fn nodes(&self) -> Vec { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.map.keys().cloned().collect() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.nodes() + }), + } + } + + pub fn get_nodes_under(&self, parent: TreeParentId) -> Vec { + match &self.inner { + MaybeDetached::Detached(t) => t.lock().value.get_nodes_under(parent), + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_all_tree_nodes_under(parent) + }), + } + } + pub fn roots(&self) -> Vec { + self.children(&TreeParentId::Root).unwrap_or_default() + } + + pub fn get_all_hierarchy_nodes_under(&self, parent: TreeParentId) -> Vec { + match &self.inner { + MaybeDetached::Detached(_t) => { + unreachable!() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_all_hierarchy_nodes_under(parent) + }), + } + } + + #[allow(non_snake_case)] + pub fn __internal__next_tree_id(&self) -> TreeID { + match &self.inner { + MaybeDetached::Detached(d) => { + let d = d.lock(); + TreeID::new(PeerID::MAX, d.value.next_counter) + } + MaybeDetached::Attached(a) => a + .with_txn(|txn| Ok(TreeID::from_id(txn.next_id()))) + .unwrap(), + } + } + + fn generate_position_at( + &self, + target: &TreeID, + parent: &TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> FractionalIndexGenResult { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!() + }; + a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.generate_position_at(target, parent, index, cfg) + }) + } + + /// Get the index of the target node in the parent node + /// + /// O(logN) + pub fn get_index_by_tree_id(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.get_index_by_tree_id(target) + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_index_by_tree_id(target) + }), + } + } + + pub fn get_position_by_tree_id(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(t) => t + .lock() + .value + .parent_links + .contains_key(target) + .then(FractionalIndex::default), + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_position(target) + }), + } + } + + fn delete_position(&self, parent: &TreeParentId, target: &TreeID) { + let MaybeDetached::Attached(a) = &self.inner else { + unreachable!() + }; + a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.try_delete_position_cache(parent, target) + }) + } + + // use for apply diff + pub(crate) fn get_index_by_fractional_index( + &self, + parent: &TreeParentId, + node_position: &NodePosition, + ) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => { + unreachable!(); + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_index_by_position(parent, node_position) + }), + } + } + + pub(crate) fn next_idlp(&self) -> IdLp { + match &self.inner { + MaybeDetached::Detached(_) => { + unreachable!() + } + MaybeDetached::Attached(a) => a.with_txn(|txn| Ok(txn.next_idlp())).unwrap(), + } + } + + pub fn is_fractional_index_enabled(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => true, + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_fractional_index_enabled() + }), + } + } + + /// Set whether to generate fractional index for Tree Position. The LoroDoc is set to use jitter 0 by default. + /// + /// The jitter is used to avoid conflicts when multiple users are creating the node at the same position. + /// value 0 is default, which means no jitter, any value larger than 0 will enable jitter. + /// + /// Generally speaking, jitter will affect the growth rate of document size. + /// [Read more about it](https://www.loro.dev/blog/movable-tree#implementation-and-encoding-size) + pub fn enable_fractional_index(&self, jitter: u8) { + match &self.inner { + MaybeDetached::Detached(_) => { + // No-op on detached trees + let _ = jitter; + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.enable_generate_fractional_index(jitter); + }), + } + } + + /// Disable the fractional index generation for Tree Position when + /// you don't need the Tree's siblings to be sorted. The fractional index will be always default. + /// + /// The LoroDoc is set to disable fractional index by default. + pub fn disable_fractional_index(&self) { + match &self.inner { + MaybeDetached::Detached(_) => { + // No-op on detached trees + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state_mut().unwrap(); + a.disable_generate_fractional_index(); + }), + } + } + + pub fn is_deleted(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(_) => false, + MaybeDetached::Attached(a) => a.is_deleted(), + } + } + + pub fn is_empty(&self) -> bool { + match &self.inner { + MaybeDetached::Detached(t) => { + let t = t.lock(); + t.value.map.is_empty() + } + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.is_empty() + }), + } + } + + pub fn get_last_move_id(&self, target: &TreeID) -> Option { + match &self.inner { + MaybeDetached::Detached(_) => None, + MaybeDetached::Attached(a) => a.with_state(|state| { + let a = state.as_tree_state().unwrap(); + a.get_last_move_id(target) + }), + } + } + + pub fn clear(&self) -> LoroResult<()> { + match &self.inner { + MaybeDetached::Detached(t) => { + let mut t = t.lock(); + t.value.map.clear(); + t.value.children_links.clear(); + t.value.parent_links.clear(); + Ok(()) + } + MaybeDetached::Attached(_) => { + let nodes = self.get_nodes_under(TreeParentId::Root); + for node in nodes { + self.delete(node.id)?; + } + Ok(()) + } + } + } +} diff --git a/vendor/loro-internal/src/history_cache.rs b/vendor/loro-internal/src/history_cache.rs new file mode 100644 index 00000000..cccb0db7 --- /dev/null +++ b/vendor/loro-internal/src/history_cache.rs @@ -0,0 +1,952 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + marker::PhantomData, + ops::Bound, + sync::Arc, +}; + +use crate::sync::Mutex; +use either::Either; +use enum_as_inner::EnumAsInner; +use enum_dispatch::enum_dispatch; +use generic_btree::rle::Sliceable; +use loro_common::{ + ContainerType, Counter, HasLamport, IdFull, IdLp, InternalString, LoroValue, PeerID, ID, +}; +use rle::HasLength; +use rustc_hash::FxHashMap; + +use crate::{ + change::{Change, Lamport}, + container::{ + idx::ContainerIdx, list::list_op::InnerListOp, + richtext::richtext_state::RichtextStateChunk, tree::tree_op::TreeOp, + }, + delta::MapValue, + diff_calc::tree::{MoveLamportAndID, TreeCacheForDiff}, + encoding::value_register::ValueRegister, + op::{InnerContent, RichOp, SliceWithId}, + oplog::ChangeStore, + state::{ContainerCreationContext, GcStore}, + OpLog, VersionVector, +}; + +/// A cache for the history of a container. +/// +/// There are cases where a container needs a view of the history of the container. +/// For example, when switching to an older version, a map container may need to know the value of a key at a certain version. +/// This cache provides a faster way to do that than scanning the oplog. +#[derive(Debug)] +pub(crate) struct ContainerHistoryCache { + change_store: ChangeStore, + shallow_root_state: Option>, + for_checkout: Option, + for_importing: Option>, +} + +#[derive(Debug, Default)] +pub(crate) struct ForCheckout { + pub(crate) map: MapHistoryCache, + pub(crate) movable_list: MovableListHistoryCache, +} + +#[derive(Clone, Copy)] +pub(crate) struct HasImportingCacheMark { + _private: PhantomData<()>, +} + +impl HistoryCacheTrait for ForCheckout { + fn insert(&mut self, op: &RichOp) { + match op.raw_op().container.get_type() { + ContainerType::Map => self.map.insert(op), + ContainerType::MovableList => self.movable_list.insert(op), + _ => {} + } + } +} + +impl ContainerHistoryCache { + pub(crate) fn new(change_store: ChangeStore, gc: Option>) -> Self { + Self { + change_store, + for_checkout: Default::default(), + for_importing: Default::default(), + shallow_root_state: gc, + } + } + + pub(crate) fn insert_by_new_change( + &mut self, + change: &Change, + for_checkout: bool, + for_importing: bool, + ) { + if self.for_checkout.is_none() && self.for_importing.is_none() { + return; + } + + for op in change.ops.iter() { + match op.container.get_type() { + ContainerType::Map | ContainerType::MovableList + if self.for_checkout.is_some() && for_checkout => + { + let rich_op = RichOp::new_by_change(change, op); + self.for_checkout.as_mut().unwrap().insert(&rich_op) + } + ContainerType::Tree if self.for_importing.is_some() && for_importing => { + let container_idx = op.container; + let rich_op = RichOp::new_by_change(change, op); + let manager = self + .for_importing + .as_mut() + .unwrap() + .entry(container_idx) + .or_insert_with(|| match op.container.get_type() { + ContainerType::Tree => { + HistoryCacheForImporting::Tree(TreeOpGroup::default()) + } + _ => unreachable!(), + }); + manager.insert(&rich_op) + } + _ => continue, + } + } + } + + pub(crate) fn get_checkout_index(&mut self) -> &ForCheckout { + self.ensure_all_caches_exist(); + self.for_checkout.as_ref().unwrap() + } + + pub(crate) fn ensure_all_caches_exist(&mut self) { + let mut record_for_checkout = false; + let mut record_for_importing = false; + if self.for_checkout.is_none() { + self.for_checkout = Some(ForCheckout::default()); + record_for_checkout = true; + } + + if self.for_importing.is_none() { + self.for_importing = Some(FxHashMap::default()); + record_for_importing = true; + } + + if !record_for_checkout && !record_for_importing { + return; + } + + self.init_cache_by_visit_all_change_slow(record_for_checkout, record_for_importing); + } + + pub(crate) fn ensure_importing_caches_exist(&mut self) -> HasImportingCacheMark { + if self.for_importing.is_some() { + return HasImportingCacheMark { + _private: PhantomData, + }; + } + + self.for_importing = Some(FxHashMap::default()); + self.init_cache_by_visit_all_change_slow(false, true); + HasImportingCacheMark { + _private: PhantomData, + } + } + + fn init_cache_by_visit_all_change_slow(&mut self, for_checkout: bool, for_importing: bool) { + if self.for_checkout.is_none() && self.for_importing.is_none() { + return; + } + + if !for_checkout && !for_importing { + return; + } + + self.change_store.visit_all_changes(&mut |c| { + for op in c.ops.iter() { + match op.container.get_type() { + ContainerType::Map | ContainerType::MovableList + if self.for_checkout.is_some() && for_checkout => + { + let rich_op = RichOp::new_by_change(c, op); + self.for_checkout.as_mut().unwrap().insert(&rich_op) + } + ContainerType::Tree if self.for_importing.is_some() && for_importing => { + let container_idx = op.container; + let rich_op = RichOp::new_by_change(c, op); + let manager = self + .for_importing + .as_mut() + .unwrap() + .entry(container_idx) + .or_insert_with(|| match op.container.get_type() { + ContainerType::Tree => { + HistoryCacheForImporting::Tree(TreeOpGroup::default()) + } + _ => unreachable!(), + }); + manager.insert(&rich_op) + } + _ => continue, + } + } + }); + + let default_ctx = ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }; + + if let Some(state) = self.shallow_root_state.as_ref() { + ensure_cov::notify_cov( + "loro_internal::history_cache::init_cache_by_visit_all_change_slow::visit_gc", + ); + let mut store = state.store.lock(); + for (idx, c) in store.iter_all_containers_mut() { + match idx.get_type() { + ContainerType::Text | ContainerType::List | ContainerType::Unknown(_) => { + continue + } + #[cfg(feature = "counter")] + ContainerType::Counter => continue, + ContainerType::Map => {} + ContainerType::MovableList => {} + ContainerType::Tree => {} + } + + let state = c.get_state_mut(idx, default_ctx); + + match state { + crate::state::State::MapState(m) => { + if for_checkout { + let c = self.for_checkout.as_mut().unwrap(); + for (k, v) in m.iter() { + c.map.record_shallow_root_state_entry(idx, k, v); + } + } + } + crate::state::State::MovableListState(l) => { + for (idlp, elem) in l.elements() { + if for_checkout { + let c = self.for_checkout.as_mut().unwrap(); + let item = l.get_list_item(elem.pos).unwrap(); + c.movable_list.record_shallow_root_state( + item.id, + idlp.peer, + idlp.lamport.into(), + elem.value.clone(), + ); + } + } + } + crate::state::State::TreeState(t) => { + if for_importing { + let c = self.for_importing.as_mut().unwrap(); + let tree = c.entry(idx).or_insert_with(|| { + HistoryCacheForImporting::Tree(Default::default()) + }); + tree.as_tree_mut().unwrap().record_shallow_root_state( + t.tree_nodes() + .into_iter() + .map(|node| MoveLamportAndID { + id: node.last_move_op, + op: Arc::new(TreeOp::Create { + target: node.id, + parent: node.parent.tree_id(), + position: node.fractional_index.clone(), + }), + effected: true, + }) + .collect(), + ); + } + } + _ => unreachable!(), + } + } + } + } + + pub(crate) fn get_importing_cache( + &self, + container_idx: &ContainerIdx, + _: HasImportingCacheMark, + ) -> Option<&HistoryCacheForImporting> { + self.for_importing.as_ref().unwrap().get(container_idx) + } + + pub(crate) fn get_tree( + &self, + container_idx: &ContainerIdx, + _: HasImportingCacheMark, + ) -> Option<&TreeOpGroup> { + self.for_importing + .as_ref() + .unwrap() + .get(container_idx) + .map(|group| match group { + HistoryCacheForImporting::Tree(tree) => tree, + }) + } + + pub(crate) fn has_cache(&self) -> bool { + self.for_checkout.is_some() + } + + pub(crate) fn free(&mut self) { + self.for_checkout = None; + } + + pub(crate) fn free_all(&mut self) { + self.for_checkout = None; + self.for_importing = None; + } + + pub(crate) fn set_shallow_root_store(&mut self, shallow_root_store: Option>) { + self.shallow_root_state = shallow_root_store; + } + + pub(crate) fn find_text_chunks_in( + &self, + idx: ContainerIdx, + target_span: loro_common::IdSpan, + ) -> Vec { + ensure_cov::notify_cov("loro_internal::history_cache::find_text_chunks_in"); + let Some(state) = self.shallow_root_state.as_ref() else { + return Vec::new(); + }; + + let mut binding = state.store.lock(); + let Some(text) = binding.get_mut(idx) else { + return Vec::new(); + }; + + let text_state = text + .get_state( + idx, + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .as_richtext_state() + .unwrap(); + + // TODO: FIXME: This could be super slow in some cases + let mut ans = Vec::new(); + text_state.iter_raw(&mut |chunk| { + let c_span = chunk.get_id_span(); + if let Some(range) = target_span.get_slice_range_on(&c_span) { + let c = chunk.slice(range.0..range.1); + ans.push(c); + } + }); + + ans.sort_unstable_by_key(|x| x.counter()); + ans + } + + pub(crate) fn find_list_chunks_in( + &self, + idx: ContainerIdx, + target_span: loro_common::IdSpan, + ) -> Vec { + ensure_cov::notify_cov("loro_internal::history_cache::find_list_chunks_in"); + let Some(state) = self.shallow_root_state.as_ref() else { + return Vec::new(); + }; + + let mut binding = state.store.lock(); + let Some(list) = binding.get_mut(idx) else { + return Vec::new(); + }; + + let list_state = list.get_state( + idx, + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ); + + let mut ans = Vec::with_capacity(target_span.atom_len()); + match list_state { + crate::state::State::ListState(list) => { + for v in list.iter_with_id() { + if target_span.contains(v.id.id()) { + ans.push(SliceWithId { + values: Either::Right(v.v.clone()), + id: v.id, + elem_id: None, + }) + } + } + } + crate::state::State::MovableListState(list) => { + for (move_id, elem_id, v) in list.iter_with_last_move_id_and_elem_id() { + if target_span.contains(move_id.id()) { + ans.push(SliceWithId { + values: Either::Right(v.clone()), + id: move_id, + elem_id: Some(elem_id), + }) + } + } + } + _ => unreachable!(), + } + + ans.sort_unstable_by_key(|x| x.id.counter); + ans + } +} + +#[enum_dispatch(OpGroupTrait)] +#[derive(Debug, EnumAsInner)] +pub(crate) enum HistoryCacheForImporting { + Tree(TreeOpGroup), +} + +impl HistoryCacheForImporting { + fn insert(&mut self, op: &RichOp) { + match self { + HistoryCacheForImporting::Tree(t) => t.insert(op), + } + } +} + +#[enum_dispatch] +trait HistoryCacheTrait { + fn insert(&mut self, op: &RichOp); +} + +#[derive(Debug, Clone)] +pub(crate) struct GroupedMapOpInfo> { + pub(crate) value: T, + pub(crate) lamport: Lamport, + pub(crate) peer: PeerID, +} + +impl PartialEq for GroupedMapOpInfo { + fn eq(&self, other: &Self) -> bool { + self.lamport == other.lamport && self.peer == other.peer + } +} + +impl Eq for GroupedMapOpInfo {} + +impl PartialOrd for GroupedMapOpInfo { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for GroupedMapOpInfo { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.lamport + .cmp(&other.lamport) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +#[derive(Debug, Clone)] +struct MapHistoryCacheEntry { + container: ContainerIdx, + key: u32, + lamport: Lamport, + peer: PeerID, + counter_or_value: Either>>, +} + +impl PartialEq for MapHistoryCacheEntry { + fn eq(&self, other: &Self) -> bool { + self.container == other.container + && self.key == other.key + && self.lamport == other.lamport + && self.peer == other.peer + } +} + +impl Eq for MapHistoryCacheEntry {} + +impl PartialOrd for MapHistoryCacheEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MapHistoryCacheEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.container + .cmp(&other.container) + .then_with(|| self.key.cmp(&other.key)) + .then_with(|| self.lamport.cmp(&other.lamport)) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +#[derive(Debug, Default)] +pub(crate) struct MapHistoryCache { + keys: ValueRegister, + map: BTreeSet, +} + +impl HistoryCacheTrait for MapHistoryCache { + fn insert(&mut self, op: &RichOp) { + let container = op.raw_op().container; + let key = match &op.raw_op().content { + InnerContent::Map(map) => map.key.clone(), + _ => unreachable!(), + }; + + let key_idx = self.keys.register(&key); + self.map.insert(MapHistoryCacheEntry { + container, + key: key_idx as u32, + lamport: op.lamport(), + peer: op.peer, + counter_or_value: Either::Left(op.counter()), + }); + } +} + +impl MapHistoryCache { + fn record_shallow_root_state_entry( + &mut self, + idx: ContainerIdx, + k: &InternalString, + v: &MapValue, + ) { + let key_idx = self.keys.register(k); + self.map.insert(MapHistoryCacheEntry { + container: idx, + key: key_idx as u32, + lamport: v.lamport(), + peer: v.peer, + counter_or_value: Either::Right(Box::new(v.value.clone())), + }); + } + + /// §flowstate vendor patch (oom-leads #4): the per-KEY sibling of + /// [`Self::get_container_latest_op_at_vv`] — resolve one key's winner at + /// `vv` via a `(container, key)` prefix range instead of walking every key + /// the container ever had. `Import`-mode map diffs only need the keys the + /// span actually touched (`from ⊆ to` ⇒ an untouched key cannot change + /// winners), and registry maps on large documents hold thousands of keys. + pub fn get_container_key_latest_op_at_vv( + &self, + container: ContainerIdx, + key: &InternalString, + vv: &VersionVector, + oplog: &OpLog, + ) -> Option { + let key_ix = self.keys.get(key)? as u32; + let range = ( + Bound::Included(MapHistoryCacheEntry { + container, + key: key_ix, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + Bound::Excluded(MapHistoryCacheEntry { + container, + key: key_ix.checked_add(1)?, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + ); + for entry in self.map.range(range).rev() { + match &entry.counter_or_value { + Either::Left(cnt) => { + if vv.get(&entry.peer).copied().unwrap_or(0) > *cnt { + let id = ID::new(entry.peer, *cnt); + let op = oplog.get_op_that_includes(id).unwrap(); + debug_assert_eq!(op.atom_len(), 1); + match &op.content { + InnerContent::Map(map) => { + return Some(GroupedMapOpInfo { + value: map.value.clone(), + lamport: entry.lamport, + peer: entry.peer, + }); + } + _ => unreachable!(), + } + } + } + Either::Right(v) => { + return Some(GroupedMapOpInfo { + value: (**v).clone(), + lamport: entry.lamport, + peer: entry.peer, + }); + } + } + } + None + } + + pub fn get_container_latest_op_at_vv( + &self, + container: ContainerIdx, + vv: &VersionVector, + // PERF: utilize this lamport + _max_lamport: Lamport, + oplog: &OpLog, + ) -> FxHashMap { + let mut ans = FxHashMap::default(); + let mut last_key = u32::MAX; + + 'outer: loop { + let range = ( + Bound::Included(MapHistoryCacheEntry { + container, + key: 0, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + Bound::Excluded(MapHistoryCacheEntry { + container, + key: last_key, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + ); + + for entry in self.map.range(range).rev() { + match &entry.counter_or_value { + Either::Left(cnt) => { + if vv.get(&entry.peer).copied().unwrap_or(0) > *cnt { + let id = ID::new(entry.peer, *cnt); + let op = oplog.get_op_that_includes(id).unwrap(); + debug_assert_eq!(op.atom_len(), 1); + match &op.content { + InnerContent::Map(map) => { + ans.insert( + self.keys.get_value(entry.key as usize).unwrap().clone(), + GroupedMapOpInfo { + value: map.value.clone(), + lamport: entry.lamport, + peer: entry.peer, + }, + ); + } + _ => unreachable!(), + } + last_key = entry.key; + continue 'outer; + } + } + Either::Right(v) => { + let k = self.keys.get_value(entry.key as usize).unwrap().clone(); + ans.insert( + k, + GroupedMapOpInfo { + value: (**v).clone(), + lamport: entry.lamport, + peer: entry.peer, + }, + ); + last_key = entry.key; + continue 'outer; + } + } + } + + break; + } + + ans + } +} + +#[derive(Debug, Clone)] +pub(crate) struct GroupedTreeOpInfo { + pub(crate) counter: Counter, + pub(crate) value: Arc, +} + +#[derive(Debug, Default)] +pub(crate) struct TreeOpGroup { + ops: BTreeMap, + tree_for_diff: Mutex, +} + +impl HistoryCacheTrait for TreeOpGroup { + fn insert(&mut self, op: &RichOp) { + let tree_op = op.raw_op().content.as_tree().unwrap(); + self.ops.insert( + op.idlp(), + GroupedTreeOpInfo { + value: tree_op.clone(), + counter: op.raw_op().counter, + }, + ); + } +} + +impl TreeOpGroup { + pub fn ops(&self) -> &BTreeMap { + &self.ops + } + + pub fn tree(&self) -> &Mutex { + &self.tree_for_diff + } + + pub(crate) fn record_shallow_root_state(&mut self, nodes: Vec) { + let mut tree = self.tree_for_diff.lock(); + for node in nodes.iter() { + self.ops.insert( + node.id.idlp(), + GroupedTreeOpInfo { + counter: node.id.counter, + value: node.op.clone(), + }, + ); + } + tree.init_tree_with_shallow_root_version(nodes); + } +} + +#[derive(Debug, Default)] +pub(crate) struct MovableListHistoryCache { + move_set: BTreeSet, + set_set: BTreeSet, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +struct MovableListInnerDeltaEntry { + element_lamport: Lamport, + element_peer: PeerID, + lamport: Lamport, + peer: PeerID, + counter: Counter, +} + +#[derive(Debug, Eq, PartialEq, Clone)] +struct MovableListSetDeltaEntry { + element_lamport: Lamport, + element_peer: PeerID, + lamport: Lamport, + peer: PeerID, + counter_or_value: Either>, +} + +impl Ord for MovableListInnerDeltaEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.element_lamport + .cmp(&other.element_lamport) + .then_with(|| self.element_peer.cmp(&other.element_peer)) + .then_with(|| self.lamport.cmp(&other.lamport)) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +impl PartialOrd for MovableListInnerDeltaEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialOrd for MovableListSetDeltaEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for MovableListSetDeltaEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.element_lamport + .cmp(&other.element_lamport) + .then_with(|| self.element_peer.cmp(&other.element_peer)) + .then_with(|| self.lamport.cmp(&other.lamport)) + .then_with(|| self.peer.cmp(&other.peer)) + } +} + +impl HistoryCacheTrait for MovableListHistoryCache { + fn insert(&mut self, op: &RichOp) { + let cur_id = op.id_full(); + match &op.op().content { + InnerContent::List(l) => match l { + crate::container::list::list_op::InnerListOp::Move { elem_id, .. } => { + self.move_set.insert(MovableListInnerDeltaEntry { + element_lamport: elem_id.lamport, + element_peer: elem_id.peer, + lamport: cur_id.lamport, + peer: cur_id.peer, + counter: cur_id.counter, + }); + } + crate::container::list::list_op::InnerListOp::Set { elem_id, .. } => { + self.set_set.insert(MovableListSetDeltaEntry { + element_lamport: elem_id.lamport, + element_peer: elem_id.peer, + lamport: cur_id.lamport, + peer: cur_id.peer, + counter_or_value: Either::Left(cur_id.counter), + }); + } + _ => {} + }, + _ => unreachable!(), + } + } +} + +impl MovableListHistoryCache { + pub(crate) fn record_shallow_root_state( + &mut self, + id: IdFull, + elem_peer: PeerID, + elem_lamport: Lamport, + value: LoroValue, + ) { + self.set_set.insert(MovableListSetDeltaEntry { + element_lamport: elem_lamport, + element_peer: elem_peer, + lamport: id.lamport, + peer: id.peer, + counter_or_value: Either::Right(Box::new(value.clone())), + }); + self.move_set.insert(MovableListInnerDeltaEntry { + element_lamport: elem_lamport, + element_peer: elem_peer, + lamport: id.lamport, + peer: id.peer, + counter: id.counter, + }); + } + + pub(crate) fn last_value( + &self, + key: IdLp, + vv: &VersionVector, + max_lamport: Lamport, + oplog: &OpLog, + ) -> Option> { + self.set_set + .range(( + Bound::Included(MovableListSetDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: 0, + peer: 0, + counter_or_value: Either::Left(0), + }), + Bound::Excluded(MovableListSetDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: max_lamport, + peer: PeerID::MAX, + counter_or_value: Either::Left(Counter::MAX), + }), + )) + .rev() + .find(|e| { + let counter = match &e.counter_or_value { + Either::Left(c) => *c, + Either::Right(_) => -1, + }; + vv.get(&e.peer).copied().unwrap_or(0) > counter + }) + .map_or_else( + || { + let id = oplog.idlp_to_id(key).unwrap(); + if vv.get(&id.peer).copied().unwrap_or(0) <= id.counter { + return None; + } + let op = oplog.get_op_that_includes(id).unwrap(); + let offset = id.counter - op.counter; + match &op.content { + InnerContent::List(InnerListOp::Insert { slice, .. }) => { + let value = oplog + .arena + .get_value(slice.0.start as usize + offset as usize) + .unwrap(); + Some(GroupedMapOpInfo { + value, + lamport: key.lamport, + peer: id.peer, + }) + } + _ => { + unreachable!() + } + } + }, + |e| { + let (lamport, value) = match &e.counter_or_value { + Either::Left(c) => { + let id = ID::new(e.peer, *c); + let op = oplog.get_op_that_includes(id).unwrap(); + debug_assert_eq!(op.atom_len(), 1); + let lamport = op.lamport(); + match &op.content { + InnerContent::List(InnerListOp::Set { value, .. }) => { + (lamport, value.clone()) + } + _ => { + unreachable!() + } + } + } + Either::Right(v) => (e.lamport, v.as_ref().clone()), + }; + Some(GroupedMapOpInfo { + value: value.clone(), + lamport, + peer: e.peer, + }) + }, + ) + } + + pub(crate) fn last_pos( + &self, + key: IdLp, + vv: &VersionVector, + max_lamport: Lamport, + oplog: &OpLog, + ) -> Option { + self.move_set + .range(( + Bound::Included(MovableListInnerDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: 0, + peer: 0, + counter: 0, + }), + Bound::Excluded(MovableListInnerDeltaEntry { + element_lamport: key.lamport, + element_peer: key.peer, + lamport: max_lamport, + peer: PeerID::MAX, + counter: Counter::MAX, + }), + )) + .rev() + .find(|e| vv.get(&e.peer).copied().unwrap_or(0) > e.counter) + .map_or_else( + || { + let id = oplog.idlp_to_id(key).unwrap(); + if vv.get(&id.peer).copied().unwrap_or(0) > id.counter { + Some(IdFull::new(id.peer, id.counter, key.lamport)) + } else { + None + } + }, + |e| { + let id = ID::new(e.peer, e.counter); + let lamport = oplog.get_lamport_at(id).unwrap_or(e.lamport); + Some(IdFull::new(e.peer, e.counter, lamport)) + }, + ) + } +} diff --git a/vendor/loro-internal/src/id.rs b/vendor/loro-internal/src/id.rs new file mode 100644 index 00000000..1c8882be --- /dev/null +++ b/vendor/loro-internal/src/id.rs @@ -0,0 +1,4 @@ +pub use loro_common::Counter; +pub use loro_common::PeerID; +pub use loro_common::TreeID; +pub use loro_common::ID; diff --git a/vendor/loro-internal/src/jsonpath/ast.rs b/vendor/loro-internal/src/jsonpath/ast.rs new file mode 100644 index 00000000..cd06c317 --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/ast.rs @@ -0,0 +1,458 @@ +use crate::jsonpath::errors::JSONPathError; +use crate::jsonpath::JSONPathParser; +use std::fmt::{self, Write}; + +#[derive(Debug, Clone)] +pub struct Query { + pub segments: Segment, +} + +#[derive(Debug, Clone)] +pub enum Segment { + Root {}, + Child { + left: Box, + selectors: Vec, + }, + Recursive { + left: Box, + selectors: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum Selector { + Name { + name: String, + }, + Index { + index: i64, + }, + Slice { + start: Option, + stop: Option, + step: Option, + }, + Wild {}, + Filter { + expression: Box, + }, +} + +#[derive(Debug, Clone)] +pub enum FilterExpression { + True_ {}, + False_ {}, + Null {}, + Array { + values: Vec, + }, + StringLiteral { + value: String, + }, + Int { + value: i64, + }, + Float { + value: f64, + }, + Not { + expression: Box, + }, + Logical { + left: Box, + operator: LogicalOperator, + right: Box, + }, + Comparison { + left: Box, + operator: ComparisonOperator, + right: Box, + }, + RelativeQuery { + query: Box, + }, + RootQuery { + query: Box, + }, + Function { + name: String, + args: Vec, + }, +} + +#[derive(Debug, Clone)] +pub enum LogicalOperator { + And, + Or, +} + +#[derive(Debug, Clone)] +pub enum ComparisonOperator { + Eq, + Ne, + Ge, + Gt, + Le, + Lt, + Contains, + In, +} + +impl Query { + pub fn standard(expr: &str) -> Result { + JSONPathParser::new().parse(expr) + } + + pub fn is_singular(&self) -> bool { + self.segments.is_singular() + } +} + +impl Segment { + // Returns `true` if this query can resolve to at most one node, or `false` otherwise. + pub fn is_singular(&self) -> bool { + match self { + Segment::Child { left, selectors } => { + selectors.len() == 1 + && selectors.first().is_some_and(|selector| { + matches!(selector, Selector::Name { .. } | Selector::Index { .. }) + }) + && left.is_singular() + } + Segment::Recursive { .. } => false, + Segment::Root {} => true, + } + } +} + +impl FilterExpression { + pub fn is_literal(&self) -> bool { + matches!( + self, + FilterExpression::True_ { .. } + | FilterExpression::False_ { .. } + | FilterExpression::Null { .. } + | FilterExpression::StringLiteral { .. } + | FilterExpression::Int { .. } + | FilterExpression::Float { .. } + | FilterExpression::Array { .. } + ) + } +} + +impl fmt::Display for Query { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "${}", self.segments) + } +} + +impl fmt::Display for Segment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Segment::Child { left, selectors } => write!( + f, + "{}[{}]", + left, + selectors + .iter() + .map(|s| s.to_string()) + .collect::>() + .join(", ") + ), + Segment::Recursive { left, selectors } => write!( + f, + "{}..[{}]", + left, + selectors + .iter() + .map(|s| s.to_string()) + .collect::>() + .join(", ") + ), + Segment::Root {} => Ok(()), + } + } +} + +impl fmt::Display for Selector { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Selector::Name { name, .. } => write!(f, "'{name}'"), + Selector::Index { + index: array_index, .. + } => write!(f, "{array_index}"), + Selector::Slice { + start, stop, step, .. + } => { + write!( + f, + "{}:{}:{}", + start + .and_then(|i| Some(i.to_string())) + .unwrap_or(String::from("")), + stop.and_then(|i| Some(i.to_string())) + .unwrap_or(String::from("")), + step.and_then(|i| Some(i.to_string())) + .unwrap_or(String::from("1")), + ) + } + Selector::Wild { .. } => f.write_char('*'), + Selector::Filter { expression, .. } => write!(f, "?{expression}"), + } + } +} + +impl fmt::Display for FilterExpression { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FilterExpression::True_ { .. } => f.write_str("true"), + FilterExpression::False_ { .. } => f.write_str("false"), + FilterExpression::Null { .. } => f.write_str("null"), + FilterExpression::StringLiteral { value, .. } => write!(f, "'{value}'"), + FilterExpression::Array { values, .. } => { + write!( + f, + "[{}]", + values + .iter() + .map(|e| e.to_string()) + .collect::>() + .join(", ") + ) + } + FilterExpression::Int { value, .. } => write!(f, "{value}"), + FilterExpression::Float { value, .. } => write!(f, "{value}"), + FilterExpression::Not { expression, .. } => write!(f, "!{expression}"), + FilterExpression::Logical { + left, + operator, + right, + .. + } => write!(f, "({left} {operator} {right})"), + FilterExpression::Comparison { + left, + operator, + right, + .. + } => write!(f, "{left} {operator} {right}"), + FilterExpression::RelativeQuery { query, .. } => { + write!(f, "@{}", query.segments) + } + FilterExpression::RootQuery { query, .. } => { + write!(f, "${}", query.segments) + } + FilterExpression::Function { name, args, .. } => { + write!( + f, + "{}({})", + name, + args.iter() + .map(|a| a.to_string()) + .collect::>() + .join(", ") + ) + } + } + } +} + +impl fmt::Display for LogicalOperator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + LogicalOperator::And => f.write_str("&&"), + LogicalOperator::Or => f.write_str("||"), + } + } +} + +impl fmt::Display for ComparisonOperator { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ComparisonOperator::Eq => f.write_str("=="), + ComparisonOperator::Ne => f.write_str("!="), + ComparisonOperator::Ge => f.write_str(">="), + ComparisonOperator::Gt => f.write_str(">"), + ComparisonOperator::Le => f.write_str("<="), + ComparisonOperator::Lt => f.write_str("<"), + ComparisonOperator::Contains => f.write_str("contains"), + ComparisonOperator::In => f.write_str("in"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(expr: &str) -> Query { + Query::standard(expr).unwrap() + } + + fn relative_query(selector: Selector) -> Query { + Query { + segments: Segment::Child { + left: Box::new(Segment::Root {}), + selectors: vec![selector], + }, + } + } + + #[test] + fn query_standard_reports_singular_queries_correctly() { + let cases = [ + ("$", true), + ("$.store", true), + ("$['store']", true), + ("$.store.books[0]", true), + ("$.store.books[0][1]", true), + ("$.store.books[0, 1]", false), + ("$.store.books[*]", false), + ("$.store.books[?(@.available == true)]", false), + ("$.store..title", false), + ]; + + for (expr, expected) in cases { + assert_eq!( + parse(expr).is_singular(), + expected, + "{expr} should have singular={expected}" + ); + } + } + + #[test] + fn filter_expression_is_literal_matches_jsonpath_contract() { + let literal_cases = [ + FilterExpression::True_ {}, + FilterExpression::False_ {}, + FilterExpression::Null {}, + FilterExpression::StringLiteral { + value: "hello".to_owned(), + }, + FilterExpression::Int { value: 42 }, + FilterExpression::Float { value: 3.5 }, + FilterExpression::Array { + values: vec![ + FilterExpression::Int { value: 1 }, + FilterExpression::StringLiteral { + value: "two".to_owned(), + }, + FilterExpression::Array { + values: vec![FilterExpression::Null {}], + }, + ], + }, + ]; + + for expr in literal_cases { + assert!(expr.is_literal(), "{expr:?} should be literal"); + } + + let non_literal_cases = [ + FilterExpression::RelativeQuery { + query: Box::new(relative_query(Selector::Name { + name: "title".to_owned(), + })), + }, + FilterExpression::RootQuery { + query: Box::new(parse("$.store.books[0]")), + }, + FilterExpression::Function { + name: "count".to_owned(), + args: vec![FilterExpression::RelativeQuery { + query: Box::new(relative_query(Selector::Name { + name: "title".to_owned(), + })), + }], + }, + FilterExpression::Not { + expression: Box::new(FilterExpression::True_ {}), + }, + FilterExpression::Logical { + left: Box::new(FilterExpression::True_ {}), + operator: LogicalOperator::And, + right: Box::new(FilterExpression::False_ {}), + }, + FilterExpression::Comparison { + left: Box::new(FilterExpression::Int { value: 1 }), + operator: ComparisonOperator::Eq, + right: Box::new(FilterExpression::Int { value: 2 }), + }, + ]; + + for expr in non_literal_cases { + assert!(!expr.is_literal(), "{expr:?} should not be literal"); + } + } + + #[test] + fn query_and_ast_display_are_stable() { + let query = parse("$.store.books[0]"); + assert_eq!(query.to_string(), "$['store']['books'][0]"); + assert_eq!( + parse("$.store.books[0, 1]").to_string(), + "$['store']['books'][0, 1]" + ); + assert_eq!(parse("$.store..title").to_string(), "$['store']..['title']"); + + let selector_display = [ + ( + Selector::Name { + name: "field".into(), + }, + "'field'", + ), + (Selector::Index { index: -2 }, "-2"), + ( + Selector::Slice { + start: Some(1), + stop: Some(3), + step: None, + }, + "1:3:1", + ), + (Selector::Wild {}, "*"), + ]; + + for (selector, expected) in selector_display { + assert_eq!(selector.to_string(), expected); + } + + let filter_display = FilterExpression::Logical { + left: Box::new(FilterExpression::Comparison { + left: Box::new(FilterExpression::RelativeQuery { + query: Box::new(relative_query(Selector::Name { + name: "available".to_owned(), + })), + }), + operator: ComparisonOperator::Eq, + right: Box::new(FilterExpression::True_ {}), + }), + operator: LogicalOperator::And, + right: Box::new(FilterExpression::Not { + expression: Box::new(FilterExpression::Function { + name: "count".to_owned(), + args: vec![FilterExpression::RootQuery { + query: Box::new(parse("$.store.books[*]")), + }], + }), + }), + }; + assert_eq!( + filter_display.to_string(), + "(@['available'] == true && !count($['store']['books'][*]))" + ); + + assert_eq!(LogicalOperator::And.to_string(), "&&"); + assert_eq!(LogicalOperator::Or.to_string(), "||"); + assert_eq!(ComparisonOperator::Eq.to_string(), "=="); + assert_eq!(ComparisonOperator::Ne.to_string(), "!="); + assert_eq!(ComparisonOperator::Ge.to_string(), ">="); + assert_eq!(ComparisonOperator::Gt.to_string(), ">"); + assert_eq!(ComparisonOperator::Le.to_string(), "<="); + assert_eq!(ComparisonOperator::Lt.to_string(), "<"); + assert_eq!(ComparisonOperator::Contains.to_string(), "contains"); + assert_eq!(ComparisonOperator::In.to_string(), "in"); + } +} diff --git a/vendor/loro-internal/src/jsonpath/errors.rs b/vendor/loro-internal/src/jsonpath/errors.rs new file mode 100644 index 00000000..65e13119 --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/errors.rs @@ -0,0 +1,61 @@ +use std::fmt; + +#[derive(Debug)] +pub enum JSONPathErrorType { + LexerError, + SyntaxError, + TypeError, + NameError, +} + +#[derive(Debug)] +pub struct JSONPathError { + pub kind: JSONPathErrorType, + pub msg: String, +} + +impl JSONPathError { + pub fn new(error: JSONPathErrorType, msg: String) -> Self { + Self { kind: error, msg } + } + + pub fn syntax(msg: String) -> Self { + Self { + kind: JSONPathErrorType::SyntaxError, + msg, + } + } + + pub fn typ(msg: String) -> Self { + Self { + kind: JSONPathErrorType::TypeError, + msg, + } + } + + pub fn name(msg: String) -> Self { + Self { + kind: JSONPathErrorType::NameError, + msg, + } + } +} + +impl fmt::Display for JSONPathErrorType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JSONPathErrorType::LexerError => f.write_str("lexer error:"), + JSONPathErrorType::SyntaxError => f.write_str("syntax error:"), + JSONPathErrorType::TypeError => f.write_str("type error:"), + JSONPathErrorType::NameError => f.write_str("name error:"), + } + } +} + +impl std::error::Error for JSONPathError {} + +impl fmt::Display for JSONPathError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.msg) + } +} diff --git a/vendor/loro-internal/src/jsonpath/jsonpath.pest b/vendor/loro-internal/src/jsonpath/jsonpath.pest new file mode 100644 index 00000000..6a34d33c --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/jsonpath.pest @@ -0,0 +1,110 @@ +// JSONPath Parser +// This grammar defines the structure for parsing JSONPath expressions. +// It follows a structured approach with grouped rules for clarity and maintainability. + +// Entry Point +// Defines the overall structure of a JSONPath expression, starting with SOI (start of input) and ending with EOI (end of input). +jsonpath = _{ SOI ~ jsonpath_query ~ EOI } + +// Core Query Structure +jsonpath_query = _{ root_identifier ~ segments } + +// Segments +segments = _{ (S ~ segment)* } + +// Whitespace Handling +B = _{ "\x20" | "\x09" | "\x0A" | "\x0D" } // Basic whitespace characters: space, tab, newline, carriage return +S = _{ B* } // Optional whitespace + +// Identifiers +root_identifier = _{ "$" } // Root node identifier +current_node_identifier = _{ "@" } // Current node identifier + +// Segment Types +segment = _{ child_segment | descendant_segment } +child_segment = { bracketed_selection | ("." ~ (wildcard_selector | member_name_shorthand)) } +descendant_segment = { ".." ~ (bracketed_selection | wildcard_selector | member_name_shorthand) } + +// Bracketed Selection +bracketed_selection = { "[" ~ S ~ selector ~ (S ~ "," ~ S ~ selector)* ~ S ~ "]" } + +// Selectors +selector = _{ name_selector | wildcard_selector | slice_selector | index_selector | filter_selector } +wildcard_selector = { "*" } // Matches any element +name_selector = _{ string_literal } // Matches a specific name +index_selector = @{ int } // Matches a specific index +slice_selector = { (start ~ S)? ~ ":" ~ S ~ (stop ~ S)? ~ (":" ~ (S ~ step)?)? } // Array slicing +filter_selector = { "?" ~ S ~ logical_expr } // Filter based on logical expression + +// Integer and Slice Components +int = { "0" | ("-"? ~ ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*) } +start = @{ int } +stop = @{ int } +step = @{ int } + +// String Literals +string_literal = _{ ("\x22" ~ double_quoted ~ "\x22") | ("\x27" ~ single_quoted ~ "\x27") } +double_quoted = { double_quoted_char* } +single_quoted = { single_quoted_char* } +double_quoted_char = _{ unescaped | "\x27" | (ESC ~ "\x22") | (ESC ~ escapable) } +single_quoted_char = _{ unescaped | "\x22" | (ESC ~ "\x27") | (ESC ~ escapable) } +ESC = _{ "\x5C" } // Escape character (backslash) + +// Escapable Characters +escapable = _{ "\x62" | "\x66" | "\x6E" | "\x72" | "\x74" | "/" | "\\" | ("\x75" ~ hexchar) } +hexchar = _{ non_surrogate | (high_surrogate ~ "\\u" ~ low_surrogate) } +non_surrogate = _{ ((ASCII_DIGIT | ^"A" | ^"B" | ^"C" | ^"E" | ^"F") ~ ASCII_HEX_DIGIT{3}) | (^"D" ~ '\u{30}'..'\u{37}' ~ ASCII_HEX_DIGIT{2}) } +high_surrogate = _{ ^"D" ~ ("8" | "9" | ^"A" | ^"B") ~ ASCII_HEX_DIGIT{2} } +low_surrogate = _{ ^"D" ~ (^"C" | ^"D" | ^"E" | ^"F") ~ ASCII_HEX_DIGIT{2} } + +// Unescaped Characters +unescaped = _{ '\u{20}'..'\u{21}' | '\u{23}'..'\u{26}' | '\u{28}'..'\u{5B}' | '\u{5D}'..'\u{D7FF}' | '\u{E000}'..'\u{10FFFF}' } + +// Logical Expressions +logical_expr = _{ logical_or_expr } +logical_or_expr = { logical_and_expr ~ (S ~ "||" ~ S ~ logical_and_expr)* } +logical_and_expr = { basic_expr ~ (S ~ "&&" ~ S ~ basic_expr)* } +basic_expr = _{ paren_expr | comparison_expr | test_expr } +paren_expr = { (logical_not_op ~ S)? ~ "(" ~ S ~ logical_expr ~ S ~ ")" } +logical_not_op = { "!" } +test_expr = { (logical_not_op ~ S)? ~ (filter_query | function_expr) } + +// Queries in Filters +filter_query = _{ rel_query | root_query } +root_query = { jsonpath_query } +rel_query = { current_node_identifier ~ segments } + +// Comparison Expressions +comparison_expr = { comparable ~ S ~ comparison_op ~ S ~ comparable } +comparison_op = { "==" | "!=" | "<=" | ">=" | "<" | ">" | "contains" | "in" } +comparable = _{ literal | singular_query | function_expr } + +// Literals +literal = _{ number | string_literal | array_literal | true_literal | false_literal | null } +true_literal = { "true" } +false_literal = { "false" } +null = { "null" } +number = { (int | "-0") ~ frac? ~ exp? } +frac = { "." ~ ASCII_DIGIT+ } +exp = { ^"e" ~ ("-" | "+")? ~ ASCII_DIGIT+ } +array_literal = { "[" ~ S ~ (literal ~ (S ~ "," ~ S ~ literal)*)? ~ S ~ "]" } + +// Singular Queries +singular_query = _{ rel_singular_query | abs_singular_query } +rel_singular_query = { current_node_identifier ~ singular_query_segments } +abs_singular_query = { root_identifier ~ singular_query_segments } +singular_query_segments = _{ (S ~ (name_segment | index_segment))* } +name_segment = { ("[" ~ name_selector ~ "]") | ("." ~ member_name_shorthand) } +index_segment = { "[" ~ index_selector ~ "]" } + +// Function Expressions +function_expr = { function_name ~ "(" ~ S ~ (function_argument ~ (S ~ "," ~ S ~ function_argument)*)? ~ S ~ ")" } +function_name = { function_name_first ~ function_name_char* } +function_name_first = _{ ASCII_ALPHA_LOWER } +function_name_char = _{ function_name_first | "_" | ASCII_DIGIT } +function_argument = _{ filter_query | logical_expr | function_expr | literal } + +// Member Names +member_name_shorthand = { name_first ~ name_char* } +name_first = _{ ASCII_ALPHA | "_" | '\u{80}'..'\u{D7FF}' | '\u{E000}'..'\u{10FFFF}' } +name_char = _{ name_first | ASCII_DIGIT } \ No newline at end of file diff --git a/vendor/loro-internal/src/jsonpath/jsonpath_impl.rs b/vendor/loro-internal/src/jsonpath/jsonpath_impl.rs new file mode 100644 index 00000000..9a9aed78 --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/jsonpath_impl.rs @@ -0,0 +1,1635 @@ +use crate::handler::{ + Handler, ListHandler, MapHandler, MovableListHandler, TextHandler, TreeHandler, ValueOrHandler, +}; +use crate::jsonpath::ast::{ + ComparisonOperator, FilterExpression, LogicalOperator, Segment, Selector, +}; +use crate::jsonpath::JSONPathParser; +use crate::{HandlerTrait, LoroDoc}; +use loro_common::{ContainerID, LoroValue}; +use std::ops::ControlFlow; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum JsonPathError { + #[error("Invalid JSONPath: {0}")] + InvalidJsonPath(String), + #[error("JSONPath evaluation error: {0}")] + EvaluationError(String), +} +impl LoroDoc { + #[inline] + pub fn jsonpath(&self, jsonpath: &str) -> Result, JsonPathError> { + evaluate_jsonpath(self, jsonpath) + } +} + +enum ExprValue { + Bool(bool), + Value(LoroValue), + Nodes(Vec), +} + +pub fn evaluate_jsonpath( + root: &dyn PathValue, + jsonpath: &str, +) -> Result, JsonPathError> { + let parser = JSONPathParser::new(); + let query = parser + .parse(jsonpath) + .map_err(|e| JsonPathError::InvalidJsonPath(e.to_string()))?; + + let mut results = Vec::new(); + evaluate_segment(root, &query.segments, &mut results); + Ok(results) +} + +fn evaluate_segment(root: &dyn PathValue, segment: &Segment, results: &mut Vec) { + match segment { + Segment::Root {} => { + if let Ok(cloned) = root.clone_this() { + results.push(cloned); + } + } + Segment::Child { left, selectors } => { + if left.is_singular() && matches!(**left, Segment::Root {}) { + apply_selectors(root, root, selectors, results); + } else { + let mut intermediate = Vec::new(); + evaluate_segment(root, left, &mut intermediate); + for node in intermediate { + apply_selectors(root, &node, selectors, results); + } + } + } + Segment::Recursive { left, selectors } => { + if left.is_singular() && matches!(**left, Segment::Root {}) { + recursive_descent(root, root, selectors, results); + } else { + let mut intermediate = Vec::new(); + evaluate_segment(root, left, &mut intermediate); + for node in intermediate { + recursive_descent(root, &node, selectors, results); + } + } + } + } +} + +fn apply_selectors( + root: &dyn PathValue, + node: &dyn PathValue, + selectors: &[Selector], + results: &mut Vec, +) { + for sel in selectors { + match sel { + Selector::Name { name } => { + if let Some(child) = node.get_by_key(name) { + results.push(child); + } + } + Selector::Index { index } => { + let mut idx = *index; + if idx < 0 { + idx += node.length_for_path() as i64; + } + if idx >= 0 { + if let Some(child) = node.get_by_index(idx as isize) { + results.push(child); + } + } + } + Selector::Slice { start, stop, step } => { + let len = node.length_for_path() as i64; + let eff_start = start.unwrap_or(if step.unwrap_or(1) > 0 { 0 } else { len - 1 }); + let eff_stop = stop.unwrap_or(if step.unwrap_or(1) > 0 { len } else { -len - 1 }); + let eff_step = step.unwrap_or(1); + + if eff_step == 0 { + continue; + } + + let mut start_idx = if eff_start < 0 { + eff_start + len + } else { + eff_start + }; + let mut stop_idx = if eff_stop < 0 { + eff_stop + len + } else { + eff_stop + }; + start_idx = start_idx.max(0).min(len); + stop_idx = stop_idx.max(-1).min(len); + + // Pre-allocate if positive step for better perf + if eff_step > 0 { + let approx_count = ((stop_idx - start_idx) / eff_step).max(0) as usize; + results.reserve(approx_count); + let mut i = start_idx; + while i < stop_idx { + if let Some(child) = node.get_by_index(i as isize) { + results.push(child); + } + i += eff_step; + } + } else { + let approx_count = ((start_idx - stop_idx) / -eff_step).max(0) as usize; + results.reserve(approx_count); + let mut i = start_idx; + while i > stop_idx { + if let Some(child) = node.get_by_index(i as isize) { + results.push(child); + } + i += eff_step; + } + } + } + Selector::Wild {} => { + node.for_each_for_path(&mut |child| { + results.push(child); + ControlFlow::Continue(()) + }); + } + Selector::Filter { expression } => { + node.for_each_for_path(&mut |child| { + if eval_filter_expr(root, &child, expression) { + results.push(child); + } + ControlFlow::Continue(()) + }); + } + } + } +} + +fn recursive_descent( + root: &dyn PathValue, + node: &dyn PathValue, + selectors: &[Selector], + results: &mut Vec, +) { + // 1. apply selectors to the *current* node + apply_selectors(root, node, selectors, results); + + // 2. recurse into children + node.for_each_for_path(&mut |child| { + recursive_descent(root, &child, selectors, results); + ControlFlow::Continue(()) + }); +} + +fn eval_filter_expr( + root: &dyn PathValue, + current: &dyn PathValue, + expr: &FilterExpression, +) -> bool { + to_logical(eval_expr(root, current, expr)) +} + +fn eval_expr(root: &dyn PathValue, current: &dyn PathValue, expr: &FilterExpression) -> ExprValue { + match expr { + FilterExpression::True_ {} => ExprValue::Bool(true), + FilterExpression::False_ {} => ExprValue::Bool(false), + FilterExpression::Null {} => ExprValue::Value(LoroValue::Null), + FilterExpression::StringLiteral { value } => { + ExprValue::Value(LoroValue::String(value.clone().into())) + } + FilterExpression::Int { value } => ExprValue::Value(LoroValue::I64(*value)), + FilterExpression::Float { value } => ExprValue::Value(LoroValue::Double(*value)), + FilterExpression::Array { values } => { + let mut list = Vec::with_capacity(values.len()); + for val in values { + match eval_expr(root, current, val) { + ExprValue::Value(v) => list.push(v), + ExprValue::Bool(b) => list.push(LoroValue::Bool(b)), + ExprValue::Nodes(_) => return ExprValue::Value(LoroValue::Null), + } + } + ExprValue::Value(LoroValue::List(list.into())) + } + FilterExpression::Not { expression } => { + ExprValue::Bool(!to_logical(eval_expr(root, current, expression))) + } + FilterExpression::Logical { + left, + operator, + right, + } => { + // Short-circuit evaluation + let l = to_logical(eval_expr(root, current, left)); + match operator { + LogicalOperator::And => { + if !l { + return ExprValue::Bool(false); + } + ExprValue::Bool(to_logical(eval_expr(root, current, right))) + } + LogicalOperator::Or => { + if l { + return ExprValue::Bool(true); + } + ExprValue::Bool(to_logical(eval_expr(root, current, right))) + } + } + } + FilterExpression::Comparison { + left, + operator, + right, + } => { + let l = eval_expr(root, current, left); + let r = eval_expr(root, current, right); + ExprValue::Bool(compare_expr(l, operator, r)) + } + FilterExpression::RelativeQuery { query } => { + let mut query_results = Vec::new(); + evaluate_segment(current, &query.segments, &mut query_results); + ExprValue::Nodes(query_results) + } + FilterExpression::RootQuery { query } => { + let mut query_results = Vec::new(); + evaluate_segment(root, &query.segments, &mut query_results); + ExprValue::Nodes(query_results) + } + FilterExpression::Function { name, args } => eval_function(root, current, name, args), + } +} + +#[inline] +fn eval_function( + root: &dyn PathValue, + current: &dyn PathValue, + name: &str, + args: &[FilterExpression], +) -> ExprValue { + match name { + "count" if args.len() == 1 => { + let arg = eval_expr(root, current, &args[0]); + if let ExprValue::Nodes(ns) = arg { + ExprValue::Value(LoroValue::I64(ns.len() as i64)) + } else { + ExprValue::Value(LoroValue::I64(0)) + } + } + "length" if args.len() == 1 => { + let arg = eval_expr(root, current, &args[0]); + if let ExprValue::Value(v) = arg { + let len = match v { + LoroValue::List(l) => l.len(), + LoroValue::Map(m) => m.len(), + LoroValue::String(s) => s.len(), + _ => 0, + }; + ExprValue::Value(LoroValue::I64(len as i64)) + } else { + ExprValue::Value(LoroValue::I64(0)) + } + } + "value" if args.len() == 1 => { + let arg = eval_expr(root, current, &args[0]); + if let ExprValue::Nodes(ns) = arg { + if ns.len() == 1 { + match &ns[0] { + ValueOrHandler::Value(v) => ExprValue::Value(v.clone()), + ValueOrHandler::Handler(h) => ExprValue::Value(h.get_value()), + } + } else { + ExprValue::Value(LoroValue::Null) + } + } else { + ExprValue::Value(LoroValue::Null) + } + } + "match" => { + unimplemented!("JSONPath function `match()` is declared but not implemented"); + } + "search" => { + unimplemented!("JSONPath function `search()` is declared but not implemented"); + } + other => { + unimplemented!( + "JSONPath function `{}` is declared but does not have an evaluator", + other + ); + } + } +} + +#[inline(always)] +fn to_logical(v: ExprValue) -> bool { + match v { + ExprValue::Bool(b) => b, + ExprValue::Value(v) => match v { + LoroValue::Bool(b) => b, + LoroValue::Null => false, + LoroValue::String(s) => !s.is_empty(), + LoroValue::I64(0) => false, + LoroValue::I64(_) => true, + LoroValue::Double(f) => f != 0.0 && !f.is_nan(), + LoroValue::List(l) => !l.is_empty(), + LoroValue::Map(m) => !m.is_empty(), + _ => false, + }, + ExprValue::Nodes(ns) => !ns.is_empty(), + } +} + +#[inline] +fn compare_expr(l: ExprValue, op: &ComparisonOperator, r: ExprValue) -> bool { + match (l, r) { + (ExprValue::Value(l), ExprValue::Value(r)) => compare_values(&l, op, &r), + (ExprValue::Value(l), ExprValue::Bool(r)) => compare_values(&l, op, &LoroValue::Bool(r)), + (ExprValue::Bool(l), ExprValue::Value(r)) => compare_values(&LoroValue::Bool(l), op, &r), + (ExprValue::Bool(l), ExprValue::Bool(r)) => compare_bools(l, op, r), + (ExprValue::Nodes(ls), ExprValue::Nodes(rs)) => { + // Optimized: check if either is empty first + if ls.is_empty() || rs.is_empty() { + return false; + } + ls.iter().any(|a| { + let a_val = get_value(a); + rs.iter().any(|b| compare_values(&a_val, op, &get_value(b))) + }) + } + (ExprValue::Nodes(ls), v) => { + if ls.is_empty() { + return false; + } + let r_val = get_other_value(&v); + ls.iter().any(|a| compare_values(&get_value(a), op, &r_val)) + } + (v, ExprValue::Nodes(rs)) => { + if rs.is_empty() { + return false; + } + let l_val = get_other_value(&v); + rs.iter().any(|b| compare_values(&l_val, op, &get_value(b))) + } + } +} + +#[inline(always)] +fn compare_values(l: &LoroValue, op: &ComparisonOperator, r: &LoroValue) -> bool { + match op { + ComparisonOperator::In => { + if let LoroValue::List(list) = r { + list.iter() + .any(|item| compare_values(l, &ComparisonOperator::Eq, item)) + } else { + false + } + } + _ => match (l, r) { + (LoroValue::Double(a), LoroValue::Double(b)) => compare_nums(*a, op, *b), + (LoroValue::I64(a), LoroValue::I64(b)) => compare_i64(*a, op, *b), + (LoroValue::Double(a), LoroValue::I64(b)) => compare_nums(*a, op, *b as f64), + (LoroValue::I64(a), LoroValue::Double(b)) => compare_nums(*a as f64, op, *b), + (LoroValue::String(a), LoroValue::String(b)) => { + compare_strs(a.as_ref(), op, b.as_ref()) + } + (LoroValue::Bool(a), LoroValue::Bool(b)) => compare_bools(*a, op, *b), + (LoroValue::Null, LoroValue::Null) => matches!(op, ComparisonOperator::Eq), + _ => false, + }, + } +} + +#[inline(always)] +fn compare_i64(a: i64, op: &ComparisonOperator, b: i64) -> bool { + match op { + ComparisonOperator::Eq => a == b, + ComparisonOperator::Ne => a != b, + ComparisonOperator::Lt => a < b, + ComparisonOperator::Le => a <= b, + ComparisonOperator::Gt => a > b, + ComparisonOperator::Ge => a >= b, + _ => false, + } +} + +#[inline(always)] +fn compare_nums(a: f64, op: &ComparisonOperator, b: f64) -> bool { + match op { + ComparisonOperator::Eq => (a - b).abs() < f64::EPSILON, + ComparisonOperator::Ne => (a - b).abs() >= f64::EPSILON, + ComparisonOperator::Lt => a < b, + ComparisonOperator::Le => a <= b, + ComparisonOperator::Gt => a > b, + ComparisonOperator::Ge => a >= b, + _ => false, + } +} + +#[inline(always)] +fn compare_strs(a: &str, op: &ComparisonOperator, b: &str) -> bool { + match op { + ComparisonOperator::Eq => a == b, + ComparisonOperator::Ne => a != b, + ComparisonOperator::Lt => a < b, + ComparisonOperator::Le => a <= b, + ComparisonOperator::Gt => a > b, + ComparisonOperator::Ge => a >= b, + ComparisonOperator::Contains => a.contains(b), + _ => false, + } +} + +#[inline(always)] +fn compare_bools(a: bool, op: &ComparisonOperator, b: bool) -> bool { + match op { + ComparisonOperator::Eq => a == b, + ComparisonOperator::Ne => a != b, + _ => false, + } +} + +#[inline(always)] +fn get_value(node: &ValueOrHandler) -> LoroValue { + match node { + ValueOrHandler::Value(v) => v.clone(), + ValueOrHandler::Handler(h) => h.get_value(), + } +} + +#[inline(always)] +fn get_other_value(v: &ExprValue) -> LoroValue { + match v { + ExprValue::Value(v) => v.clone(), + ExprValue::Bool(b) => LoroValue::Bool(*b), + _ => LoroValue::Null, + } +} + +// Implement necessary trait bounds for PathValue +pub trait PathValue { + fn get_by_key(&self, key: &str) -> Option; + fn get_by_index(&self, index: isize) -> Option; + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>); + fn length_for_path(&self) -> usize; + fn get_child_by_id(&self, id: ContainerID) -> Option; + fn clone_this(&self) -> Result; +} + +// Implement PathValue for ValueOrHandler +impl PathValue for ValueOrHandler { + fn get_by_key(&self, key: &str) -> Option { + match self { + ValueOrHandler::Value(v) => v.get_by_key(key).cloned().map(ValueOrHandler::Value), + ValueOrHandler::Handler(h) => h.get_by_key(key), + } + } + + fn get_by_index(&self, index: isize) -> Option { + match self { + ValueOrHandler::Value(v) => v.get_by_index(index).cloned().map(ValueOrHandler::Value), + ValueOrHandler::Handler(h) => h.get_by_index(index), + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + match self { + ValueOrHandler::Value(v) => v.for_each_for_path(f), + ValueOrHandler::Handler(h) => h.for_each_for_path(f), + } + } + + fn length_for_path(&self) -> usize { + match self { + ValueOrHandler::Value(v) => v.length_for_path(), + ValueOrHandler::Handler(h) => h.length_for_path(), + } + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + match self { + ValueOrHandler::Handler(h) => h.get_child_by_id(id), + _ => None, + } + } + + fn clone_this(&self) -> Result { + match self { + ValueOrHandler::Value(v) => Ok(ValueOrHandler::Value(v.clone())), + ValueOrHandler::Handler(h) => Ok(ValueOrHandler::Handler(h.clone())), + } + } +} + +// Implement PathValue for LoroDoc +impl PathValue for LoroDoc { + fn get_by_key(&self, key: &str) -> Option { + self.get_by_str_path(key) + } + + fn get_by_index(&self, _index: isize) -> Option { + None // LoroDoc doesn't support index-based access + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let roots = self.state.lock().preferred_root_containers(); + let arena = self.arena(); + for c in roots { + let cid = arena.idx_to_id(c).unwrap(); + let h = self.get_handler(cid).unwrap(); + if f(ValueOrHandler::Handler(h)) == ControlFlow::Break(()) { + break; + } + } + } + + fn length_for_path(&self) -> usize { + self.state.lock().preferred_root_containers().len() + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + self.get_handler(id) + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Value(self.get_deep_value())) + } +} + +// Implement PathValue for Handler +impl PathValue for Handler { + fn get_by_key(&self, key: &str) -> Option { + match self { + Handler::Map(h) => h.get_by_key(key), + Handler::Tree(h) => h.get_by_key(key), + _ => None, + } + } + + fn get_by_index(&self, index: isize) -> Option { + match self { + Handler::List(h) => h.get_by_index(index), + Handler::MovableList(h) => h.get_by_index(index), + _ => None, + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + match self { + Handler::Map(h) => h.for_each_for_path(f), + Handler::List(h) => h.for_each_for_path(f), + Handler::MovableList(h) => h.for_each_for_path(f), + Handler::Tree(h) => h.for_each_for_path(f), + _ => {} + } + } + + fn length_for_path(&self) -> usize { + match self { + Handler::Map(h) => h.length_for_path(), + Handler::List(h) => h.length_for_path(), + Handler::MovableList(h) => h.length_for_path(), + Handler::Tree(h) => h.length_for_path(), + Handler::Text(h) => h.length_for_path(), + _ => 0, + } + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + match self { + Handler::Map(h) => h.get_child_by_id(id), + Handler::List(h) => h.get_child_by_id(id), + Handler::MovableList(h) => h.get_child_by_id(id), + Handler::Tree(h) => h.get_child_by_id(id), + _ => None, + } + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(self.clone())) + } +} + +// Implementations for specific handlers +impl PathValue for MapHandler { + fn get_by_key(&self, key: &str) -> Option { + self.get_(key) + } + + fn get_by_index(&self, _index: isize) -> Option { + None + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let mut done = false; + self.for_each(|_, v| { + if done { + return; + } + + if let ControlFlow::Break(_) = f(v) { + done = true; + } + }); + } + + fn length_for_path(&self) -> usize { + self.len() + } + + fn get_child_by_id(&self, id: ContainerID) -> Option { + self.get_child_handler(id.to_string().as_str()).ok() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::Map(self.clone()))) + } +} + +impl PathValue for ListHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, index: isize) -> Option { + if index < 0 { + self.get_(self.len() - (-index) as usize) + } else { + self.get_(index as usize) + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let mut done = false; + self.for_each(|v| { + if done { + return; + } + + if let ControlFlow::Break(_) = f(v) { + done = true; + } + }); + } + + fn length_for_path(&self) -> usize { + self.len() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + unimplemented!() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::List(self.clone()))) + } +} + +impl PathValue for MovableListHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, index: isize) -> Option { + if index < 0 { + if self.len() > (-index) as usize { + self.get_(self.len() - (-index) as usize) + } else { + None + } + } else { + self.get_(index as usize) + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + let mut done = false; + self.for_each(|v| { + if done { + return; + } + + if let ControlFlow::Break(_) = f(v) { + done = true; + } + }) + } + + fn length_for_path(&self) -> usize { + self.len() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + unimplemented!() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::MovableList(self.clone()))) + } +} + +impl PathValue for TextHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, _index: isize) -> Option { + None + } + + fn for_each_for_path(&self, _f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + // TextHandler doesn't have children to iterate over + } + + fn length_for_path(&self) -> usize { + self.len_unicode() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + None + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::Text(self.clone()))) + } +} + +impl PathValue for TreeHandler { + fn get_by_key(&self, _key: &str) -> Option { + None + } + + fn get_by_index(&self, _index: isize) -> Option { + None + } + + fn for_each_for_path(&self, _f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + unimplemented!() + } + + fn length_for_path(&self) -> usize { + unimplemented!() + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + unimplemented!() + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Handler(Handler::Tree(self.clone()))) + } +} + +impl PathValue for LoroValue { + fn get_by_key(&self, key: &str) -> Option { + match self { + LoroValue::Map(map) => map.get(key).map(|v| ValueOrHandler::Value(v.clone())), + _ => None, + } + } + + fn get_by_index(&self, index: isize) -> Option { + match self { + LoroValue::List(list) => { + let index = if index < 0 { + if list.len() > (-index) as usize { + list.len() - (-index) as usize + } else { + return None; + } + } else { + index as usize + }; + list.get(index).map(|v| ValueOrHandler::Value(v.clone())) + } + _ => None, + } + } + + fn for_each_for_path(&self, f: &mut dyn FnMut(ValueOrHandler) -> ControlFlow<()>) { + match self { + LoroValue::List(list) => { + for item in list.iter() { + if let ControlFlow::Break(_) = f(ValueOrHandler::Value(item.clone())) { + break; + } + } + } + LoroValue::Map(map) => { + for (_, value) in map.iter() { + if let ControlFlow::Break(_) = f(ValueOrHandler::Value(value.clone())) { + break; + } + } + } + _ => {} + } + } + + fn length_for_path(&self) -> usize { + match self { + LoroValue::List(list) => list.len(), + LoroValue::Map(map) => map.len(), + LoroValue::String(s) => s.len(), + _ => 0, + } + } + + fn get_child_by_id(&self, _id: ContainerID) -> Option { + None + } + + fn clone_this(&self) -> Result { + Ok(ValueOrHandler::Value(self.clone())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ListHandler, LoroDoc, LoroValue, MapHandler}; + + fn setup_test_doc() -> LoroDoc { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let store = doc.get_map("store"); + let books = store + .insert_container("books", ListHandler::new_detached()) + .unwrap(); + + // Book 1: 1984 + let book = books + .insert_container(0, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "1984").unwrap(); + book.insert("author", "George Orwell").unwrap(); + book.insert("price", 10).unwrap(); + book.insert("available", true).unwrap(); + book.insert("isbn", "978-0451524935").unwrap(); + + // Book 2: Animal Farm + let book = books + .insert_container(1, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Animal Farm").unwrap(); + book.insert("author", "George Orwell").unwrap(); + book.insert("price", 8).unwrap(); + book.insert("available", true).unwrap(); + + // Book 3: Brave New World + let book = books + .insert_container(2, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Brave New World").unwrap(); + book.insert("author", "Aldous Huxley").unwrap(); + book.insert("price", 12).unwrap(); + book.insert("available", false).unwrap(); + book.insert("isbn", "978-0060850524").unwrap(); + + // Book 4: Fahrenheit 451 + let book = books + .insert_container(3, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Fahrenheit 451").unwrap(); + book.insert("author", "Ray Bradbury").unwrap(); + book.insert("price", 9).unwrap(); + book.insert("available", true).unwrap(); + book.insert("isbn", "978-1451673319").unwrap(); + + // Book 5: The Great Gatsby + let book = books + .insert_container(4, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "The Great Gatsby").unwrap(); + book.insert("author", "F. Scott Fitzgerald").unwrap(); + book.insert("price", LoroValue::Null).unwrap(); + book.insert("available", true).unwrap(); + + // Book 6: To Kill a Mockingbird + let book = books + .insert_container(5, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "To Kill a Mockingbird").unwrap(); + book.insert("author", "Harper Lee").unwrap(); + book.insert("price", 11).unwrap(); + book.insert("available", true).unwrap(); + + // Book 7: The Catcher in the Rye + let book = books + .insert_container(6, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "The Catcher in the Rye").unwrap(); + book.insert("author", "J.D. Salinger").unwrap(); + book.insert("price", 10).unwrap(); + book.insert("available", false).unwrap(); + + // Book 8: Lord of the Flies + let book = books + .insert_container(7, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Lord of the Flies").unwrap(); + book.insert("author", "William Golding").unwrap(); + book.insert("price", 9).unwrap(); + book.insert("available", true).unwrap(); + + // Book 9: Pride and Prejudice + let book = books + .insert_container(8, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "Pride and Prejudice").unwrap(); + book.insert("author", "Jane Austen").unwrap(); + book.insert("price", 7).unwrap(); + book.insert("available", true).unwrap(); + + // Book 10: The Hobbit + let book = books + .insert_container(9, MapHandler::new_detached()) + .unwrap(); + book.insert("title", "The Hobbit").unwrap(); + book.insert("author", "J.R.R. Tolkien").unwrap(); + book.insert("price", 14).unwrap(); + book.insert("available", true).unwrap(); + + // Additional metadata + store.insert("featured_author", "George Orwell").unwrap(); + let featured_authors = store + .insert_container("featured_authors", ListHandler::new_detached()) + .unwrap(); + featured_authors.push("George Orwell").unwrap(); + featured_authors.push("Aldous Huxley").unwrap(); + featured_authors.push("Ray Bradbury").unwrap(); + store.insert("min_price", 10).unwrap(); + doc + } + + mod jsonpath_selectors { + use super::*; + + #[test] + fn handles_child_selectors() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + + #[test] + fn handles_wildcard_selector() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[*].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 10); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Brave New World", + "Fahrenheit 451", + "The Great Gatsby", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn handles_recursive_descent() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 10); + Ok(()) + } + + #[test] + fn handles_quoted_keys() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store['books'][0]['title']"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + } + + mod string_filters { + use super::*; + + #[test] + fn filters_by_exact_string_match() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.title == '1984')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + + #[test] + fn filters_by_string_contains() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.title contains 'Farm')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Animal Farm" + ); + Ok(()) + } + + #[test] + fn filters_by_recursive_string_match() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.author contains 'Orwell')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod logical_operators { + use super::*; + + #[test] + fn filters_with_and_operator() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' && @.price < 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Animal Farm" + ); + Ok(()) + } + + #[test] + fn filters_with_or_operator() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' || @.price >= 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 6); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Brave New World", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_complex_and_or_combination() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' && (@.price < 10 || @.available == true))].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_not_operator() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(!(@.available == false))].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 8); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Fahrenheit 451", + "The Great Gatsby", + "To Kill a Mockingbird", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod in_operator { + use super::*; + + #[test] + fn filters_by_author_in_list() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author in ['George Orwell', 'Jane Austen'])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm", "Pride and Prejudice"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_by_price_in_list() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price in [7, 10, 14])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 4); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Pride and Prejudice", + "The Catcher in the Rye", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_in_operator_and_null_values() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price in [null, 9])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["Fahrenheit 451", "Lord of the Flies", "The Great Gatsby"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_in_operator_in_recursive_descent() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.author in ['George Orwell', 'Ray Bradbury'])].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm", "Fahrenheit 451"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_list_in() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author in $.store.featured_authors)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 4); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm", "Brave New World", "Fahrenheit 451"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod union_and_slice_operations { + use super::*; + + #[test] + fn handles_union_indexes() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0,2].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + assert_eq!( + &**result[1].as_value().unwrap().as_string().unwrap(), + "Brave New World" + ); + Ok(()) + } + + #[test] + fn handles_union_keys() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0]['title','author']"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + assert_eq!( + &**result[1].as_value().unwrap().as_string().unwrap(), + "George Orwell" + ); + Ok(()) + } + + #[test] + fn handles_union_with_negative_indexes() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[-2,-1].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Pride and Prejudice" + ); + assert_eq!( + &**result[1].as_value().unwrap().as_string().unwrap(), + "The Hobbit" + ); + Ok(()) + } + + #[test] + fn handles_basic_slice() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0:3].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + assert_eq!(titles, vec!["1984", "Animal Farm", "Brave New World"]); + Ok(()) + } + + #[test] + fn handles_slice_with_step() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[0:5:2].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + assert_eq!(titles, vec!["1984", "Brave New World", "The Great Gatsby"]); + Ok(()) + } + + #[test] + fn handles_negative_slice() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[-2:].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + assert_eq!(titles, vec!["Pride and Prejudice", "The Hobbit"]); + Ok(()) + } + } + + mod complex_and_recursive_filters { + use super::*; + + #[test] + fn filters_with_multiple_conditions() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price >= 10 && @.available == true && @.title contains '1984')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "1984" + ); + Ok(()) + } + + #[test] + fn filters_with_path_expressions() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell' && @.title != '1984')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "Animal Farm" + ); + Ok(()) + } + + #[test] + fn filters_with_null_checks() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price == null || @.price < 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 5); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "Animal Farm", + "Fahrenheit 451", + "The Great Gatsby", + "Pride and Prejudice", + "Lord of the Flies", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn handles_recursive_filter_with_price_condition() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.price > 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["Brave New World", "To Kill a Mockingbird", "The Hobbit"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn handles_recursive_filter_with_logical_operators() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$..[?(@.author == 'George Orwell' || @.price > 10)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 5); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "1984", + "Animal Farm", + "Brave New World", + "To Kill a Mockingbird", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod edge_cases_and_error_handling { + use super::*; + + #[test] + fn handles_quoted_keys_with_special_characters() -> Result<(), JsonPathError> { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let map = doc.get_map("store"); + let book = map + .insert_container("book-with-dash", MapHandler::new_detached()) + .unwrap(); + book.insert("price-$10", "cheap").unwrap(); + let path = "$['store']['book-with-dash']['price-$10']"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 1); + assert_eq!( + &**result[0].as_value().unwrap().as_string().unwrap(), + "cheap" + ); + Ok(()) + } + + #[test] + fn handles_quoted_keys_with_escaped_quotes() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == 'George Orwell')].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod root_filters { + use super::*; + + #[test] + fn filters_with_root_reference() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == $.store.featured_author)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_numerical_comparison() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price > $.store.min_price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["Brave New World", "The Hobbit", "To Kill a Mockingbird"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_not_equal() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author != $.store.featured_author)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 8); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "Brave New World", + "Fahrenheit 451", + "The Great Gatsby", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_with_root_complex() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.author == $.store.featured_author && @.price <= $.store.min_price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 2); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Animal Farm"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + } + + mod existence_filters { + use super::*; + #[test] + fn filters_by_existence_of_isbn() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.isbn)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 3); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec!["1984", "Brave New World", "Fahrenheit 451"]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_by_non_existence_of_isbn() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(!@.isbn)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 7); + let mut titles: Vec<&str> = result + .iter() + .map(|v| v.as_value().unwrap().as_string().unwrap().as_str()) + .collect(); + titles.sort(); + let mut expected = vec![ + "Animal Farm", + "The Great Gatsby", + "To Kill a Mockingbird", + "The Catcher in the Rye", + "Lord of the Flies", + "Pride and Prejudice", + "The Hobbit", + ]; + expected.sort(); + assert_eq!(titles, expected); + Ok(()) + } + + #[test] + fn filters_by_existence_of_price() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(@.price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 10); + Ok(()) + } + + #[test] + fn filters_by_non_existence_of_price() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?(!@.price)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 0); + Ok(()) + } + + #[test] + fn filters_by_non_existence_of_root_path() -> Result<(), JsonPathError> { + let doc = setup_test_doc(); + let path = "$.store.books[?($.store.nonexistent)].title"; + let result = evaluate_jsonpath(&doc, path)?; + assert_eq!(result.len(), 0); + Ok(()) + } + } +} diff --git a/vendor/loro-internal/src/jsonpath/mod.rs b/vendor/loro-internal/src/jsonpath/mod.rs new file mode 100644 index 00000000..9bbcdee9 --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/mod.rs @@ -0,0 +1,10 @@ +pub mod ast; +pub mod errors; +pub mod jsonpath_impl; +pub mod subscription; + +pub use subscription::SubscribeJsonPathCallback; +pub mod parser; + +pub use ast::Query; +pub use parser::JSONPathParser; diff --git a/vendor/loro-internal/src/jsonpath/parser.rs b/vendor/loro-internal/src/jsonpath/parser.rs new file mode 100644 index 00000000..60425fe4 --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/parser.rs @@ -0,0 +1,708 @@ +//! A JSONPath parser using [pest]. +//! +//! Refer to `jsonpath.pest` and the [pest book]. +//! +//! [pest]: https://pest.rs/ +//! [pest book]: https://pest.rs/book/ + +use std::{collections::HashMap, ops::RangeInclusive}; + +use pest::{iterators::Pair, Parser}; +use pest_derive::Parser; + +use crate::jsonpath::ast::{ + ComparisonOperator, FilterExpression, LogicalOperator, Segment, Selector, +}; +use crate::jsonpath::errors::JSONPathError; +use crate::jsonpath::Query; + +#[derive(Parser)] +#[grammar = "jsonpath/jsonpath.pest"] +struct JSONPath; + +#[derive(Debug)] +pub enum ExpressionType { + Logical, + Nodes, + Value, +} + +#[derive(Debug)] +pub struct FunctionSignature { + pub param_types: Vec, + pub return_type: ExpressionType, +} + +pub fn standard_functions() -> HashMap { + let mut functions = HashMap::new(); + + functions.insert( + "count".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Nodes], + return_type: ExpressionType::Value, + }, + ); + + functions.insert( + "length".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Value], + return_type: ExpressionType::Value, + }, + ); + + functions.insert( + "match".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Value, ExpressionType::Value], + return_type: ExpressionType::Logical, + }, + ); + + functions.insert( + "search".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Value, ExpressionType::Value], + return_type: ExpressionType::Logical, + }, + ); + + functions.insert( + "value".to_owned(), + FunctionSignature { + param_types: vec![ExpressionType::Nodes], + return_type: ExpressionType::Value, + }, + ); + + functions +} + +#[derive(Debug)] +pub struct JSONPathParser { + pub index_range: RangeInclusive, + pub function_signatures: HashMap, +} + +impl Default for JSONPathParser { + fn default() -> Self { + Self::new() + } +} + +impl JSONPathParser { + pub fn new() -> Self { + JSONPathParser { + index_range: ((-2_i64).pow(53) + 1..=2_i64.pow(53) - 1), + function_signatures: standard_functions(), + } + } + + pub fn add_function( + &mut self, + name: &str, + params: Vec, + returns: ExpressionType, + ) { + self.function_signatures.insert( + name.to_owned(), + FunctionSignature { + param_types: params, + return_type: returns, + }, + ); + } + + pub fn parse(&self, query: &str) -> Result { + let segments = JSONPath::parse(Rule::jsonpath, query) + .map_err(|err| JSONPathError::syntax(err.to_string()))? + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + })?; + Ok(Query { segments }) + } + + fn parse_segment(&self, acc: Segment, segment: Pair) -> Result { + Ok(match segment.as_rule() { + Rule::child_segment => Segment::Child { + left: Box::new(acc), + selectors: self.parse_segment_inner(segment.into_inner().next().unwrap())?, + }, + Rule::descendant_segment => Segment::Recursive { + left: Box::new(acc), + selectors: self.parse_segment_inner(segment.into_inner().next().unwrap())?, + }, + Rule::name_segment | Rule::index_segment => Segment::Child { + left: Box::new(acc), + selectors: vec![self.parse_selector(segment.into_inner().next().unwrap())?], + }, + Rule::EOI => acc, + _ => unreachable!(), + }) + } + + fn parse_segment_inner(&self, segment: Pair) -> Result, JSONPathError> { + Ok(match segment.as_rule() { + Rule::bracketed_selection => { + let seg: Result, _> = segment + .into_inner() + .map(|selector| self.parse_selector(selector)) + .collect(); + seg? + } + Rule::wildcard_selector => vec![Selector::Wild {}], + Rule::member_name_shorthand => vec![Selector::Name { + // for child_segment name: + name: segment.as_str().to_owned(), + }], + _ => unreachable!(), + }) + } + + fn parse_selector(&self, selector: Pair) -> Result { + Ok(match selector.as_rule() { + Rule::double_quoted => Selector::Name { + name: unescape_string(selector.as_str()), + }, + Rule::single_quoted => Selector::Name { + name: unescape_string(&selector.as_str().replace("\\'", "'")), + }, + Rule::wildcard_selector => Selector::Wild {}, + Rule::slice_selector => self.parse_slice_selector(selector)?, + Rule::index_selector => Selector::Index { + index: self.parse_i_json_int(selector.as_str())?, + }, + Rule::filter_selector => self.parse_filter_selector(selector)?, + Rule::member_name_shorthand => Selector::Name { + // for name_segment name: + name: selector.as_str().to_owned(), + }, + _ => unreachable!(), + }) + } + + fn parse_slice_selector(&self, selector: Pair) -> Result { + let mut start: Option = None; + let mut stop: Option = None; + let mut step: Option = None; + + for i in selector.into_inner() { + match i.as_rule() { + Rule::start => start = Some(self.parse_i_json_int(i.as_str())?), + Rule::stop => stop = Some(self.parse_i_json_int(i.as_str())?), + Rule::step => step = Some(self.parse_i_json_int(i.as_str())?), + _ => unreachable!(), + } + } + + Ok(Selector::Slice { start, stop, step }) + } + + fn parse_filter_selector(&self, selector: Pair) -> Result { + Ok(Selector::Filter { + expression: Box::new( + self.parse_logical_or_expression(selector.into_inner().next().unwrap(), true)?, + ), + }) + } + + fn parse_logical_or_expression( + &self, + expr: Pair, + assert_compared: bool, + ) -> Result { + let mut it = expr.into_inner(); + let mut or_expr = self.parse_logical_and_expression(it.next().unwrap(), assert_compared)?; + if assert_compared { + self.assert_compared(&or_expr)?; + } + for and_expr in it { + let right = self.parse_logical_and_expression(and_expr, assert_compared)?; + if assert_compared { + self.assert_compared(&right)?; + } + or_expr = FilterExpression::Logical { + left: Box::new(or_expr), + operator: LogicalOperator::Or, + right: Box::new(right), + }; + } + Ok(or_expr) + } + + fn parse_logical_and_expression( + &self, + expr: Pair, + assert_compared: bool, + ) -> Result { + let mut it = expr.into_inner(); + let mut and_expr = self.parse_basic_expression(it.next().unwrap())?; + if assert_compared { + self.assert_compared(&and_expr)?; + } + for basic_expr in it { + let right = self.parse_basic_expression(basic_expr)?; + if assert_compared { + self.assert_compared(&right)?; + } + and_expr = FilterExpression::Logical { + left: Box::new(and_expr), + operator: LogicalOperator::And, + right: Box::new(right), + }; + } + Ok(and_expr) + } + + fn parse_basic_expression(&self, expr: Pair) -> Result { + match expr.as_rule() { + Rule::paren_expr => self.parse_paren_expression(expr), + Rule::comparison_expr => self.parse_comparison_expression(expr), + Rule::test_expr => self.parse_test_expression(expr), + _ => unreachable!(), + } + } + + fn parse_paren_expression(&self, expr: Pair) -> Result { + let mut it = expr.into_inner(); + let p = it.next().unwrap(); + match p.as_rule() { + Rule::logical_not_op => Ok(FilterExpression::Not { + expression: Box::new(self.parse_logical_or_expression(it.next().unwrap(), true)?), + }), + Rule::logical_or_expr => self.parse_logical_or_expression(p, true), + _ => unreachable!(), + } + } + + fn parse_comparison_expression( + &self, + expr: Pair, + ) -> Result { + let mut it = expr.into_inner(); + let left = self.parse_comparable(it.next().unwrap())?; + let operator = match it.next().unwrap().as_str() { + "==" => ComparisonOperator::Eq, + "!=" => ComparisonOperator::Ne, + "<=" => ComparisonOperator::Le, + ">=" => ComparisonOperator::Ge, + "<" => ComparisonOperator::Lt, + ">" => ComparisonOperator::Gt, + "contains" => ComparisonOperator::Contains, + "in" => ComparisonOperator::In, + _ => unreachable!(), + }; + let right = self.parse_comparable(it.next().unwrap())?; + self.assert_comparable(&left)?; + self.assert_comparable(&right)?; + Ok(FilterExpression::Comparison { + left: Box::new(left), + operator, + right: Box::new(right), + }) + } + + fn parse_comparable(&self, expr: Pair) -> Result { + Ok(match expr.as_rule() { + Rule::number => self.parse_number(expr)?, + Rule::double_quoted => FilterExpression::StringLiteral { + value: unescape_string(expr.as_str()), + }, + Rule::single_quoted => FilterExpression::StringLiteral { + value: unescape_string(&expr.as_str().replace("\\'", "'")), + }, + Rule::true_literal => FilterExpression::True_ {}, + Rule::false_literal => FilterExpression::False_ {}, + Rule::null => FilterExpression::Null {}, + Rule::array_literal => self.parse_array_literal(expr)?, + Rule::rel_singular_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RelativeQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::abs_singular_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RootQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::function_expr => self.parse_function_expression(expr)?, + _ => unreachable!(), + }) + } + + fn parse_number(&self, expr: Pair) -> Result { + let raw = expr.as_str(); + if raw == "-0" { + return Ok(FilterExpression::Int { value: 0 }); + } + let is_float = raw.contains('.') || raw.contains('e') || raw.contains('E'); + if !is_float { + if let Ok(value) = raw.parse::() { + return Ok(FilterExpression::Int { value }); + } + } + Ok(FilterExpression::Float { + value: raw + .parse::() + .map_err(|_| JSONPathError::syntax(String::from("invalid float literal")))?, + }) + } + + fn parse_array_literal(&self, expr: Pair) -> Result { + let values: Result, _> = expr + .into_inner() + .map(|p| self.parse_comparable(p)) + .collect(); + Ok(FilterExpression::Array { values: values? }) + } + + fn parse_test_expression(&self, expr: Pair) -> Result { + let mut it = expr.into_inner(); + let pair = it.next().unwrap(); + Ok(match pair.as_rule() { + Rule::logical_not_op => FilterExpression::Not { + expression: Box::new(self.parse_test_expression_inner(it.next().unwrap())?), + }, + _ => self.parse_test_expression_inner(pair)?, + }) + } + + fn parse_test_expression_inner( + &self, + expr: Pair, + ) -> Result { + Ok(match expr.as_rule() { + Rule::rel_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RelativeQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::root_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RootQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::function_expr => self.parse_function_expression(expr)?, + _ => unreachable!(), + }) + } + + fn parse_function_expression( + &self, + expr: Pair, + ) -> Result { + let mut it = expr.into_inner(); + let name = it.next().unwrap().as_str(); + let args: Result, _> = it.map(|ex| self.parse_function_argument(ex)).collect(); + Ok(FilterExpression::Function { + name: name.to_string(), + args: self.assert_well_typed(name, args?)?, + }) + } + + fn parse_function_argument(&self, expr: Pair) -> Result { + Ok(match expr.as_rule() { + Rule::number => self.parse_number(expr)?, + Rule::double_quoted => FilterExpression::StringLiteral { + value: unescape_string(expr.as_str()), + }, + Rule::single_quoted => FilterExpression::StringLiteral { + value: unescape_string(&expr.as_str().replace("\\'", "'")), + }, + Rule::true_literal => FilterExpression::True_ {}, + Rule::false_literal => FilterExpression::False_ {}, + Rule::null => FilterExpression::Null {}, + Rule::array_literal => self.parse_array_literal(expr)?, + Rule::rel_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RelativeQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::root_query => { + let segments = expr + .into_inner() + .try_fold(Segment::Root {}, |acc, segment| { + self.parse_segment(acc, segment) + }); + FilterExpression::RootQuery { + query: Box::new(Query { + segments: segments?, + }), + } + } + Rule::logical_or_expr => self.parse_logical_or_expression(expr, false)?, + Rule::function_expr => self.parse_function_expression(expr)?, + _ => unreachable!(), + }) + } + + fn parse_i_json_int(&self, value: &str) -> Result { + let i = value + .parse::() + .map_err(|_| JSONPathError::syntax(format!("index out of range `{}`", value)))?; + if !self.index_range.contains(&i) { + return Err(JSONPathError::syntax(format!( + "index out of range `{}`", + value + ))); + } + Ok(i) + } + + fn assert_comparable(&self, expr: &FilterExpression) -> Result<(), JSONPathError> { + // TODO: accept span/position for better errors + match expr { + FilterExpression::RelativeQuery { query, .. } + | FilterExpression::RootQuery { query, .. } => { + if !query.is_singular() { + Err(JSONPathError::typ(String::from( + "non-singular query is not comparable", + ))) + } else { + Ok(()) + } + } + FilterExpression::Function { name, .. } => { + if let Some(FunctionSignature { + return_type: ExpressionType::Value, + .. + }) = self.function_signatures.get(name) + { + Ok(()) + } else { + Err(JSONPathError::typ(format!( + "result of {}() is not comparable", + name + ))) + } + } + _ => Ok(()), + } + } + + fn assert_compared(&self, expr: &FilterExpression) -> Result<(), JSONPathError> { + match expr { + FilterExpression::Function { name, .. } => { + if let Some(FunctionSignature { + return_type: ExpressionType::Value, + .. + }) = self.function_signatures.get(name) + { + Err(JSONPathError::typ(format!( + "result of {}() must be compared", + name + ))) + } else { + Ok(()) + } + } + _ => Ok(()), + } + } + + fn assert_well_typed( + &self, + func_name: &str, + args: Vec, + ) -> Result, JSONPathError> { + // TODO: accept span/position for better errors + let signature = self + .function_signatures + .get(func_name) + .ok_or_else(|| JSONPathError::name(format!("unknown function `{}`", func_name)))?; + + // correct number of arguments? + if args.len() != signature.param_types.len() { + return Err(JSONPathError::typ(format!( + "{}() takes {} argument{} but {} were given", + func_name, + signature.param_types.len(), + if signature.param_types.len() > 1 { + "s" + } else { + "" + }, + args.len() + ))); + } + + // correct argument types? + for (idx, typ) in signature.param_types.iter().enumerate() { + let arg = &args[idx]; + match typ { + ExpressionType::Value => { + if !self.is_value_type(arg) { + return Err(JSONPathError::typ(format!( + "argument {} of {}() must be of a 'Value' type", + idx + 1, + func_name + ))); + } + } + ExpressionType::Logical => { + if !matches!( + arg, + FilterExpression::RelativeQuery { .. } + | FilterExpression::RootQuery { .. } + | FilterExpression::Logical { .. } + | FilterExpression::Comparison { .. }, + ) { + return Err(JSONPathError::typ(format!( + "argument {} of {}() must be of a 'Logical' type", + idx + 1, + func_name + ))); + } + } + ExpressionType::Nodes => { + if !self.is_nodes_type(arg) { + return Err(JSONPathError::typ(format!( + "argument {} of {}() must be of a 'Nodes' type", + idx + 1, + func_name + ))); + } + } + } + } + + Ok(args) + } + + fn is_value_type(&self, expr: &FilterExpression) -> bool { + // literals are values + if expr.is_literal() { + return true; + } + + match expr { + FilterExpression::RelativeQuery { query, .. } + | FilterExpression::RootQuery { query, .. } => { + // singular queries will be coerced to a value + query.is_singular() + } + FilterExpression::Function { name, .. } => { + // some functions return a value + matches!( + self.function_signatures.get(name), + Some(FunctionSignature { + return_type: ExpressionType::Value, + .. + }) + ) + } + FilterExpression::Array { .. } => true, + _ => false, + } + } + + fn is_nodes_type(&self, expr: &FilterExpression) -> bool { + match expr { + FilterExpression::RelativeQuery { .. } | FilterExpression::RootQuery { .. } => true, + FilterExpression::Function { name, .. } => { + matches!( + self.function_signatures.get(name), + Some(FunctionSignature { + return_type: ExpressionType::Nodes, + .. + }) + ) + } + _ => false, + } + } +} + +fn unescape_string(value: &str) -> String { + let chars = value.chars().collect::>(); + let length = chars.len(); + let mut rv = String::new(); + let mut index: usize = 0; + + while index < length { + match chars[index] { + '\\' => { + index += 1; + match chars[index] { + '"' => rv.push('"'), + '\\' => rv.push('\\'), + '/' => rv.push('/'), + 'b' => rv.push('\x08'), + 'f' => rv.push('\x0C'), + 'n' => rv.push('\n'), + 'r' => rv.push('\r'), + 't' => rv.push('\t'), + 'u' => { + index += 1; + let digits = chars + .get(index..index + 4) + .unwrap() + .iter() + .collect::(); + let mut codepoint = u32::from_str_radix(&digits, 16).unwrap(); + + if index + 5 < length && chars[index + 4] == '\\' && chars[index + 5] == 'u' + { + let digits = &chars + .get(index + 6..index + 10) + .unwrap() + .iter() + .collect::(); + let low_surrogate = u32::from_str_radix(digits, 16).unwrap(); + codepoint = + 0x10000 + (((codepoint & 0x03FF) << 10) | (low_surrogate & 0x03FF)); + index += 6; + } + + let unescaped = char::from_u32(codepoint).unwrap(); + rv.push(unescaped); + index += 3; + } + _ => unreachable!(), + } + } + c => { + rv.push(c); + } + } + index += 1; + } + + rv +} diff --git a/vendor/loro-internal/src/jsonpath/subscription.rs b/vendor/loro-internal/src/jsonpath/subscription.rs new file mode 100644 index 00000000..599bc18a --- /dev/null +++ b/vendor/loro-internal/src/jsonpath/subscription.rs @@ -0,0 +1,551 @@ +//! # JSONPath Subscription Module +//! +//! This module provides a mechanism to subscribe to changes that may affect a JSONPath query +//! result, without re-evaluating the query on every change. +//! +//! ## Design Goals +//! +//! - **Conservative matching**: The matcher may produce false positives (triggering when +//! the query result hasn't actually changed) but must never produce false negatives +//! (missing a notification when the result did change). +//! - **Lightweight notifications**: Callbacks receive no payload; applications can +//! debounce/throttle and evaluate the JSONPath themselves. +//! - **Efficient matching**: Uses an NFA (non-deterministic finite automaton) approach +//! to match event paths against compiled JSONPath queries in O(path_len × steps) time. +//! +//! ## Algorithm Overview +//! +//! 1. **Compilation**: JSONPath is parsed and converted into a sequence of lightweight +//! `MatchSelector`s (no AST storage needed for matching). +//! +//! 2. **NFA Simulation**: When an event arrives, we simulate an NFA where: +//! - States are positions in the step sequence (0..steps.len()) +//! - Transitions occur when a selector matches the current path element +//! - Recursive steps allow staying at the same state while consuming input +//! - Acceptance occurs when any state reaches or exceeds `steps.len()` + +use std::sync::Arc; + +use loro_common::TreeID; +use smallvec::SmallVec; + +use crate::{ + event::{Diff, Index}, + jsonpath::{ + ast::{Query, Segment, Selector}, + JSONPathParser, + }, + utils::subscription::Subscription, + LoroDoc, LoroError, LoroResult, +}; + +/// Callback used by `subscribe_jsonpath`. +/// +/// Note: the callback does **not** carry the query result. It is intended as a +/// lightweight notification so applications can debounce/throttle and evaluate +/// JSONPath themselves if needed. +pub type SubscribeJsonPathCallback = Arc; + +/// Represents a single path segment used for matching against events. +/// +/// Path elements are derived from event paths and represent the location of a change +/// in the document tree. They form the "input alphabet" for our NFA simulation. +#[derive(Debug, Clone, PartialEq, Eq)] +enum PathElem { + /// A map key (e.g., "title" in `$.books[0].title`) + Key(Arc), + /// A sequence (list/array) index. + /// - `Some(i)` means a specific index changed + /// - `None` means "some index changed" (used for list/tree child mutations + /// where we don't know which specific indices are affected) + Seq(Option), + /// A tree node identifier (for Loro's tree CRDT) + Node(TreeID), +} + +impl From<&Index> for PathElem { + fn from(value: &Index) -> Self { + match value { + Index::Key(k) => PathElem::Key(k.as_ref().into()), + Index::Seq(i) => PathElem::Seq(Some(*i)), + Index::Node(n) => PathElem::Node(*n), + } + } +} + +// ============================================================================= +// Lightweight Matching Types +// ============================================================================= +// +// Instead of storing the full Selector AST (which includes heavyweight +// FilterExpression trees), we convert selectors to a simple matching-only enum. +// This reduces memory usage and code complexity. + +/// Simplified selector for matching only - no AST storage needed. +/// +/// All complex selectors (Slice, Wild, Filter, negative Index) become `Wildcard` +/// since they can match any element during conservative path matching. +#[derive(Debug, Clone)] +enum MatchSelector { + /// Exact key match (from `Selector::Name`) + Name(Arc), + /// Exact non-negative index match (from `Selector::Index` where index >= 0) + Index(usize), + /// Matches anything (from Wild, Slice, Filter, or negative Index) + Wildcard, +} + +/// A single step in the linearized JSONPath query. +#[derive(Debug, Clone)] +struct Step { + /// If true, this step uses recursive descent (`..`) and can match at any depth. + recursive: bool, + /// Simplified selectors for matching. + selectors: SmallVec<[MatchSelector; 2]>, +} + +/// A compiled matcher that conservatively checks whether a change at some path +/// can affect the JSONPath query. +#[derive(Debug, Clone)] +struct JsonPathMatcher { + /// Linearized sequence of matching steps + steps: Vec, +} + +impl JsonPathMatcher { + /// Compile a JSONPath query into a matcher. + fn new(query: &Query) -> Self { + let mut steps = Vec::new(); + build_steps(&query.segments, &mut steps); + JsonPathMatcher { steps } + } + + /// Returns true if the provided path (from root) could affect the query result. + fn may_match(&self, path: &[PathElem]) -> bool { + if self.steps.is_empty() { + return true; + } + let positions = self.positions_after(path); + positions.iter().any(|&p| p >= self.steps.len()) + } + + /// Check if any position was reached by passing through a Wildcard step. + /// Wildcard steps come from filter/wild/slice selectors - any key change + /// under them could affect the query result. + #[inline] + fn passed_through_wildcard(&self, positions: &[usize]) -> bool { + positions.iter().any(|&pos| { + pos > 0 + && self.steps[pos - 1] + .selectors + .iter() + .any(|s| matches!(s, MatchSelector::Wildcard)) + }) + } + + /// Simulate the NFA on a path and return all reachable positions after consuming it. + /// + /// - Position `== steps.len()`: Query fully matched + /// - Position `< steps.len()`: Partial match + /// - Empty result: No match possible + fn positions_after(&self, path: &[PathElem]) -> SmallVec<[usize; 8]> { + let mut positions = SmallVec::<[usize; 8]>::new(); + positions.push(0); + + for elem in path { + let mut next = SmallVec::<[usize; 8]>::new(); + + for &pos in positions.iter() { + if pos >= self.steps.len() { + next.push(pos); + continue; + } + + let step = &self.steps[pos]; + + // Recursive descent: can stay at same position + if step.recursive { + next.push(pos); + } + + // If selector matches, advance to next step + if selector_matches(&step.selectors, elem) { + next.push(pos + 1); + } + } + + dedup_positions(&mut next); + if next.is_empty() { + return next; + } + positions = next; + } + + positions + } +} + +/// Check if any selector matches the given path element. +fn selector_matches(selectors: &[MatchSelector], elem: &PathElem) -> bool { + selectors.iter().any(|sel| match sel { + MatchSelector::Name(name) => matches!(elem, PathElem::Key(k) if k == name), + MatchSelector::Index(idx) => match elem { + PathElem::Seq(Some(i)) => *i == *idx, + PathElem::Seq(None) => true, // Unknown index - conservative match + _ => false, + }, + MatchSelector::Wildcard => true, + }) +} + +/// Convert AST Selector to lightweight MatchSelector. +fn to_match_selector(sel: &Selector) -> MatchSelector { + match sel { + Selector::Name { name } => MatchSelector::Name(name.as_str().into()), + Selector::Index { index } if *index >= 0 => MatchSelector::Index(*index as usize), + // Negative index, Slice, Wild, Filter all become Wildcard + _ => MatchSelector::Wildcard, + } +} + +/// Linearize the JSONPath AST into a sequence of matching steps. +fn build_steps(segment: &Segment, steps: &mut Vec) { + match segment { + Segment::Root {} => {} + Segment::Child { left, selectors } => { + build_steps(left, steps); + steps.push(Step { + recursive: false, + selectors: selectors.iter().map(to_match_selector).collect(), + }); + } + Segment::Recursive { left, selectors } => { + build_steps(left, steps); + steps.push(Step { + recursive: true, + selectors: selectors.iter().map(to_match_selector).collect(), + }); + } + } +} + +/// Deduplicate positions in-place. +#[inline] +fn dedup_positions(v: &mut SmallVec<[usize; 8]>) { + v.sort_unstable(); + v.dedup(); +} + +// ============================================================================= +// Public API +// ============================================================================= + +impl LoroDoc { + /// Subscribe to updates that may affect the given JSONPath query. + /// + /// ## Behavior + /// + /// - **Conservative matching**: The callback may fire for changes that don't + /// actually affect the query result (false positives), but it will never + /// miss a change that does affect it (no false negatives). + /// - **Lightweight notifications**: The callback receives no payload. Applications + /// should debounce/throttle and re-evaluate the JSONPath if needed. + /// + /// ## Returns + /// + /// A `Subscription` handle. Drop it to unsubscribe. + #[cfg(feature = "jsonpath")] + pub fn subscribe_jsonpath( + &self, + jsonpath: &str, + callback: SubscribeJsonPathCallback, + ) -> LoroResult { + let query = JSONPathParser::new() + .parse(jsonpath) + .map_err(|e| LoroError::ArgErr(e.to_string().into_boxed_str()))?; + + let matcher = Arc::new(JsonPathMatcher::new(&query)); + + let sub = self.subscribe_root(Arc::new(move |event| { + if event.events.is_empty() { + return; + } + + let matcher = &matcher; + let mut fired = false; + + for container_diff in event.events.iter() { + if fired { + break; + } + + // Convert the container's path to PathElem representation + let base_path: SmallVec<[PathElem; 8]> = container_diff + .path + .iter() + .map(|(_, idx)| PathElem::from(idx)) + .collect(); + + // Check 1: Does the container path itself match the query? + if matcher.may_match(&base_path) { + fired = true; + break; + } + + // Check 2: Map-specific handling - check each changed key + if let Diff::Map(map) = &container_diff.diff { + let base_positions = matcher.positions_after(&base_path); + if base_positions.is_empty() { + continue; + } + + // If we passed through a Wildcard (filter/wild/slice), any key + // change could affect the result - be conservative. + let past_wildcard = matcher.passed_through_wildcard(&base_positions); + + for key in map.updated.keys() { + let mut extended: SmallVec<[PathElem; 8]> = base_path.clone(); + extended.push(PathElem::Key(key.as_ref().into())); + + let extended_positions = matcher.positions_after(&extended); + + // Trigger if key path could match OR we're inside a wildcard selection + if !extended_positions.is_empty() || past_wildcard { + fired = true; + break; + } + } + + if fired { + break; + } + } + + // Check 3: List/Tree/Counter child mutations + // These container types can have child changes at unknown indices + let has_child_changes = matches!( + &container_diff.diff, + Diff::List(_) | Diff::Tree(_) | Diff::Unknown + ); + #[cfg(feature = "counter")] + let has_child_changes = + has_child_changes || matches!(&container_diff.diff, Diff::Counter(_)); + + if has_child_changes { + let mut extended: SmallVec<[PathElem; 8]> = base_path.clone(); + extended.push(PathElem::Seq(None)); // "some child changed" + if !matcher.positions_after(&extended).is_empty() { + fired = true; + } + } + } + + if fired { + (callback)(); + } + })); + + Ok(sub) + } +} + +// ============================================================================= +// Tests +// ============================================================================= + +#[cfg(test)] +mod tests { + use super::*; + use crate::{LoroDoc, MapHandler}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn make_book( + doc: &LoroDoc, + idx: usize, + title: &str, + available: bool, + price: i64, + ) -> MapHandler { + let books = doc.get_list("books"); + let book = books + .insert_container(idx, MapHandler::new_detached()) + .unwrap(); + book.insert("title", title).unwrap(); + book.insert("available", available).unwrap(); + book.insert("price", price).unwrap(); + book + } + + #[test] + fn jsonpath_subscribe_triggers_on_specific_key() { + let doc = LoroDoc::new_auto_commit(); + let first_book = make_book(&doc, 0, "Old", true, 10); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[0].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + first_book.insert("title", "New").unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_wildcard_on_list() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + let second_book = make_book(&doc, 1, "B", true, 20); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[*].price", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + second_book.insert("price", 25).unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_negative_index() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + make_book(&doc, 1, "B", true, 20); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[-1].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + let books = doc.get_list("books"); + let last = books.get_child_handler(1).unwrap(); + last.as_map().unwrap().insert("title", "B updated").unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_slice_range() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + let second_book = make_book(&doc, 1, "B", true, 20); + make_book(&doc, 2, "C", true, 30); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[0:2].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + second_book.insert("title", "B updated").unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_recursive() { + let doc = LoroDoc::new_auto_commit(); + let store = doc.get_map("store"); + let nested = store + .insert_container("inventory", MapHandler::new_detached()) + .unwrap(); + nested.insert("total", 3).unwrap(); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$..total", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + nested.insert("total", 4).unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_filter_treated_as_wildcard() { + let doc = LoroDoc::new_auto_commit(); + make_book(&doc, 0, "A", true, 10); + let second_book = make_book(&doc, 1, "B", false, 20); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[?@.available].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + second_book.insert("available", true).unwrap(); + doc.commit_then_renew(); + + assert!(hit.load(Ordering::SeqCst) >= 1); + } + + #[test] + fn jsonpath_subscribe_triggers_once_per_commit() { + let doc = LoroDoc::new_auto_commit(); + let book = make_book(&doc, 0, "A", true, 10); + doc.commit_then_renew(); + + let hit = Arc::new(AtomicUsize::new(0)); + let hit_ref = hit.clone(); + let _sub = doc + .subscribe_jsonpath( + "$.books[0].title", + Arc::new(move || { + hit_ref.fetch_add(1, Ordering::SeqCst); + }), + ) + .unwrap(); + + book.insert("title", "X").unwrap(); + book.insert("title", "Y").unwrap(); + doc.commit_then_renew(); + + assert_eq!(hit.load(Ordering::SeqCst), 1); + } +} diff --git a/vendor/loro-internal/src/kv_store.rs b/vendor/loro-internal/src/kv_store.rs new file mode 100644 index 00000000..9cd773a2 --- /dev/null +++ b/vendor/loro-internal/src/kv_store.rs @@ -0,0 +1,249 @@ +use crate::sync::Mutex; +use bytes::Bytes; +pub use loro_kv_store::compress::CompressionType; +pub use loro_kv_store::MemKvStore; +use std::sync::Arc; +use std::{collections::BTreeMap, ops::Bound}; + +pub trait KvStore: std::fmt::Debug + Send + Sync { + fn get(&self, key: &[u8]) -> Option; + fn set(&mut self, key: &[u8], value: Bytes); + fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool; + fn remove(&mut self, key: &[u8]) -> Option; + fn contains_key(&self, key: &[u8]) -> bool; + fn scan( + &self, + start: Bound<&[u8]>, + end: Bound<&[u8]>, + ) -> Box + '_>; + fn len(&self) -> usize; + fn is_empty(&self) -> bool; + fn size(&self) -> usize; + fn export_all(&mut self) -> Bytes; + fn import_all(&mut self, bytes: Bytes) -> Result<(), String>; + fn clone_store(&self) -> Arc>; +} + +fn get_common_prefix_len_and_strip<'a, T: AsRef<[u8]> + ?Sized>( + this: &'a T, + last: &T, +) -> (u8, &'a [u8]) { + let mut common_prefix_len = 0; + for (i, (a, b)) in this.as_ref().iter().zip(last.as_ref().iter()).enumerate() { + if a != b || i == 255 { + common_prefix_len = i; + break; + } + } + + let suffix = &this.as_ref()[common_prefix_len..]; + (common_prefix_len as u8, suffix) +} + +impl KvStore for MemKvStore { + fn get(&self, key: &[u8]) -> Option { + self.get(key) + } + + fn set(&mut self, key: &[u8], value: Bytes) { + self.set(key, value) + } + + fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool { + self.compare_and_swap(key, old, new) + } + + fn remove(&mut self, key: &[u8]) -> Option { + let ans = self.get(key); + self.remove(key); + ans + } + + fn contains_key(&self, key: &[u8]) -> bool { + self.contains_key(key) + } + + fn scan( + &self, + start: Bound<&[u8]>, + end: Bound<&[u8]>, + ) -> Box + '_> { + self.scan(start, end) + } + + fn len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn size(&self) -> usize { + self.size() + } + + fn export_all(&mut self) -> Bytes { + self.export_all() + } + + fn import_all(&mut self, bytes: Bytes) -> Result<(), String> { + self.import_all(bytes) + } + + fn clone_store(&self) -> Arc> { + Arc::new(Mutex::new(self.clone())) + } +} + +mod default_binary_format { + //! Default binary format for the key-value store. + //! + //! It will compress the prefix of the keys that are common with the previous key. + + use bytes::Bytes; + + use super::get_common_prefix_len_and_strip; + + pub fn export_by_scan(store: &impl super::KvStore) -> bytes::Bytes { + let mut buf = Vec::new(); + let mut last_key: Option = None; + for (k, v) in store.scan(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded) { + { + // Write the key + match last_key.take() { + None => { + leb128::write::unsigned(&mut buf, k.len() as u64).unwrap(); + buf.extend_from_slice(&k); + } + Some(last) => { + let (common, suffix) = get_common_prefix_len_and_strip(&k, &last); + buf.push(common); + leb128::write::unsigned(&mut buf, suffix.len() as u64).unwrap(); + buf.extend_from_slice(suffix); + } + }; + + last_key = Some(k); + } + + // Write the value + leb128::write::unsigned(&mut buf, v.len() as u64).unwrap(); + buf.extend_from_slice(&v); + } + + buf.into() + } + + pub fn import(store: &mut impl super::KvStore, bytes: bytes::Bytes) -> Result<(), String> { + let mut bytes: &[u8] = &bytes; + let mut last_key = Vec::new(); + + while !bytes.is_empty() { + // Read the key + let mut key = Vec::new(); + if last_key.is_empty() { + let key_len = leb128::read::unsigned(&mut bytes).map_err(|e| e.to_string())?; + let key_len = key_len as usize; + key.extend_from_slice(&bytes[..key_len]); + bytes = &bytes[key_len..]; + } else { + let common_prefix_len = bytes[0] as usize; + bytes = &bytes[1..]; + key.extend_from_slice(&last_key[..common_prefix_len]); + let suffix_len = leb128::read::unsigned(&mut bytes).map_err(|e| e.to_string())?; + let suffix_len = suffix_len as usize; + key.extend_from_slice(&bytes[..suffix_len]); + bytes = &bytes[suffix_len..]; + } + + // Read the value + let value_len = leb128::read::unsigned(&mut bytes).map_err(|e| e.to_string())?; + let value_len = value_len as usize; + let value = Bytes::copy_from_slice(&bytes[..value_len]); + bytes = &bytes[value_len..]; + + // Store the key-value pair + store.set(&key, value); + + last_key = key; + } + + Ok(()) + } +} + +impl KvStore for BTreeMap { + fn get(&self, key: &[u8]) -> Option { + self.get(key).cloned() + } + + fn set(&mut self, key: &[u8], value: Bytes) { + self.insert(Bytes::copy_from_slice(key), value); + } + + fn compare_and_swap(&mut self, key: &[u8], old: Option, new: Bytes) -> bool { + let key = Bytes::copy_from_slice(key); + match self.get_mut(&key) { + Some(v) => { + if old.as_ref() == Some(v) { + self.insert(key, new); + true + } else { + false + } + } + None => { + if old.is_none() { + self.insert(key, new); + true + } else { + false + } + } + } + } + + fn remove(&mut self, key: &[u8]) -> Option { + self.remove(key) + } + + fn contains_key(&self, key: &[u8]) -> bool { + self.contains_key(key) + } + + fn scan( + &self, + start: Bound<&[u8]>, + end: Bound<&[u8]>, + ) -> Box + '_> { + Box::new( + self.range::<[u8], _>((start, end)) + .map(|(k, v)| (k.clone(), v.clone())), + ) + } + + fn len(&self) -> usize { + self.len() + } + + fn is_empty(&self) -> bool { + self.is_empty() + } + + fn size(&self) -> usize { + self.iter().fold(0, |acc, (k, v)| acc + k.len() + v.len()) + } + + fn export_all(&mut self) -> Bytes { + default_binary_format::export_by_scan(self) + } + + fn import_all(&mut self, bytes: Bytes) -> Result<(), String> { + default_binary_format::import(self, bytes) + } + + fn clone_store(&self) -> Arc> { + Arc::new(Mutex::new(self.clone())) + } +} diff --git a/vendor/loro-internal/src/lib.rs b/vendor/loro-internal/src/lib.rs new file mode 100644 index 00000000..256b7b96 --- /dev/null +++ b/vendor/loro-internal/src/lib.rs @@ -0,0 +1,195 @@ +//! loro-internal is a CRDT framework. +//! +//! +//! +//! +#![deny(clippy::undocumented_unsafe_blocks)] +#![allow(unused_assignments)] +#![allow(clippy::uninlined_format_args)] +#![warn(rustdoc::broken_intra_doc_links)] +#![warn(missing_debug_implementations)] + +pub mod arena; +pub mod diff; +pub mod diff_calc; +pub mod handler; +pub mod sync; +use crate::sync::{AtomicBool, AtomicUsize}; +use std::sync::Arc; +mod change_meta; +pub(crate) mod lock; +use arena::SharedArena; +use configure::Configure; +use diff_calc::DiffCalculator; +use lock::LoroMutex; + +pub use change_meta::ChangeMeta; +pub use event::{ContainerDiff, DiffEvent, DocDiff, ListDiff, ListDiffInsertItem, ListDiffItem}; +pub use handler::{ + BasicHandler, HandlerTrait, ListHandler, MapHandler, MovableListHandler, TextHandler, + TreeHandler, UnknownHandler, +}; +pub use loro_common; +pub use oplog::OpLog; +use pre_commit::{ + FirstCommitFromPeerCallback, FirstCommitFromPeerPayload, PreCommitCallback, + PreCommitCallbackPayload, +}; +pub use rustc_hash::FxHashMap; +pub use state::DocState; +pub use state::{TreeNode, TreeNodeWithChildren, TreeParentId}; +use subscription::{LocalUpdateCallback, Observer, PeerIdUpdateCallback}; +use txn::Transaction; +pub use undo::UndoManager; +pub use utils::subscription::SubscriberSetWithQueue; +pub use utils::subscription::Subscription; +pub mod allocation; +pub mod awareness; +pub mod change; +pub mod configure; +pub mod container; +pub mod cursor; +pub mod dag; +pub mod encoding; +pub(crate) mod fork; +pub mod id; +#[cfg(feature = "jsonpath")] +pub mod jsonpath; +pub mod kv_store; +pub mod loro; +pub mod op; +pub mod oplog; +pub mod subscription; +pub mod txn; +pub mod version; + +mod error; +#[cfg(feature = "test_utils")] +pub mod fuzz; +mod parent; +pub mod pre_commit; +mod span; +#[cfg(test)] +pub mod tests; +mod utils; +pub use utils::string_slice::StringSlice; + +pub mod delta; +pub use loro_delta; +pub mod event; + +pub mod estimated_size; +pub(crate) mod history_cache; +pub(crate) mod macros; +pub(crate) mod state; +pub mod undo; +pub(crate) mod value; + +// TODO: rename as Key? +pub(crate) use loro_common::InternalString; + +pub use container::ContainerType; +pub use encoding::json_schema::json; +pub use fractional_index::FractionalIndex; +pub use loro_common::{loro_value, to_value}; +pub use loro_common::{ + Counter, CounterSpan, IdLp, IdSpan, IdSpanVector, Lamport, LoroEncodeError, LoroError, + LoroResult, LoroTreeError, PeerID, TreeID, ID, +}; +pub use loro_common::{LoroBinaryValue, LoroListValue, LoroMapValue, LoroStringValue}; +#[cfg(feature = "wasm")] +pub use value::wasm; +pub use value::{ApplyDiff, LoroValue, ToJson}; +pub use version::VersionVector; + +/// [`LoroDoc`] serves as the library's primary entry point. +/// It's constituted by an [OpLog] and an [DocState]. +/// +/// - [OpLog] encompasses all operations, signifying the document history. +/// - [DocState] signifies the current document state. +/// +/// They will share a [super::arena::SharedArena] +/// +/// # Detached Mode +/// +/// This mode enables separate usage of [OpLog] and [DocState]. +/// It facilitates temporal navigation. [DocState] can be reverted to +/// any version contained within the [OpLog]. +/// +/// `LoroDoc::detach()` separates [DocState] from [OpLog]. In this mode, +/// updates to [OpLog] won't affect [DocState], while updates to [DocState] +/// will continue to affect [OpLog]. +#[derive(Debug, Clone)] +#[repr(transparent)] +pub struct LoroDoc { + inner: Arc, +} + +impl LoroDoc { + pub(crate) fn from_inner(inner: Arc) -> Self { + Self { inner } + } +} + +impl std::ops::Deref for LoroDoc { + type Target = LoroDocInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +pub struct LoroDocInner { + oplog: Arc>, + state: Arc>, + arena: SharedArena, + config: Configure, + visible_op_count: Arc, + observer: Arc, + diff_calculator: Arc>, + /// §act-five P1.B (flowstate vendor patch): a RETAINED diff calculator for the + /// IMPORT path. Upstream imports build a FRESH `DiffCalculator::new(false)` per + /// import (loro.rs), so each import rebuilds every affected container's tracker + /// from the LCA — the fixed ~O(history-to-LCA) cost that dominates a receiving + /// peer's runtime (154 MB/import on the impact doc, all reallocated each time). + /// Reusing a persistent calculator advances the trackers incrementally: a + /// sequential import (the common case) skips the rebuild entirely. Bounded + /// memory (one O(doc) calculator, not the 33 GB/run churn of transient ones) + /// and correctness-safe (persist mode uses Loro's general `Checkout` apply, + /// valid for any version transition). Separate from `diff_calculator` (which + /// checkout uses) so the two never contend; freed on `free_diff_calculator`. + import_diff_calculator: Arc>, + /// When dropping the doc, the txn will be committed + /// + /// # Internal Notes + /// + /// Txn can be accessed by different threads. But for certain methods we need to lock the txn and ensure it's empty: + /// + /// - `import` + /// - `export` + /// - `checkout` + /// - `checkout_to_latest` + /// - ... + /// + /// We need to lock txn and keep it None because otherwise the DocState may change due to a parallel edit on a new Txn, + /// which may break the invariants of `import`, `export` and `checkout`. + txn: Arc>>, + auto_commit: AtomicBool, + detached: AtomicBool, + local_update_subs: SubscriberSetWithQueue<(), LocalUpdateCallback, Vec>, + peer_id_change_subs: SubscriberSetWithQueue<(), PeerIdUpdateCallback, ID>, + first_commit_from_peer_subs: + SubscriberSetWithQueue<(), FirstCommitFromPeerCallback, FirstCommitFromPeerPayload>, + pre_commit_subs: SubscriberSetWithQueue<(), PreCommitCallback, PreCommitCallbackPayload>, +} + +/// The version of the loro crate +pub const LORO_VERSION: &str = include_str!("../VERSION"); + +impl Drop for LoroDoc { + fn drop(&mut self) { + if Arc::strong_count(&self.inner) == 1 { + let _ = self.implicit_commit_then_stop(); + } + } +} diff --git a/vendor/loro-internal/src/lock.rs b/vendor/loro-internal/src/lock.rs new file mode 100644 index 00000000..dc88929d --- /dev/null +++ b/vendor/loro-internal/src/lock.rs @@ -0,0 +1,383 @@ +//! Lock-order-checked mutexes. +//! +//! This module provides a small utility to prevent deadlocks by enforcing a +//! strict, per-thread lock acquisition order across a set of related locks. +//! +//! Core ideas: +//! - Locks are created from a [`LoroLockGroup`]. Locks in the same group share +//! a per-thread stack that tracks the last acquired lock kind. +//! - Each lock has an associated [`LockKind`]. A thread may only acquire locks +//! in strictly increasing kind order (e.g. `Txn` → `OpLog` → `DocState` → `DiffCalculator`). +//! - Locks must be released in the reverse order they were acquired (LIFO). +//! - Violations are detected and reported with helpful panics that include the +//! callsite (via `#[track_caller]`) and a backtrace on release-order errors. +//! +//! The actual locking is backed by [`crate::sync::Mutex`], which resolves to +//! `std::sync::Mutex` in normal builds and `loom::sync::Mutex` under loom. This +//! keeps the code testable with loom while maintaining the same API. +use crate::sync::ThreadLocal; +use crate::sync::{Mutex, MutexGuard}; +use std::backtrace::Backtrace; +use std::fmt::{Debug, Display}; +use std::ops::{Deref, DerefMut}; +use std::panic::Location; +use std::sync::Arc; + +/// A mutex that verifies lock acquisition and release order against a group-wide +/// strict ordering by [`LockKind`]. +/// +/// Create instances via [`LoroLockGroup::new_lock`]. Calling [`LoroMutex::lock`] +/// will panic if the current thread has already acquired a lock with a kind that +/// is greater than or equal to this lock’s kind, or if the underlying mutex was +/// poisoned by an earlier panic. Release order is also checked; dropping the +/// guard out of LIFO order results in a panic. +/// +/// This type wraps [`crate::sync::Mutex`], so it remains compatible with loom-based +/// concurrency testing. +#[derive(Debug)] +pub struct LoroMutex { + lock: Mutex, + kind: u8, + currently_locked_in_this_thread: Arc>>, +} + +/// Internal per-thread lock information used for diagnostics and order checks. +#[derive(Debug, Copy, Clone, Default)] +struct LockInfo { + kind: u8, + caller_location: Option<&'static Location<'static>>, +} + +impl Display for LockInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.caller_location { + Some(location) => write!( + f, + "LockInfo(kind: {}, location: {}:{}:{})", + self.kind, + location.file(), + location.line(), + location.column() + ), + None => write!(f, "LockInfo(kind: {}, location: None)", self.kind), + } + } +} + +#[derive(Debug)] +/// A group that defines a shared locking order domain. +/// +/// All [`LoroMutex`] created from the same group participate in a single, +/// per-thread lock-order stack. Locks from different groups are independent and +/// do not affect each other’s ordering checks. +/// +/// Use [`LoroLockGroup::new_lock`] to create locks of specific [`LockKind`]. +pub struct LoroLockGroup { + g: Arc>>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// Logical lock kinds that define the allowed acquisition order. +/// +/// Kinds with smaller numeric values must be acquired before larger ones, and +/// they must be released in reverse order. The specific variants reflect the +/// high-level components in the system; extend this enum carefully to preserve +/// a consistent global ordering. +pub enum LockKind { + None = 0, + Txn = 1, + OpLog = 2, + DocState = 3, + DiffCalculator = 4, +} + +impl LoroLockGroup { + /// Create a new lock group. + /// + /// Cloning the returned group is cheap. All locks created from clones of + /// the same group still share the same ordering domain. + pub fn new() -> Self { + let g = Arc::new(ThreadLocal::new()); + LoroLockGroup { g } + } + + /// Create a new lock associated with this group and [`LockKind`]. + /// + /// The created lock participates in this group’s order checks. + pub fn new_lock(&self, value: T, kind: LockKind) -> LoroMutex { + LoroMutex { + lock: Mutex::new(value), + currently_locked_in_this_thread: self.g.clone(), + kind: kind as u8, + } + } +} + +impl Default for LoroLockGroup { + fn default() -> Self { + Self::new() + } +} + +impl LoroMutex { + #[track_caller] + /// Acquire the lock, enforcing strict increasing [`LockKind`] order. + /// + /// Returns a guard that unlocks when dropped and verifies that it is being + /// released in the reverse acquisition order (LIFO). The callsite is + /// recorded to improve panic diagnostics. + /// + /// Panics: + /// - If the underlying mutex was poisoned by an earlier panic. + /// - If the current thread already holds a lock with kind `>= self.kind`. + /// - If the guard is later dropped out of acquisition order. + pub fn lock(&self) -> LoroMutexGuard<'_, T> { + let caller = Location::caller(); + let v = self.currently_locked_in_this_thread.get_or_default(); + let last = *v.lock(); + let this = LockInfo { + kind: self.kind, + caller_location: Some(caller), + }; + if last.kind >= self.kind { + panic!( + "Locking order violation. Current lock: {}, New lock: {}", + last, this + ); + } + + let guard = self.lock.lock_with_kind("LoroMutex"); + *v.lock() = this; + LoroMutexGuard { + guard, + _inner: LoroMutexGuardInner { + inner: self, + this, + last, + }, + } + } + + /// Returns whether the mutex appears locked at this instant. + /// + /// This is implemented via `try_lock().is_err()` and is intended only for + /// diagnostics. It is race-prone and should not be used to implement logic + /// that depends on the lock state. + pub fn is_locked(&self) -> bool { + self.lock.is_locked() + } + + /// Returns whether acquiring this lock would satisfy the current thread's + /// lock-order constraints. + /// + /// This only checks the order tracker for the current thread. It does not + /// guarantee that acquiring the underlying mutex will be non-blocking. + pub(crate) fn can_lock_in_this_thread(&self) -> bool { + let v = self.currently_locked_in_this_thread.get_or_default(); + let last = *v.lock(); + last.kind < self.kind + } +} + +/// Guard returned by [`LoroMutex::lock`]. +/// +/// Dereferences to the protected data and enforces release-order checks on drop. +/// In most cases, you should keep using this guard type so order tracking remains +/// intact for the duration of the critical section. +pub struct LoroMutexGuard<'a, T> { + guard: MutexGuard<'a, T>, + _inner: LoroMutexGuardInner<'a, T>, +} + +/// RAII helper that updates the per-thread lock info on drop. +/// +/// This is an implementation detail of [`LoroMutexGuard`]. +struct LoroMutexGuardInner<'a, T> { + inner: &'a LoroMutex, + this: LockInfo, + last: LockInfo, +} + +impl Deref for LoroMutexGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.guard + } +} + +impl DerefMut for LoroMutexGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + &mut self.guard + } +} + +impl std::fmt::Debug for LoroMutexGuard<'_, T> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LoroMutex") + .field("data", &self.guard) + .finish() + } +} + +impl<'a, T> LoroMutexGuard<'a, T> { + /// Extract the underlying [`MutexGuard`], detaching order tracking. + /// + /// This consumes `self`, performs the release-order bookkeeping immediately + /// (making the thread-local lock info revert to the previous state), and + /// returns the raw guard. Subsequent lock acquisitions in this thread will no + /// longer consider this guard as held, which means order violations will not + /// be detected relative to it. + /// + /// Prefer to keep using [`LoroMutexGuard`] unless integrating with APIs that + /// require a plain [`MutexGuard`]. Misuse can lead to missing diagnostics. + pub fn take_guard(self) -> MutexGuard<'a, T> { + self.guard + } +} + +impl Drop for LoroMutexGuardInner<'_, T> { + fn drop(&mut self) { + let cur = self.inner.currently_locked_in_this_thread.get_or_default(); + let current_lock_info = *cur.lock(); + if current_lock_info.kind != self.this.kind { + let bt = Backtrace::capture(); + eprintln!("Locking release order violation callstack:\n{}", bt); + panic!( + "Locking release order violation. self.this: {}, self.last: {}, current: {}", + self.this, self.last, current_lock_info + ); + } + + *cur.lock() = self.last; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic(expected = "Locking order violation")] + fn test_locking_order_violation_shows_caller() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::DocState); + let mutex2 = group.new_lock(2, LockKind::Txn); + + let _guard1 = mutex1.lock(); // Lock higher priority first + let _guard2 = mutex2.lock(); // This should panic with caller info + } + + #[test] + fn test_locking_order_when_dropped_in_order() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let mutex3 = group.new_lock(3, LockKind::DocState); + let _guard1 = mutex1.lock(); + drop(_guard1); + let _guard2 = mutex2.lock(); + drop(_guard2); + let _guard3 = mutex3.lock(); + } + + #[test] + #[should_panic] + fn test_locking_order_when_not_dropped_in_reverse_order() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let _guard1 = mutex1.lock(); + let _guard2 = mutex2.lock(); + drop(_guard1); + drop(_guard2); + } + + #[test] + fn test_dropping_should_restore_last_lock_info_0() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let mutex3 = group.new_lock(3, LockKind::DocState); + let _guard1 = mutex1.lock(); + let _guard3 = mutex3.lock(); + drop(_guard3); + let _guard2 = mutex2.lock(); + drop(_guard2); + } + + #[test] + #[should_panic] + fn test_dropping_should_restore_last_lock_info_1() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::OpLog); + let mutex3 = group.new_lock(3, LockKind::DocState); + let _guard2 = mutex2.lock(); + let _guard3 = mutex3.lock(); + drop(_guard3); + let _guard1 = mutex1.lock(); + } + + #[test] + fn test_nested_locking_same_kind() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::Txn); + let mutex2 = group.new_lock(2, LockKind::Txn); + + let guard1 = mutex1.lock(); + // Locking same kind should work (cur >= self.kind, so this would fail) + // Actually, let's test this properly - same kind should fail + drop(guard1); + + let _guard2 = mutex2.lock(); // This should work when guard1 is dropped + } + + #[test] + fn test_lock_kind_enum_values() { + assert_eq!(LockKind::None as u8, 0); + assert_eq!(LockKind::Txn as u8, 1); + assert_eq!(LockKind::OpLog as u8, 2); + assert_eq!(LockKind::DocState as u8, 3); + assert_eq!(LockKind::DiffCalculator as u8, 4); + } + + #[test] + fn test_is_locked_functionality() { + let group = LoroLockGroup::new(); + let mutex = group.new_lock(42, LockKind::Txn); + + assert!(!mutex.is_locked()); + + let _guard = mutex.lock(); + assert!(mutex.is_locked()); + } + + // Helper function to test that panic messages contain caller info + #[test] + #[should_panic(expected = "Locking order violation")] + fn test_panic_message_contains_location_info() { + let group = LoroLockGroup::new(); + let mutex1 = group.new_lock(1, LockKind::DocState); + let mutex2 = group.new_lock(2, LockKind::Txn); + + let _guard1 = mutex1.lock(); + + // This line should be reported in the panic message + let _guard2 = mutex2.lock(); + } + + #[test] + #[should_panic(expected = "poisoned LoroMutex")] + fn test_poisoned_lock_panics_after_unwind() { + let group = LoroLockGroup::new(); + let mutex = group.new_lock(42, LockKind::Txn); + + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = mutex.lock(); + panic!("poison the lock"); + })); + + let _ = mutex.lock(); + } +} diff --git a/vendor/loro-internal/src/loro.rs b/vendor/loro-internal/src/loro.rs new file mode 100644 index 00000000..d195b7fa --- /dev/null +++ b/vendor/loro-internal/src/loro.rs @@ -0,0 +1,3186 @@ +use crate::encoding::json_schema::{encode_change, export_json_in_id_span}; +pub use crate::encoding::ExportMode; +use crate::pre_commit::{FirstCommitFromPeerCallback, FirstCommitFromPeerPayload}; +pub use crate::state::analyzer::{ContainerAnalysisInfo, DocAnalysis}; +use crate::sync::{AtomicBool, AtomicUsize}; +pub(crate) use crate::LoroDocInner; +use crate::{ + arena::SharedArena, + change::{Change, Timestamp}, + configure::{Configure, DefaultRandom, SecureRandomGenerator, StyleConfig}, + container::{ + idx::ContainerIdx, list::list_op::InnerListOp, richtext::config::StyleConfigMap, + IntoContainerId, + }, + cursor::{AbsolutePosition, CannotFindRelativePosition, Cursor, PosQueryResult}, + dag::{Dag, DagUtils}, + diff_calc::DiffCalculator, + encoding::{ + self, decode_snapshot, export_fast_snapshot, export_fast_updates, + export_fast_updates_in_range, export_shallow_snapshot, export_snapshot_at, + export_state_only_snapshot, + json_schema::{encode_change_to_json, json::JsonSchema}, + parse_header_and_body, EncodeMode, ImportBlobMetadata, ImportStatus, ParsedHeaderAndBody, + }, + event::{str_to_path, EventTriggerKind, Index, InternalDocDiff}, + handler::{Handler, MovableListHandler, TextHandler, TreeHandler, ValueOrHandler}, + id::PeerID, + json::JsonChange, + op::InnerContent, + oplog::{loro_dag::FrontiersNotIncluded, OpLog}, + state::DocState, + subscription::{LocalUpdateCallback, Observer, Subscriber}, + undo::DiffBatch, + utils::subscription::{SubscriberSetWithQueue, Subscription}, + version::{shrink_frontiers, Frontiers, ImVersionVector, VersionRange, VersionVectorDiff}, + ChangeMeta, DocDiff, HandlerTrait, InternalString, ListHandler, LoroDoc, LoroError, MapHandler, + VersionVector, +}; +use crate::{change::ChangeRef, lock::LockKind}; +use crate::{lock::LoroMutexGuard, pre_commit::PreCommitCallback}; +use crate::{ + lock::{LoroLockGroup, LoroMutex}, + txn::Transaction, +}; +use either::Either; +use loro_common::{ + ContainerID, ContainerType, HasCounterSpan, HasIdSpan, HasLamportSpan, IdSpan, LoroEncodeError, + LoroResult, LoroValue, ID, +}; +use rle::HasLength; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::{ + borrow::Cow, + cmp::Ordering, + collections::{hash_map::Entry, BinaryHeap}, + ops::ControlFlow, + sync::{ + atomic::Ordering::{Acquire, Release}, + Arc, + }, +}; +use tracing::{debug_span, info_span, instrument, warn}; + +impl Default for LoroDoc { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for LoroDocInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LoroDoc") + .field("config", &self.config) + .field("auto_commit", &self.auto_commit) + .field("detached", &self.detached) + .finish() + } +} + +impl LoroDoc { + /// Run the provided closure within a commit barrier. + /// + /// This finalizes any pending auto-commit transaction first (preserving + /// options across an empty txn), executes `f`, then renews the transaction + /// (carrying preserved options) if auto-commit is enabled. This is the + /// common implicit-commit pattern used by internal operations such as + /// import/export/checkouts. + #[inline] + pub fn with_barrier(&self, f: F) -> R + where + F: FnOnce() -> R, + { + let (options, guard) = self.implicit_commit_then_stop(); + let result = f(); + drop(guard); + self.renew_txn_if_auto_commit(options); + result + } + + pub fn new() -> Self { + let visible_op_count = Arc::new(AtomicUsize::new(0)); + let oplog = OpLog::new(visible_op_count.clone()); + let arena = oplog.arena.clone(); + let config: Configure = oplog.configure.clone(); + let lock_group = LoroLockGroup::new(); + let global_txn = Arc::new(lock_group.new_lock(None, LockKind::Txn)); + let inner = Arc::new_cyclic(|w| { + let state = DocState::new_arc(w.clone(), arena.clone(), config.clone(), &lock_group); + LoroDocInner { + oplog: Arc::new(lock_group.new_lock(oplog, LockKind::OpLog)), + state, + config, + visible_op_count, + detached: AtomicBool::new(false), + auto_commit: AtomicBool::new(false), + observer: Arc::new(Observer::new(arena.clone())), + diff_calculator: Arc::new( + lock_group.new_lock(DiffCalculator::new(true), LockKind::DiffCalculator), + ), + // §act-five P1.B: retained import-path calculator (see lib.rs). + import_diff_calculator: Arc::new( + lock_group.new_lock(DiffCalculator::new(true), LockKind::DiffCalculator), + ), + txn: global_txn, + arena, + local_update_subs: SubscriberSetWithQueue::new(), + peer_id_change_subs: SubscriberSetWithQueue::new(), + pre_commit_subs: SubscriberSetWithQueue::new(), + first_commit_from_peer_subs: SubscriberSetWithQueue::new(), + } + }); + LoroDoc { inner } + } + + pub fn fork(&self) -> Self { + if self.is_detached() { + return self + .fork_at(&self.state_frontiers()) + .expect("fork_at on detached doc should not fail"); + } + + let snapshot = self.with_barrier(|| encoding::fast_snapshot::encode_snapshot_inner(self)); + let doc = Self::new(); + doc.with_barrier(|| { + encoding::fast_snapshot::decode_snapshot_inner(snapshot, &doc, Default::default()) + }) + .unwrap(); + doc.set_config(&self.config); + if self.auto_commit.load(std::sync::atomic::Ordering::Relaxed) { + doc.start_auto_commit(); + } + doc + } + /// Enables editing of the document in detached mode. + /// + /// By default, the document cannot be edited in detached mode (after calling + /// `detach` or checking out a version other than the latest). This method + /// allows editing in detached mode. + /// + /// # Important Notes: + /// + /// - After enabling this mode, the document will use a different PeerID. Each + /// time you call checkout, a new PeerID will be used. + /// - If you set a custom PeerID while this mode is enabled, ensure that + /// concurrent operations with the same PeerID are not possible. + /// - On detached mode, importing will not change the state of the document. + /// It also doesn't change the version of the [DocState]. The changes will be + /// recorded into [OpLog] only. You need to call `checkout` to make it take effect. + pub fn set_detached_editing(&self, enable: bool) { + self.config.set_detached_editing(enable); + if enable && self.is_detached() { + self.with_barrier(|| { + self.renew_peer_id(); + }); + } + } + + /// Create a doc with auto commit enabled. + #[inline] + pub fn new_auto_commit() -> Self { + let doc = Self::new(); + doc.start_auto_commit(); + doc + } + + #[inline(always)] + pub fn set_peer_id(&self, peer: PeerID) -> LoroResult<()> { + if peer == PeerID::MAX { + return Err(LoroError::InvalidPeerID); + } + let next_id = self.oplog.lock().next_id(peer); + if self.auto_commit.load(Acquire) { + let doc_state = self.state.lock(); + doc_state + .peer + .store(peer, std::sync::atomic::Ordering::Relaxed); + + if doc_state.is_in_txn() { + drop(doc_state); + // Use implicit-style barrier to avoid swallowing next-commit options + self.with_barrier(|| {}); + } + self.peer_id_change_subs.emit(&(), next_id); + return Ok(()); + } + + let doc_state = self.state.lock(); + if doc_state.is_in_txn() { + return Err(LoroError::TransactionError( + "Cannot change peer id during transaction" + .to_string() + .into_boxed_str(), + )); + } + + doc_state + .peer + .store(peer, std::sync::atomic::Ordering::Relaxed); + drop(doc_state); + self.peer_id_change_subs.emit(&(), next_id); + Ok(()) + } + + /// Renews the PeerID for the document. + pub(crate) fn renew_peer_id(&self) { + let mut peer_id = DefaultRandom.next_u64(); + while peer_id == PeerID::MAX { + peer_id = DefaultRandom.next_u64(); + } + self.set_peer_id(peer_id).unwrap(); + } + + /// Implicitly commit the cumulative auto-commit transaction. + /// This method only has effect when `auto_commit` is true. + /// + /// Follow-ups: the caller is responsible for renewing the transaction + /// as needed (e.g., via `renew_txn_if_auto_commit`). Prefer using + /// `with_barrier(...)` for most internal flows to handle this safely. + /// + /// Empty-commit behavior: if the pending transaction is empty, the returned + /// `Some(CommitOptions)` preserves next-commit options such as message and + /// timestamp so they can carry into the renewed transaction. Transient + /// labels like `origin` do not carry across an empty commit. + #[inline] + #[must_use] + pub fn implicit_commit_then_stop( + &self, + ) -> ( + Option, + LoroMutexGuard<'_, Option>, + ) { + // Implicit commit: preserve options on empty commit + let (a, b) = self.commit_internal(CommitOptions::new().immediate_renew(false), true); + (a, b.unwrap()) + } + + /// Commit the cumulative auto commit transaction. + /// It will start the next one immediately + /// + /// It only returns Some(options_of_the_empty_txn) when the txn is empty + #[inline] + pub fn commit_then_renew(&self) -> Option { + // Explicit commit: swallow options on empty commit + self.commit_internal(CommitOptions::new().immediate_renew(true), false) + .0 + } + + /// This method is called before the commit. + /// It can be used to modify the change before it is committed. + /// + /// It return Some(txn) if the txn is None + fn before_commit(&self) -> Option>> { + let mut txn_guard = self.txn.lock(); + let Some(txn) = txn_guard.as_mut() else { + return Some(txn_guard); + }; + + if txn.is_peer_first_appearance { + txn.is_peer_first_appearance = false; + drop(txn_guard); + // First commit from a peer + self.first_commit_from_peer_subs.emit( + &(), + FirstCommitFromPeerPayload { + peer: self.peer_id(), + }, + ); + } + + None + } + + /// Core implementation for committing the cumulative auto-commit transaction. + /// + /// - When `preserve_on_empty` is true (implicit commits like export/checkout), + /// commit options from an empty transaction are carried over to the next transaction + /// (except `origin`, which never carries across an empty commit). + /// - When `preserve_on_empty` is false (explicit commits), commit options from an + /// empty transaction are swallowed and NOT carried over. + #[instrument(skip_all)] + fn commit_internal( + &self, + config: CommitOptions, + preserve_on_empty: bool, + ) -> ( + Option, + Option>>, + ) { + if !self.auto_commit.load(Acquire) { + let txn_guard = self.txn.lock(); + // if not auto_commit, nothing should happen + // because the global txn is not used + return (None, Some(txn_guard)); + } + + loop { + if let Some(txn_guard) = self.before_commit() { + return (None, Some(txn_guard)); + } + + let mut txn_guard = self.txn.lock(); + let txn = txn_guard.take(); + let Some(mut txn) = txn else { + return (None, Some(txn_guard)); + }; + let on_commit = txn.take_on_commit(); + if let Some(origin) = config.origin.clone() { + txn.set_origin(origin); + } + + if let Some(timestamp) = config.timestamp { + txn.set_timestamp(timestamp); + } + + if let Some(msg) = config.commit_msg.as_ref() { + txn.set_msg(Some(msg.clone())); + } + + let id_span = txn.id_span(); + let mut options = txn.commit().unwrap(); + // Empty commit returns Some(options). We may preserve parts of it for implicit commits. + if let Some(opts) = options.as_mut() { + // `origin` is an event-only label and never carries across an empty commit + if config.origin.is_some() { + opts.set_origin(None); + } + // For explicit commits, swallow options from empty commit entirely + if !preserve_on_empty { + options = None; + } + } + if config.immediate_renew && self.can_edit() { + let mut t = self.txn().unwrap(); + if let Some(options) = options.as_ref() { + t.set_options(options.clone()); + } + *txn_guard = Some(t); + } + + if let Some(on_commit) = on_commit { + drop(txn_guard); + on_commit(&self.state, &self.oplog, id_span); + txn_guard = self.txn.lock(); + if !config.immediate_renew && txn_guard.is_some() { + // make sure that txn_guard is None when config.immediate_renew is false + continue; + } + } + + return ( + options, + if !config.immediate_renew { + Some(txn_guard) + } else { + None + }, + ); + } + } + + /// Commit the cumulative auto commit transaction (explicit API). + /// + /// This is used by user-facing explicit commits. If the transaction is empty, + /// any provided commit options are swallowed and will NOT carry over. + #[instrument(skip_all)] + pub fn commit_with( + &self, + config: CommitOptions, + ) -> ( + Option, + Option>>, + ) { + self.commit_internal(config, false) + } + + /// Set the commit message of the next commit + pub fn set_next_commit_message(&self, message: &str) { + let mut binding = self.txn.lock(); + let Some(txn) = binding.as_mut() else { + return; + }; + + if message.is_empty() { + txn.set_msg(None) + } else { + txn.set_msg(Some(message.into())) + } + } + + /// Set the origin of the next commit + pub fn set_next_commit_origin(&self, origin: &str) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_origin(origin.into()); + } + } + + /// Set the timestamp of the next commit + pub fn set_next_commit_timestamp(&self, timestamp: Timestamp) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_timestamp(timestamp); + } + } + + /// Set the options of the next commit + pub fn set_next_commit_options(&self, options: CommitOptions) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_options(options); + } + } + + /// Clear the options of the next commit + pub fn clear_next_commit_options(&self) { + let mut txn = self.txn.lock(); + if let Some(txn) = txn.as_mut() { + txn.set_options(CommitOptions::new()); + } + } + + /// Set whether to record the timestamp of each change. Default is `false`. + /// + /// If enabled, the Unix timestamp will be recorded for each change automatically. + /// + /// You can also set each timestamp manually when you commit a change. + /// The timestamp manually set will override the automatic one. + /// + /// NOTE: Timestamps are forced to be in ascending order. + /// If you commit a new change with a timestamp that is less than the existing one, + /// the largest existing timestamp will be used instead. + #[inline] + pub fn set_record_timestamp(&self, record: bool) { + self.config.set_record_timestamp(record); + } + + /// Set the interval of mergeable changes, in seconds. + /// + /// If two continuous local changes are within the interval, they will be merged into one change. + /// The default value is 1000 seconds. + #[inline] + pub fn set_change_merge_interval(&self, interval: i64) { + self.config.set_merge_interval(interval); + } + + pub fn can_edit(&self) -> bool { + !self.is_detached() || self.config.detached_editing() + } + + pub fn is_detached_editing_enabled(&self) -> bool { + self.config.detached_editing() + } + + #[inline] + pub fn config_text_style(&self, text_style: StyleConfigMap) { + self.config.text_style_config.write().map = text_style.map; + } + + #[inline] + pub fn config_default_text_style(&self, text_style: Option) { + self.config.text_style_config.write().default_style = text_style; + } + pub fn from_snapshot(bytes: &[u8]) -> LoroResult { + let doc = Self::new(); + let ParsedHeaderAndBody { mode, body, .. } = parse_header_and_body(bytes, true)?; + if mode.is_snapshot() { + doc.with_barrier(|| -> Result<(), LoroError> { + decode_snapshot(&doc, mode, body, Default::default())?; + Ok(()) + })?; + Ok(doc) + } else { + Err(LoroError::DecodeError( + "Invalid encode mode".to_string().into(), + )) + } + } + + /// Is the document empty? (no ops) + #[inline(always)] + pub fn can_reset_with_snapshot(&self) -> bool { + let oplog = self.oplog.lock(); + if oplog.batch_importing { + return false; + } + + if self.is_detached() { + return false; + } + + oplog.is_empty() && self.state.lock().can_import_snapshot() + } + + /// Whether [OpLog] and [DocState] are detached. + /// + /// If so, the document is in readonly mode by default and importing will not change the state of the document. + /// It also doesn't change the version of the [DocState]. The changes will be recorded into [OpLog] only. + /// You need to call `checkout` to make it take effect. + #[inline(always)] + pub fn is_detached(&self) -> bool { + self.detached.load(Acquire) + } + + pub(crate) fn set_detached(&self, detached: bool) { + self.detached.store(detached, Release); + } + + #[inline(always)] + pub fn peer_id(&self) -> PeerID { + self.state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed) + } + + #[inline(always)] + pub fn detach(&self) { + self.with_barrier(|| self.set_detached(true)); + } + + #[inline(always)] + pub fn attach(&self) { + self.checkout_to_latest() + } + + /// Get the timestamp of the current state. + /// It's the last edit time of the [DocState]. + pub fn state_timestamp(&self) -> Timestamp { + // Acquire locks in correct order: read frontiers first, then query OpLog. + let f = { self.state.lock().frontiers.clone() }; + self.oplog.lock().get_timestamp_of_version(&f) + } + + #[inline(always)] + pub fn app_state(&self) -> &Arc> { + &self.state + } + + #[inline] + pub fn get_state_deep_value(&self) -> LoroValue { + self.state.lock().get_deep_value() + } + + #[inline(always)] + pub fn oplog(&self) -> &Arc> { + &self.oplog + } + + #[inline(always)] + pub fn import(&self, bytes: &[u8]) -> Result { + let s = debug_span!("import", peer = self.peer_id()); + let _e = s.enter(); + self.import_with(bytes, Default::default()) + } + + #[inline] + pub fn import_with( + &self, + bytes: &[u8], + origin: InternalString, + ) -> Result { + self.with_barrier(|| self._import_with(bytes, origin)) + } + + #[tracing::instrument(skip_all)] + fn _import_with( + &self, + bytes: &[u8], + origin: InternalString, + ) -> Result { + ensure_cov::notify_cov("loro_internal::import"); + let parsed = parse_header_and_body(bytes, true)?; + loro_common::info!("Importing with mode={:?}", &parsed.mode); + let result = match parsed.mode { + EncodeMode::OutdatedRle => { + if self.state.lock().is_in_txn() { + return Err(LoroError::ImportWhenInTxn); + } + + let s = tracing::span!( + tracing::Level::INFO, + "Import updates ", + peer = self.peer_id() + ); + let _e = s.enter(); + self.update_oplog_and_apply_delta_to_state_if_needed( + |oplog| oplog.decode(parsed), + origin, + ) + } + EncodeMode::OutdatedSnapshot => { + if self.can_reset_with_snapshot() { + loro_common::info!("Init by snapshot {}", self.peer_id()); + decode_snapshot(self, parsed.mode, parsed.body, origin) + } else { + self.update_oplog_and_apply_delta_to_state_if_needed( + |oplog| oplog.decode(parsed), + origin, + ) + } + } + EncodeMode::FastSnapshot => { + if self.can_reset_with_snapshot() { + ensure_cov::notify_cov("loro_internal::import::snapshot"); + loro_common::info!("Init by fast snapshot {}", self.peer_id()); + decode_snapshot(self, parsed.mode, parsed.body, origin) + } else { + self.import_changes_and_apply_delta_to_state_if_needed( + |oplog| encoding::decode_oplog_changes(oplog, parsed), + origin, + ) + + // let new_doc = LoroDoc::new(); + // new_doc.import(bytes)?; + // let updates = new_doc.export(ExportMode::updates(&self.oplog_vv())).unwrap(); + // return self.import_with(updates.as_slice(), origin); + } + } + EncodeMode::FastUpdates => self.import_changes_and_apply_delta_to_state_if_needed( + |oplog| encoding::decode_oplog_changes(oplog, parsed), + origin, + ), + EncodeMode::Auto => { + unreachable!() + } + }; + + self.emit_events(); + + result + } + + #[tracing::instrument(skip_all)] + pub(crate) fn update_oplog_and_apply_delta_to_state_if_needed( + &self, + f: impl FnOnce(&mut OpLog) -> Result, + origin: InternalString, + ) -> Result { + let mut oplog = self.oplog.lock(); + oplog.begin_import_rollback(); + if !self.is_detached() { + let old_vv = oplog.vv().clone(); + let old_frontiers = oplog.frontiers().clone(); + let result = f(&mut oplog); + if &old_vv != oplog.vv() { + // §act-five P1.B: acquire state BEFORE the diff calculator so the + // retained calculator (LockKind::DiffCalculator=4) is taken after + // state (=3) — valid lock order (OpLog { + oplog.commit_import_rollback(); + Ok(result) + } + Err(e) => { + // Some import errors are reported after a valid prefix has + // been inserted into the oplog. Keep that prefix if it was + // also applied to state; rollback is for failed state apply + // or decode errors that made no visible oplog progress. + if &old_vv == oplog.vv() { + oplog.rollback_import(); + } else { + oplog.commit_import_rollback(); + } + Err(e) + } + } + } else { + match f(&mut oplog) { + Ok(result) => { + oplog.commit_import_rollback(); + Ok(result) + } + Err(e) => { + oplog.rollback_import(); + Err(e) + } + } + } + } + + #[tracing::instrument(skip_all)] + pub(crate) fn import_changes_and_apply_delta_to_state_if_needed( + &self, + decode_changes: impl FnOnce(&mut OpLog) -> Result, LoroError>, + origin: InternalString, + ) -> Result { + let mut oplog = self.oplog.lock(); + let arena_checkpoint = oplog.arena.checkpoint_for_rollback(); + let changes = match decode_changes(&mut oplog) { + Ok(changes) => changes, + Err(e) => { + oplog.arena.rollback(arena_checkpoint); + return Err(e); + } + }; + + let preflight = oplog.preflight_import_changes(&changes); + if preflight.has_deps_before_shallow_root + && (self.is_detached() || !preflight.applies_to_dag) + { + oplog.arena.rollback(arena_checkpoint); + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + if self.is_detached() { + let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + if result.has_deps_before_shallow_root { + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + return Ok(result.status); + } + + if !preflight.applies_to_dag { + let pending_root_containers = pending_root_containers_to_materialize(&oplog, &changes); + let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + if result.has_deps_before_shallow_root { + oplog.arena.rollback(arena_checkpoint); + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + if !pending_root_containers.is_empty() { + let mut state = self.state.lock(); + for id in pending_root_containers { + state.ensure_container(&id); + } + } + + return Ok(result.status); + } + + let old_vv = oplog.vv().clone(); + let old_frontiers = oplog.frontiers().clone(); + let rollback_enabled = preflight.needs_state_apply_rollback; + if rollback_enabled { + oplog.begin_import_rollback_with_arena(arena_checkpoint); + } + + let result = encoding::apply_decoded_changes_to_oplog(&mut oplog, changes); + if &old_vv != oplog.vv() { + // §act-five P1.B: state before the retained calculator (valid lock order). + let mut state = self.state.lock(); + let (diff, diff_mode) = self.import_diff_calculator.lock().calc_diff_internal_tagged( + &oplog, + &old_vv, + &old_frontiers, + oplog.vv(), + oplog.dag.get_frontiers(), + None, + "import-changes", + ); + if let Err(e) = state.apply_diff( + InternalDocDiff { + origin, + diff: (diff).into(), + by: EventTriggerKind::Import, + new_version: Cow::Owned(oplog.frontiers().clone()), + }, + diff_mode, + ) { + if rollback_enabled { + oplog.rollback_import(); + return Err(e); + } + + panic!("state apply returned Err for import without rollback guard: {e}"); + } + } + + if result.has_deps_before_shallow_root { + if rollback_enabled { + oplog.commit_import_rollback(); + } + return Err(LoroError::ImportUpdatesThatDependsOnOutdatedVersion); + } + + if rollback_enabled { + oplog.commit_import_rollback(); + } + Ok(result.status) + } + + fn emit_events(&self) { + // we should not hold the lock when emitting events + let events = { + let mut state = self.state.lock(); + state.take_events() + }; + for event in events { + self.observer.emit(event); + } + } + + pub(crate) fn drop_pending_events(&self) -> Vec { + let mut state = self.state.lock(); + state.take_events() + } + + /// Import the json schema updates. + /// + /// only supports backward compatibility but not forward compatibility. + #[tracing::instrument(skip_all)] + pub fn import_json_updates>(&self, json: T) -> LoroResult { + let json = json.try_into().map_err(|_| LoroError::InvalidJsonSchema)?; + self.with_barrier(|| { + let result = self.import_changes_and_apply_delta_to_state_if_needed( + |oplog| crate::encoding::json_schema::decode_json_changes(json, &oplog.arena), + Default::default(), + ); + self.emit_events(); + result + }) + } + + pub fn export_json_updates( + &self, + start_vv: &VersionVector, + end_vv: &VersionVector, + with_peer_compression: bool, + ) -> JsonSchema { + self.with_barrier(|| { + let oplog = self.oplog.lock(); + let mut start_vv = start_vv; + let _temp: Option; + if !oplog.dag.shallow_since_vv().is_empty() { + // Make sure that start_vv >= shallow_since_vv + let mut include_all = true; + for (peer, counter) in oplog.dag.shallow_since_vv().iter() { + if start_vv.get(peer).unwrap_or(&0) < counter { + include_all = false; + break; + } + } + if !include_all { + let mut vv = start_vv.clone(); + for (&peer, &counter) in oplog.dag.shallow_since_vv().iter() { + vv.extend_to_include_end_id(ID::new(peer, counter)); + } + _temp = Some(vv); + start_vv = _temp.as_ref().unwrap(); + } + } + + crate::encoding::json_schema::export_json( + &oplog, + start_vv, + end_vv, + with_peer_compression, + ) + }) + } + + pub fn export_json_in_id_span(&self, id_span: IdSpan) -> Vec { + let oplog = self.oplog.lock(); + let mut changes = export_json_in_id_span(&oplog, id_span); + if let Some(uncommit) = oplog.get_uncommitted_change_in_span(id_span) { + let change_json = encode_change(ChangeRef::from_change(&uncommit), &self.arena, None); + changes.push(change_json); + } + changes + } + + /// Get the version vector of the current OpLog + #[inline] + pub fn oplog_vv(&self) -> VersionVector { + self.oplog.lock().vv().clone() + } + + /// Get the version vector of the current [DocState] + #[inline] + pub fn state_vv(&self) -> VersionVector { + let oplog = self.oplog.lock(); + let f = &self.state.lock().frontiers; + oplog.dag.frontiers_to_vv(f).unwrap() + } + + pub fn get_by_path(&self, path: &[Index]) -> Option { + let value: LoroValue = self.state.lock().get_value_by_path(path)?; + if let LoroValue::Container(c) = value { + Some(ValueOrHandler::Handler(Handler::new_attached( + c.clone(), + self.clone(), + ))) + } else { + Some(ValueOrHandler::Value(value)) + } + } + + /// Get the handler by the string path. + pub fn get_by_str_path(&self, path: &str) -> Option { + let path = str_to_path(path)?; + self.get_by_path(&path) + } + + pub fn get_uncommitted_ops_as_json(&self) -> Option { + let arena = &self.arena; + let txn = self.txn.lock(); + let txn = txn.as_ref()?; + let ops_ = txn.local_ops(); + let new_id = ID { + peer: *txn.peer(), + counter: ops_.first()?.counter, + }; + let change = ChangeRef { + id: &new_id, + deps: txn.frontiers(), + timestamp: &txn + .timestamp() + .as_ref() + .copied() + .unwrap_or_else(|| self.oplog.lock().get_timestamp_for_next_txn()), + commit_msg: txn.msg(), + ops: ops_, + lamport: txn.lamport(), + }; + let json = encode_change_to_json(change, arena); + Some(json) + } + + #[inline] + pub fn get_handler(&self, id: ContainerID) -> Option { + if self.has_container(&id) { + self.ensure_root_container(&id); + Some(Handler::new_attached(id, self.clone())) + } else { + None + } + } + + #[inline] + fn ensure_root_container(&self, id: &ContainerID) { + if id.is_root() { + self.state.lock().ensure_container(id); + } + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_text(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Text); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_text().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_text(&self, id: I) -> TextHandler { + self.try_get_text(id) + .expect("The container does not exist in the document. Use `try_get_text` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_list(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::List); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_list().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_list(&self, id: I) -> ListHandler { + self.try_get_list(id) + .expect("The container does not exist in the document. Use `try_get_list` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_movable_list(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::MovableList); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()) + .into_movable_list() + .ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_movable_list(&self, id: I) -> MovableListHandler { + self.try_get_movable_list(id) + .expect("The container does not exist in the document. Use `try_get_movable_list` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_map(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Map); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_map().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_map(&self, id: I) -> MapHandler { + self.try_get_map(id) + .expect("The container does not exist in the document. Use `try_get_map` or `get_container` to check for existence.") + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn try_get_tree(&self, id: I) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Tree); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_tree().ok() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + #[inline] + pub fn get_tree(&self, id: I) -> TreeHandler { + self.try_get_tree(id) + .expect("The container does not exist in the document. Use `try_get_tree` or `get_container` to check for existence.") + } + + #[cfg(feature = "counter")] + pub fn try_get_counter( + &self, + id: I, + ) -> Option { + let id = id.into_container_id(&self.arena, ContainerType::Counter); + if !self.has_container(&id) { + return None; + } + self.ensure_root_container(&id); + Handler::new_attached(id, self.clone()).into_counter().ok() + } + + #[cfg(feature = "counter")] + pub fn get_counter( + &self, + id: I, + ) -> crate::handler::counter::CounterHandler { + self.try_get_counter(id) + .expect("The container does not exist in the document. Use `try_get_counter` or `get_container` to check for existence.") + } + + #[must_use] + pub fn has_container(&self, id: &ContainerID) -> bool { + if id.is_root() && !id.is_mergeable() { + return true; + } + + let exist = self.state.lock().does_container_exist(id); + exist + } + + /// Undo the operations between the given id_span. It can be used even in a collaborative environment. + /// + /// This is an internal API. You should NOT use it directly. + /// + /// # Internal + /// + /// This method will use the diff calculator to calculate the diff required to time travel + /// from the end of id_span to the beginning of the id_span. Then it will convert the diff to + /// operations and apply them to the OpLog with a dep on the last id of the given id_span. + /// + /// This implementation is kinda slow, but it's simple and maintainable. We can optimize it + /// further when it's needed. The time complexity is O(n + m), n is the ops in the id_span, m is the + /// distance from id_span to the current latest version. + #[instrument(level = "info", skip_all)] + pub fn undo_internal( + &self, + id_span: IdSpan, + container_remap: &mut FxHashMap, + post_transform_base: Option<&DiffBatch>, + before_diff: &mut dyn FnMut(&DiffBatch), + ) -> LoroResult> { + if !self.can_edit() { + return Err(LoroError::EditWhenDetached); + } + + let (options, txn) = self.implicit_commit_then_stop(); + if !self.oplog().lock().vv().includes_id(id_span.id_last()) { + self.renew_txn_if_auto_commit(options); + return Err(LoroError::UndoInvalidIdSpan(id_span.id_last())); + } + + // §fidelity (flowstate vendor patch): the style keys each undone op + // actually MARKED, per container. The checkout-based undo diff below + // over-reports the full resolved style set of the affected span; used + // by `restrict_undo_diff_to_marked_keys` to keep only these keys so the + // undo cannot clobber a concurrent remote peer's mark of another key on + // the same range (field bug: a local run-style highlight undo reverting + // a concurrent remote paragraph-style tag). + let marked_keys_by_container: FxHashMap> = { + let oplog = self.oplog.lock(); + let mut map: FxHashMap> = FxHashMap::default(); + for rich_op in oplog.iter_ops(id_span) { + let op = rich_op.op(); + if let InnerContent::List(InnerListOp::StyleStart { key, .. }) = &op.content { + if let Some(cid) = oplog.arena.get_container_id(rich_op.container()) { + map.entry(cid).or_default().insert(key.to_string()); + } + } + } + map + }; + + let (was_recording, latest_frontiers) = { + let mut state = self.state.lock(); + let was_recording = state.is_recording(); + state.stop_and_clear_recording(); + (was_recording, state.frontiers.clone()) + }; + + // Env-gated coarse timing (`LORO_UNDO_TIMING=1`): the undo of a large + // op spends seconds in here; these split the cost across span + // calculation, per-span checkout diffing, the return checkout, and + // the diff re-apply without a profiler run. + let timing = std::env::var_os("LORO_UNDO_TIMING").is_some(); + let t0 = std::time::Instant::now(); + let spans = self.oplog.lock().split_span_based_on_deps(id_span); + let span_count = spans.len(); + let t_split = t0.elapsed(); + // §flowstate: a "warm the checkout diff calculator once from + // spans[0].deps" pre-pass was tried here (vendor #20, 2026-07-10) to + // collapse the undo's two backward-LCA tracker rebuilds into one + // build. REVERTED: across five bench runs it produced 1.5s / 3.2s / + // 79s / 99s tail undos (the pathological mode is a future-status + // `in_between` scan quadratic inside `CrdtRope::insert` when the + // build starts one op deeper, at the undone span's deps). The slow + // path stays at its BOUNDED two-rebuild cost; the durable fix is the + // P1.C incremental checkout (bounded walk from retained coverage + // instead of the LCA), tracked in flowstate_oom_leads.md #9. + let mut calc_diff_calls = 0usize; + let mut calc_diff_total = std::time::Duration::ZERO; + let t1 = std::time::Instant::now(); + let diff = crate::undo::undo( + spans, + match post_transform_base { + Some(d) => Either::Right(d), + None => Either::Left(&latest_frontiers), + }, + |from, to| { + let t = std::time::Instant::now(); + self._checkout_without_emitting(from, false, false).unwrap(); + self.state.lock().start_recording(); + self._checkout_without_emitting(to, false, false).unwrap(); + let mut state = self.state.lock(); + let e = state.take_events(); + state.stop_and_clear_recording(); + calc_diff_calls += 1; + calc_diff_total += t.elapsed(); + DiffBatch::new(e) + }, + before_diff, + ); + let t_undo_fn = t1.elapsed(); + + // println!("\nundo_internal: diff: {:?}", diff); + // println!("container remap: {:?}", container_remap); + + let t2 = std::time::Instant::now(); + self._checkout_without_emitting(&latest_frontiers, false, false)?; + let t_return_checkout = t2.elapsed(); + self.set_detached(false); + if was_recording { + self.state.lock().start_recording(); + } + drop(txn); + self.start_auto_commit(); + // Compact the restore before applying it: an undo of a large deletion + // arrives as one insert item per ORIGINAL op (op-id fragments), and + // `_apply_diff` turns every item into a separate new op — a + // whole-document restore became ~10^5 tiny ops, slow to apply and + // slow on every later replay of the resulting change. The re-applied + // content mints fresh contiguous op ids regardless, so merging + // adjacent same-attribute inserts loses nothing. + let t3 = std::time::Instant::now(); + let diff = compact_text_diff_batch(diff); + // §fidelity: restrict the undo to the style keys the undone ops actually + // marked, so it re-asserts ONLY those (never a concurrent remote peer's + // other-key marks on the same range). No-op when no mark ops were undone. + let diff = if marked_keys_by_container.is_empty() { + diff + } else { + restrict_undo_diff_to_marked_keys(diff, &marked_keys_by_container) + }; + let t_compact = t3.elapsed(); + // Try applying the diff, but ignore the error if it happens. + // MovableList's undo behavior is too tricky to handle in a collaborative env + // so in edge cases this may be an Error + let t4 = std::time::Instant::now(); + if let Err(e) = self._apply_diff(diff, container_remap, true) { + warn!("Undo Failed {:?}", e); + } + if timing { + eprintln!( + "[loro-undo-timing] spans={span_count} split={t_split:?} undo_fn={t_undo_fn:?} (calc_diff n={calc_diff_calls} total={calc_diff_total:?}) return_checkout={t_return_checkout:?} compact={t_compact:?} apply_diff={:?}", + t4.elapsed() + ); + } + + if let Some(options) = options { + self.set_next_commit_options(options); + } + Ok(CommitWhenDrop { + doc: self, + default_options: CommitOptions::new().origin("undo"), + }) + } + + /// Generate a series of local operations that can revert the current doc to the target + /// version. + /// + /// Internally, it will calculate the diff between the current state and the target state, + /// and apply the diff to the current state. + pub fn revert_to(&self, target: &Frontiers) -> LoroResult<()> { + // TODO: test when the doc is readonly + // TODO: test when the doc is detached but enabled editing + let f = self.state_frontiers(); + let diff = self.diff(&f, target)?; + self._apply_diff(diff, &mut Default::default(), false) + } + + /// Calculate the diff between two versions so that apply diff on a will make the state same as b. + /// + /// NOTE: This method will make the doc enter the **detached mode**. + // FIXME: This method needs testing (no event should be emitted during processing this) + pub fn diff(&self, a: &Frontiers, b: &Frontiers) -> LoroResult { + { + // Check whether a and b are valid before checkout so this returns a normal error + // instead of panicking on shallow docs. + let oplog = self.oplog.lock(); + let validate_frontiers = |frontiers: &Frontiers| -> LoroResult<()> { + for id in frontiers.iter() { + if !oplog.dag.contains(id) { + return Err(LoroError::FrontiersNotFound(id)); + } + } + + if oplog.dag.is_before_shallow_root(frontiers) { + return Err(LoroError::SwitchToVersionBeforeShallowRoot); + } + + Ok(()) + }; + + validate_frontiers(a)?; + validate_frontiers(b)?; + } + + let (options, txn) = self.implicit_commit_then_stop(); + let was_detached = self.is_detached(); + let old_frontiers = self.state_frontiers(); + let was_recording = { + let mut state = self.state.lock(); + let is_recording = state.is_recording(); + state.stop_and_clear_recording(); + is_recording + }; + let result = (|| { + self._checkout_without_emitting(a, true, false)?; + self.state.lock().start_recording(); + self._checkout_without_emitting(b, true, false)?; + let mut state = self.state.lock(); + let e = state.take_events(); + state.stop_and_clear_recording(); + Ok::<_, LoroError>(e) + })(); + + // Always restore state regardless of whether diff calculation succeeded + self._checkout_without_emitting(&old_frontiers, false, false) + .unwrap(); + drop(txn); + if !was_detached { + self.set_detached(false); + self.renew_txn_if_auto_commit(options); + } + if was_recording { + self.state.lock().start_recording(); + } + result.map(DiffBatch::new) + } + + /// Apply a diff to the current state. + #[inline(always)] + pub fn apply_diff(&self, diff: DiffBatch) -> LoroResult<()> { + self._apply_diff(diff, &mut Default::default(), true) + } + + /// Apply a diff to the current state. + /// + /// This method will not recreate containers with the same [ContainerID]s. + /// While this can be convenient in certain cases, it can break several internal invariants: + /// + /// 1. Each container should appear only once in the document. Allowing containers with the same ID + /// would result in multiple instances of the same container in the document. + /// 2. Unreachable containers should be removable from the state when necessary. + /// + /// However, the diff may contain operations that depend on container IDs. + /// Therefore, users need to provide a `container_remap` to record and retrieve the container ID remapping. + pub(crate) fn _apply_diff( + &self, + diff: DiffBatch, + container_remap: &mut FxHashMap, + skip_unreachable: bool, + ) -> LoroResult<()> { + if !self.can_edit() { + return Err(LoroError::EditWhenDetached); + } + + let mut ans: LoroResult<()> = Ok(()); + let mut missing_containers: Vec = Vec::new(); + for (mut id, diff) in diff.into_iter() { + let mut remapped = false; + while let Some(rid) = container_remap.get(&id) { + remapped = true; + id = rid.clone(); + } + + if matches!(&id, ContainerID::Normal { .. }) && self.arena.id_to_idx(&id).is_none() { + // Not in arena does not imply non-existent; consult state/kv and register lazily + let exists = self.state.lock().does_container_exist(&id); + if !exists { + missing_containers.push(id); + continue; + } + // Ensure registration so handlers can be created + self.state.lock().ensure_container(&id); + } + + if skip_unreachable && !remapped && !self.state.lock().get_reachable(&id) { + continue; + } + + let Some(h) = self.get_handler(id.clone()) else { + return Err(LoroError::ContainersNotFound { + containers: Box::new(vec![id]), + }); + }; + if let Err(e) = h.apply_diff(diff, container_remap) { + ans = Err(e); + } + } + + if !missing_containers.is_empty() { + return Err(LoroError::ContainersNotFound { + containers: Box::new(missing_containers), + }); + } + + ans + } + + /// This is for debugging purpose. It will travel the whole oplog + #[inline] + pub fn diagnose_size(&self) { + self.oplog().lock().diagnose_size(); + } + + #[inline] + pub fn oplog_frontiers(&self) -> Frontiers { + self.oplog().lock().frontiers().clone() + } + + #[inline] + pub fn state_frontiers(&self) -> Frontiers { + self.state.lock().frontiers.clone() + } + + /// - Ordering::Less means self is less than target or parallel + /// - Ordering::Equal means versions equal + /// - Ordering::Greater means self's version is greater than target + #[inline] + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + self.oplog().lock().cmp_with_frontiers(other) + } + + /// Compare two [Frontiers] causally. + /// + /// If one of the [Frontiers] are not included, it will return [FrontiersNotIncluded]. + #[inline] + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + self.oplog().lock().cmp_frontiers(a, b) + } + + pub fn subscribe_root(&self, callback: Subscriber) -> Subscription { + let mut state = self.state.lock(); + if !state.is_recording() { + state.start_recording(); + } + + self.observer.subscribe_root(callback) + } + + pub fn subscribe(&self, container_id: &ContainerID, callback: Subscriber) -> Subscription { + let mut state = self.state.lock(); + if !state.is_recording() { + state.start_recording(); + } + + self.observer.subscribe(container_id, callback) + } + + pub fn subscribe_local_update(&self, callback: LocalUpdateCallback) -> Subscription { + let (sub, activate) = self.local_update_subs.inner().insert((), callback); + activate(); + sub + } + + // PERF: opt + #[tracing::instrument(skip_all)] + pub fn import_batch(&self, bytes: &[Vec]) -> LoroResult { + if bytes.is_empty() { + return Ok(ImportStatus::default()); + } + + if bytes.len() == 1 { + return self.import(&bytes[0]); + } + + let mut success = VersionRange::default(); + let mut meta_arr = bytes + .iter() + .map(|b| Ok((LoroDoc::decode_import_blob_meta(b, false)?, b))) + .collect::)>>>()?; + meta_arr.sort_by(|a, b| { + a.0.mode + .cmp(&b.0.mode) + .then(b.0.change_num.cmp(&a.0.change_num)) + }); + + let (options, txn) = self.implicit_commit_then_stop(); + // Why we should keep locking `txn` here + // + // In a multi-threaded environment, `import_batch` used to drop the txn lock + // (via `commit_then_stop` + `drop(txn)`) and call `detach()`/`checkout_to_latest()` + // around the batch import. That created a race where another thread could + // start or renew the auto-commit txn and perform local edits while we were + // importing and temporarily detached. Those interleaved local edits could + // violate invariants between `OpLog` and `DocState` (e.g., state being + // updated when we expect it not to, missed events, or inconsistent + // frontiers), as exposed by the loom test `local_edits_during_batch_import`. + // + // The fix is to hold the txn mutex for the entire critical section: + // - Stop the current txn and keep the mutex guard. + // - Force-detach with `set_detached(true)` (avoids `detach()` side effects), + // then run each `_import_with(...)` while detached so imports only touch + // the `OpLog`. + // - After importing, reattach by checking out to latest and renew the txn + // using `_checkout_to_latest_with_guard`, which keeps the mutex held while + // (re)starting the auto-commit txn. + // + // Holding the lock ensures no concurrent thread can create/renew a txn and + // do local edits in the middle of the batch import, making the whole + // operation atomic with respect to local edits. + let is_detached = self.is_detached(); + self.set_detached(true); + self.oplog.lock().batch_importing = true; + let mut err = None; + for (_meta, data) in meta_arr { + match self._import_with(data, Default::default()) { + Ok(s) => { + for (peer, (start, end)) in s.success.iter() { + match success.0.entry(*peer) { + Entry::Occupied(mut e) => { + e.get_mut().1 = *end.max(&e.get().1); + } + Entry::Vacant(e) => { + e.insert((*start, *end)); + } + } + } + } + Err(e) => { + err = Some(e); + } + } + } + + let mut oplog = self.oplog.lock(); + oplog.batch_importing = false; + let pending = oplog.pending_changes.version_range(); + drop(oplog); + if !is_detached { + self._checkout_to_latest_with_guard(txn); + } else { + drop(txn); + } + + self.renew_txn_if_auto_commit(options); + if let Some(err) = err { + return Err(err); + } + + Ok(ImportStatus { + success, + pending: if pending.is_empty() { + None + } else { + Some(pending) + }, + }) + } + + /// Get shallow value of the document. + #[inline] + pub fn get_value(&self) -> LoroValue { + self.state.lock().get_value() + } + + /// Get deep value of the document. + #[inline] + pub fn get_deep_value(&self) -> LoroValue { + self.state.lock().get_deep_value() + } + + /// Get deep value of the document with container id + #[inline] + pub fn get_deep_value_with_id(&self) -> LoroValue { + self.state.lock().get_deep_value_with_id() + } + + pub fn checkout_to_latest(&self) { + let (options, _guard) = self.implicit_commit_then_stop(); + if !self.is_detached() { + drop(_guard); + self.renew_txn_if_auto_commit(options); + return; + } + + self._checkout_to_latest_without_commit(true) + .expect("checkout to oplog frontiers should succeed"); + self.emit_events(); + drop(_guard); + self.renew_txn_if_auto_commit(options); + } + + fn _checkout_to_latest_with_guard(&self, guard: LoroMutexGuard>) { + if !self.is_detached() { + self._renew_txn_if_auto_commit_with_guard(None, guard); + return; + } + + self._checkout_to_latest_without_commit(true) + .expect("checkout to oplog frontiers should succeed"); + self._renew_txn_if_auto_commit_with_guard(None, guard); + } + + /// NOTE: The caller of this method should ensure the txn is locked and set to None + pub(crate) fn _checkout_to_latest_without_commit( + &self, + to_commit_then_renew: bool, + ) -> LoroResult<()> { + tracing::info_span!("CheckoutToLatest", peer = self.peer_id()).in_scope(|| { + let f = self.oplog_frontiers(); + let this = &self; + let frontiers = &f; + this._checkout_without_emitting(frontiers, false, to_commit_then_renew)?; + // We don't need to shrink frontiers because oplog's frontiers are already shrinked. + this.emit_events(); + if this.config.detached_editing() { + this.renew_peer_id(); + } + + self.set_detached(false); + Ok(()) + }) + } + + /// Checkout [DocState] to a specific version. + /// + /// This will make the current [DocState] detached from the latest version of [OpLog]. + /// Any further import will not be reflected on the [DocState], until user call [LoroDoc::attach()] + pub fn checkout(&self, frontiers: &Frontiers) -> LoroResult<()> { + let was_detached = self.is_detached(); + let (options, guard) = self.implicit_commit_then_stop(); + let result = self._checkout_without_emitting(frontiers, true, true); + if result.is_ok() { + self.emit_events(); + } + drop(guard); + if self.config.detached_editing() { + if result.is_ok() { + self.renew_peer_id(); + } + self.renew_txn_if_auto_commit(options); + } else if result.is_err() { + if !was_detached { + self.renew_txn_if_auto_commit(options); + } + } else if !self.is_detached() { + self.renew_txn_if_auto_commit(options); + } + + result + } + + /// NOTE: The caller of this method should ensure the txn is locked and set to None + #[instrument(level = "info", skip(self))] + pub(crate) fn _checkout_without_emitting( + &self, + frontiers: &Frontiers, + to_shrink_frontiers: bool, + to_commit_then_renew: bool, + ) -> Result<(), LoroError> { + if !self.txn.is_locked() { + return Err(LoroError::TransactionError( + "checkout requires the transaction mutex to be held" + .to_string() + .into_boxed_str(), + )); + } + let from_frontiers = self.state_frontiers(); + loro_common::info!( + "checkout from={:?} to={:?} cur_vv={:?}", + from_frontiers, + frontiers, + self.oplog_vv() + ); + + if &from_frontiers == frontiers { + return Ok(()); + } + + let oplog = self.oplog.lock(); + if oplog.dag.is_before_shallow_root(frontiers) { + return Err(LoroError::SwitchToVersionBeforeShallowRoot); + } + + let frontiers = if to_shrink_frontiers { + shrink_frontiers(frontiers, &oplog.dag).map_err(LoroError::FrontiersNotFound)? + } else { + frontiers.clone() + }; + + if from_frontiers == frontiers { + return Ok(()); + } + + let mut state = self.state.lock(); + // §flowstate vendor patch #24 (act-twelve A12.2.1): route checkout + // diffs through the RETAINED import calculator instead of the + // separate checkout calculator. The import calculator's trackers are + // fed continuously (every import, plus the #21 gap feeds that cover + // local commits), so an undo/checkout between covered versions is a + // bounded tracker SWING + extraction — no rebuild. The dedicated + // checkout calculator started every deep undo COLD: `start_tracking` + // anchored the rebuild at the deep dominator and re-fed the ENTIRE + // history (measured 130s on a 64.5M-op-unit dirty-history tail undo; + // the "bimodal" good runs were the ones where an earlier slow-path + // undo happened to have paid that warm-up already). Sharing one + // calculator also makes undo feeds extend IMPORT coverage and vice + // versa. Cold behavior is unchanged (same rebuild law); lock order + // matches the import path (state before calc, P1.B). Kill switch: + // `FLOWSTATE_DISABLE_UNIFIED_CHECKOUT_CALC=1` restores the dedicated + // checkout calculator. + static UNIFIED_DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + let unified = !*UNIFIED_DISABLED + .get_or_init(|| std::env::var_os("FLOWSTATE_DISABLE_UNIFIED_CHECKOUT_CALC").is_some()); + let mut calc = if unified { + self.import_diff_calculator.lock() + } else { + self.diff_calculator.lock() + }; + for i in frontiers.iter() { + if !oplog.dag.contains(i) { + return Err(LoroError::FrontiersNotFound(i)); + } + } + + let before = oplog.dag.frontiers_to_vv(&state.frontiers).ok_or_else(|| { + LoroError::NotFoundError( + format!( + "Cannot find the current state version {:?}", + state.frontiers + ) + .into_boxed_str(), + ) + })?; + let Some(after) = &oplog.dag.frontiers_to_vv(&frontiers) else { + return Err(LoroError::NotFoundError( + format!("Cannot find the specified version {:?}", frontiers).into_boxed_str(), + )); + }; + + self.set_detached(true); + let (diff, diff_mode) = + calc.calc_diff_internal_tagged(&oplog, &before, &state.frontiers, after, &frontiers, None, "checkout"); + state.apply_diff( + InternalDocDiff { + origin: "checkout".into(), + diff: Cow::Owned(diff), + by: EventTriggerKind::Checkout, + new_version: Cow::Owned(frontiers.clone()), + }, + diff_mode, + )?; + + Ok(()) + } + + #[inline] + pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers { + self.oplog.lock().dag.vv_to_frontiers(vv) + } + + #[inline] + pub fn frontiers_to_vv(&self, frontiers: &Frontiers) -> Option { + self.oplog.lock().dag.frontiers_to_vv(frontiers) + } + + /// Import ops from other doc. + /// + /// After `a.merge(b)` and `b.merge(a)`, `a` and `b` will have the same content if they are in attached mode. + pub fn merge(&self, other: &Self) -> LoroResult { + let updates = other.export(ExportMode::updates(&self.oplog_vv())).unwrap(); + self.import(&updates) + } + + pub(crate) fn arena(&self) -> &SharedArena { + &self.arena + } + + #[inline] + pub fn len_ops(&self) -> usize { + if self.oplog.can_lock_in_this_thread() { + return self.oplog.lock().visible_op_count_exact(); + } + + self.visible_op_count.load(Acquire) + } + + #[inline] + pub fn len_changes(&self) -> usize { + let oplog = self.oplog.lock(); + oplog.len_changes() + } + + pub fn config(&self) -> &Configure { + &self.config + } + + /// This method compare the consistency between the current doc state + /// and the state calculated by diff calculator from beginning. + /// + /// Panic when it's not consistent + pub fn check_state_diff_calc_consistency_slow(&self) { + // #[cfg(any(test, debug_assertions, feature = "test_utils"))] + { + static IS_CHECKING: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + if IS_CHECKING.load(std::sync::atomic::Ordering::Acquire) { + return; + } + + IS_CHECKING.store(true, std::sync::atomic::Ordering::Release); + let peer_id = self.peer_id(); + let s = info_span!("CheckStateDiffCalcConsistencySlow", ?peer_id); + let _g = s.enter(); + let options = self.implicit_commit_then_stop().0; + self.oplog.lock().check_dag_correctness(); + if self.is_shallow() { + // For shallow documents, we cannot replay from the beginning as the history is not complete. + // + // Instead, we: + // 1. Export the initial state from the GC snapshot. + // 2. Create a new document and import the initial snapshot. + // 3. Export updates from the shallow start version vector to the current version. + // 4. Import these updates into the new document. + // 5. Compare the states of the new document and the current document. + + // Step 1: Export the initial state from the GC snapshot. + let initial_snapshot = self + .export(ExportMode::state_only(Some( + &self.shallow_since_frontiers(), + ))) + .unwrap(); + + // Step 2: Create a new document and import the initial snapshot. + let doc = LoroDoc::new(); + doc.import(&initial_snapshot).unwrap(); + self.checkout(&self.shallow_since_frontiers()).unwrap(); + assert_eq!(self.get_deep_value(), doc.get_deep_value()); + + // Step 3: Export updates since the shallow start version vector to the current version. + let updates = self.export(ExportMode::all_updates()).unwrap(); + + // Step 4: Import these updates into the new document. + doc.import(&updates).unwrap(); + self.checkout_to_latest(); + + // Step 5: Checkout to the current state's frontiers and compare the states. + // doc.checkout(&self.state_frontiers()).unwrap(); + assert_eq!(doc.get_deep_value(), self.get_deep_value()); + let mut calculated_state = doc.app_state().lock(); + let mut current_state = self.app_state().lock(); + current_state.check_is_the_same(&mut calculated_state); + } else { + let f = self.state_frontiers(); + let vv = self.oplog().lock().dag.frontiers_to_vv(&f).unwrap(); + let bytes = self.export(ExportMode::updates_till(&vv)).unwrap(); + let doc = Self::new(); + doc.import(&bytes).unwrap(); + let mut calculated_state = doc.app_state().lock(); + let mut current_state = self.app_state().lock(); + current_state.check_is_the_same(&mut calculated_state); + } + + self.renew_txn_if_auto_commit(options); + IS_CHECKING.store(false, std::sync::atomic::Ordering::Release); + } + } + + pub fn query_pos(&self, pos: &Cursor) -> Result { + self.query_pos_internal(pos, true) + } + + /// §perf (flowstate vendor patch): batch-resolve the current event-index + /// positions of many `ids` in `container`'s text state in a single pass over + /// the text chunks, instead of O(elements) per id via [`Self::query_pos`]. + /// `out[i]` is `None` when `ids[i]` is not currently live (deleted); callers + /// that need the history-traced resolution `query_pos` performs for deleted + /// ids should fall back to `query_pos` for just those ids. Non-text / missing + /// containers yield all-`None`. Each resolved entry equals the `current.pos` + /// that `query_pos` returns for a live cursor at that id. + pub fn query_text_id_positions(&self, container: &ContainerID, ids: &[ID]) -> Vec> { + if ids.is_empty() || !self.has_container(container) { + return vec![None; ids.len()]; + } + let mut state = self.state.lock(); + state.get_text_relative_positions(container, ids, true) + } + + /// Get position in a seq container + pub(crate) fn query_pos_internal( + &self, + pos: &Cursor, + ret_event_index: bool, + ) -> Result { + if !self.has_container(&pos.container) { + return Err(CannotFindRelativePosition::IdNotFound); + } + + let mut state = self.state.lock(); + if let Some(ans) = state.get_relative_position(pos, ret_event_index) { + Ok(PosQueryResult { + update: None, + current: AbsolutePosition { + pos: ans, + side: pos.side, + }, + }) + } else { + // We need to trace back to the version where the relative position is valid. + // The optimal way to find that version is to have succ info like Automerge. + // + // But we don't have that info now, so an alternative way is to trace back + // to version with frontiers of `[pos.id]`. But this may be very slow even if + // the target is just deleted a few versions ago. + // + // What we need is to trace back to the latest version that deletes the target + // id. + + // commit the txn to make sure we can query the history correctly, preserving options + drop(state); + let result = self.with_barrier(|| { + let oplog = self.oplog().lock(); + // TODO: assert pos.id is not unknown + if let Some(id) = pos.id { + // Ensure the container is registered if it exists lazily + if oplog.arena.id_to_idx(&pos.container).is_none() { + let mut s = self.state.lock(); + if !s.does_container_exist(&pos.container) { + return Err(CannotFindRelativePosition::ContainerDeleted); + } + s.ensure_container(&pos.container); + drop(s); + } + let idx = oplog.arena.id_to_idx(&pos.container).unwrap(); + // We know where the target id is when we trace back to the delete_op_id. + let Some(delete_op_id) = find_last_delete_op(&oplog, id, idx) else { + if oplog.shallow_since_vv().includes_id(id) { + return Err(CannotFindRelativePosition::HistoryCleared); + } + + tracing::error!("Cannot find id {}", id); + return Err(CannotFindRelativePosition::IdNotFound); + }; + // Should use persist mode so that it will force all the diff calculators to use the `checkout` mode + let mut diff_calc = DiffCalculator::new(true); + let before_frontiers: Frontiers = oplog.dag.find_deps_of_id(delete_op_id); + let before = &oplog.dag.frontiers_to_vv(&before_frontiers).unwrap(); + // TODO: PERF: it doesn't need to calc the effects here + diff_calc.calc_diff_internal_tagged( + &oplog, + before, + &before_frontiers, + oplog.vv(), + oplog.frontiers(), + Some(&|target| idx == target), + "cursor-deleted-pos", + ); + // TODO: remove depth info + let depth = self.arena.get_depth(idx); + let (_, diff_calc) = &mut diff_calc.get_or_create_calc(idx, depth); + match diff_calc { + crate::diff_calc::ContainerDiffCalculator::Richtext(text) => { + let c = text.get_id_latest_pos(id).unwrap(); + let new_pos = c.pos; + let handler = self.get_text(&pos.container); + let current_pos = handler.convert_entity_index_to_event_index(new_pos); + Ok(PosQueryResult { + update: handler.get_cursor(current_pos, c.side), + current: AbsolutePosition { + pos: current_pos, + side: c.side, + }, + }) + } + crate::diff_calc::ContainerDiffCalculator::List(list) => { + let c = list.get_id_latest_pos(id).unwrap(); + let new_pos = c.pos; + let handler = self.get_list(&pos.container); + Ok(PosQueryResult { + update: handler.get_cursor(new_pos, c.side), + current: AbsolutePosition { + pos: new_pos, + side: c.side, + }, + }) + } + crate::diff_calc::ContainerDiffCalculator::MovableList(list) => { + let c = list.get_id_latest_pos(id).unwrap(); + let new_pos = c.pos; + let handler = self.get_movable_list(&pos.container); + let new_pos = handler.op_pos_to_user_pos(new_pos); + Ok(PosQueryResult { + update: handler.get_cursor(new_pos, c.side), + current: AbsolutePosition { + pos: new_pos, + side: c.side, + }, + }) + } + crate::diff_calc::ContainerDiffCalculator::Tree(_) => unreachable!(), + crate::diff_calc::ContainerDiffCalculator::Map(_) => unreachable!(), + #[cfg(feature = "counter")] + crate::diff_calc::ContainerDiffCalculator::Counter(_) => unreachable!(), + crate::diff_calc::ContainerDiffCalculator::Unknown(_) => unreachable!(), + } + } else { + match pos.container.container_type() { + ContainerType::Text => { + let text = self.get_text(&pos.container); + Ok(PosQueryResult { + update: Some(Cursor { + id: None, + container: text.id(), + side: pos.side, + origin_pos: text.len_unicode(), + }), + current: AbsolutePosition { + pos: text.len_event(), + side: pos.side, + }, + }) + } + ContainerType::List => { + let list = self.get_list(&pos.container); + Ok(PosQueryResult { + update: Some(Cursor { + id: None, + container: list.id(), + side: pos.side, + origin_pos: list.len(), + }), + current: AbsolutePosition { + pos: list.len(), + side: pos.side, + }, + }) + } + ContainerType::MovableList => { + let list = self.get_movable_list(&pos.container); + Ok(PosQueryResult { + update: Some(Cursor { + id: None, + container: list.id(), + side: pos.side, + origin_pos: list.len(), + }), + current: AbsolutePosition { + pos: list.len(), + side: pos.side, + }, + }) + } + ContainerType::Map | ContainerType::Tree | ContainerType::Unknown(_) => { + unreachable!() + } + #[cfg(feature = "counter")] + ContainerType::Counter => unreachable!(), + } + } + }); + result + } + } + + /// Free the history cache that is used for making checkout faster. + /// + /// If you use checkout that switching to an old/concurrent version, the history cache will be built. + /// You can free it by calling this method. + pub fn free_history_cache(&self) { + self.oplog.lock().free_history_cache(); + } + + /// Free the cached diff calculator that is used for checkout. + pub fn free_diff_calculator(&self) { + *self.diff_calculator.lock() = DiffCalculator::new(true); + } + + /// §act-five P1.B: free the RETAINED import-path calculator (see lib.rs). The + /// receiving peer calls this on idle (not per import) so a burst/drip of + /// imports reuses the trackers, but a truly quiescent doc reclaims the memory. + pub fn free_import_diff_calculator(&self) { + *self.import_diff_calculator.lock() = DiffCalculator::new(true); + } + + /// §perf-heaven T8.18 (first cold import): eagerly build the RETAINED import + /// calculator's per-container trackers up to the current oplog frontier, so + /// the FIRST remote import of a collaboration session reuses the built + /// `id_to_cursor` index instead of constructing it cold on the + /// latency-critical receive path (the "first cold import tracker" the P1.B + /// retained calculator does not cover — it only helps the SECOND import + /// onward). + /// + /// This is a pure precomputation: the computed diff is discarded, and it is + /// SAFE by construction. `RichtextDiffCalculator::start_tracking` re-validates + /// each tracker against the requested version on every real import and + /// rebuilds it from scratch if it does not cover that version — so a warmed + /// tracker can only ever save the cold build, never change a computed diff. + /// If a later import diverges from the warmed frontier the guard simply + /// rebuilds, costing exactly what the un-warmed path would have. + /// + /// Callers should invoke this once per session while idle (e.g. just after a + /// collaboration channel attaches, before the peer starts receiving remote + /// updates). A no-op on a detached doc or an empty oplog. + pub fn warm_import_diff_calculator(&self) { + if self.is_detached() { + return; + } + let oplog = self.oplog.lock(); + if oplog.vv().is_empty() { + return; + } + let empty_vv = VersionVector::default(); + let empty_frontiers = Frontiers::default(); + // Same acquisition order as the import path (OpLog < DocState < + // DiffCalculator): take state before the retained calculator even though + // the diff is discarded, so the lock order is identical and deadlock-free. + let _state = self.state.lock(); + let _ = self.import_diff_calculator.lock().calc_diff_internal_tagged( + &oplog, + &empty_vv, + &empty_frontiers, + oplog.vv(), + oplog.dag.get_frontiers(), + None, + "warm-import-calc", + ); + } + + /// If you use checkout that switching to an old/concurrent version, the history cache will be built. + /// You can free it by calling `free_history_cache`. + pub fn has_history_cache(&self) -> bool { + self.oplog.lock().has_history_cache() + } + + /// Encoded all ops and history cache to bytes and store them in the kv store. + /// + /// The parsed ops will be dropped + #[inline] + pub fn compact_change_store(&self) { + self.with_barrier(|| { + self.oplog.lock().compact_change_store(); + }); + } + + /// Analyze the container info of the doc + /// + /// This is used for development and debugging + #[inline] + pub fn analyze(&self) -> DocAnalysis { + DocAnalysis::analyze(self) + } + + /// Get the path from the root to the container + pub fn get_path_to_container(&self, id: &ContainerID) -> Option> { + let mut state = self.state.lock(); + if state.arena.id_to_idx(id).is_none() { + if id.is_mergeable() { + // Mergeable children can be logically active via the parent map marker + // before they have their own encoded state. Register only the arena edge; do not + // create container state or change `has_container` semantics. + state.arena.register_container(id); + } else if !state.does_container_exist(id) { + return None; + } else { + state.ensure_container(id); + } + } + let idx = state.arena.id_to_idx(id).unwrap(); + state.get_path(idx) + } + + #[instrument(skip(self))] + pub fn export(&self, mode: ExportMode) -> Result, LoroEncodeError> { + self.with_barrier(|| { + let ans = match mode { + ExportMode::Snapshot => export_fast_snapshot(self), + ExportMode::Updates { from } => export_fast_updates(self, &from), + ExportMode::UpdatesInRange { spans } => { + export_fast_updates_in_range(&self.oplog.lock(), spans.as_ref()) + } + ExportMode::ShallowSnapshot(f) => export_shallow_snapshot(self, &f)?, + ExportMode::StateOnly(f) => match f { + Some(f) => export_state_only_snapshot(self, &f)?, + None => export_state_only_snapshot(self, &self.oplog_frontiers())?, + }, + ExportMode::SnapshotAt { version } => export_snapshot_at(self, &version)?, + }; + Ok(ans) + }) + } + + /// The doc only contains the history since the shallow history start version vector. + /// + /// This is empty if the doc is not shallow. + /// + /// The ops included by the shallow history start version vector are not in the doc. + pub fn shallow_since_vv(&self) -> ImVersionVector { + self.oplog().lock().shallow_since_vv().clone() + } + + pub fn shallow_since_frontiers(&self) -> Frontiers { + self.oplog().lock().shallow_since_frontiers().clone() + } + + /// Check if the doc contains the full history. + pub fn is_shallow(&self) -> bool { + !self.oplog().lock().shallow_since_vv().is_empty() + } + + /// Get the number of operations in the pending transaction. + /// + /// The pending transaction is the one that is not committed yet. It will be committed + /// after calling `doc.commit()`, `doc.export(mode)` or `doc.checkout(version)`. + pub fn get_pending_txn_len(&self) -> usize { + if let Some(txn) = self.txn.lock().as_ref() { + txn.len() + } else { + 0 + } + } + + #[inline] + pub fn find_id_spans_between(&self, from: &Frontiers, to: &Frontiers) -> VersionVectorDiff { + self.oplog().lock().dag.find_path(from, to) + } + + /// Subscribe to the first commit from a peer. Operations performed on the `LoroDoc` within this callback + /// will be merged into the current commit. + /// + /// This is useful for managing the relationship between `PeerID` and user information. + /// For example, you could store user names in a `LoroMap` using `PeerID` as the key and the `UserID` as the value. + pub fn subscribe_first_commit_from_peer( + &self, + callback: FirstCommitFromPeerCallback, + ) -> Subscription { + let (s, enable) = self + .first_commit_from_peer_subs + .inner() + .insert((), callback); + enable(); + s + } + + /// Subscribe to the pre-commit event. + /// + /// The callback will be called when the changes are committed but not yet applied to the OpLog. + /// You can modify the commit message and timestamp in the callback by [`ChangeModifier`]. + pub fn subscribe_pre_commit(&self, callback: PreCommitCallback) -> Subscription { + let (s, enable) = self.pre_commit_subs.inner().insert((), callback); + enable(); + s + } +} + +fn pending_root_containers_to_materialize(oplog: &OpLog, changes: &[Change]) -> Vec { + let mut roots = FxHashSet::default(); + for change in changes { + if change.ctr_end() <= oplog.vv().get(&change.id.peer).copied().unwrap_or(0) { + continue; + } + + if oplog.dag.is_before_shallow_root(&change.deps) + || oplog + .dag + .get_change_lamport_from_deps(&change.deps) + .is_some() + { + continue; + } + + for op in change.ops.iter() { + let id = oplog + .arena + .get_container_id(op.container) + .expect("decoded op container should be registered"); + if id.is_root() { + roots.insert(id); + } + } + } + + roots.into_iter().collect() +} + +#[derive(Debug, thiserror::Error)] +pub enum ChangeTravelError { + #[error("Target id not found {0:?}")] + TargetIdNotFound(ID), + #[error("The shallow history of the doc doesn't include the target version")] + TargetVersionNotIncluded, +} + +impl LoroDoc { + pub fn travel_change_ancestors( + &self, + ids: &[ID], + f: &mut dyn FnMut(ChangeMeta) -> ControlFlow<()>, + ) -> Result<(), ChangeTravelError> { + let (options, guard) = self.implicit_commit_then_stop(); + drop(guard); + struct PendingNode(ChangeMeta); + impl PartialEq for PendingNode { + fn eq(&self, other: &Self) -> bool { + self.0.lamport_last() == other.0.lamport_last() && self.0.id.peer == other.0.id.peer + } + } + + impl Eq for PendingNode {} + impl PartialOrd for PendingNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for PendingNode { + fn cmp(&self, other: &Self) -> Ordering { + self.0 + .lamport_last() + .cmp(&other.0.lamport_last()) + .then_with(|| self.0.id.peer.cmp(&other.0.id.peer)) + } + } + + for id in ids { + let op_log = &self.oplog().lock(); + if !op_log.vv().includes_id(*id) { + return Err(ChangeTravelError::TargetIdNotFound(*id)); + } + if op_log.dag.shallow_since_vv().includes_id(*id) { + return Err(ChangeTravelError::TargetVersionNotIncluded); + } + } + + let mut visited = FxHashSet::default(); + let mut pending: BinaryHeap = BinaryHeap::new(); + for id in ids { + pending.push(PendingNode(ChangeMeta::from_change( + &self.oplog().lock().get_change_at(*id).unwrap(), + ))); + } + while let Some(PendingNode(node)) = pending.pop() { + let deps = node.deps.clone(); + if f(node).is_break() { + break; + } + + for dep in deps.iter() { + let Some(dep_node) = self.oplog().lock().get_change_at(dep) else { + continue; + }; + if visited.contains(&dep_node.id) { + continue; + } + + visited.insert(dep_node.id); + pending.push(PendingNode(ChangeMeta::from_change(&dep_node))); + } + } + + let ans = Ok(()); + self.renew_txn_if_auto_commit(options); + ans + } + + pub fn get_changed_containers_in(&self, id: ID, len: usize) -> FxHashSet { + self.with_barrier(|| { + let mut set = FxHashSet::default(); + let len = i64::try_from(len).unwrap_or(i64::MAX); + let start = i64::from(id.counter); + let end = start.saturating_add(len); + if end <= 0 { + return set; + } + + let start = start.max(0).min(i64::from(i32::MAX)); + let end = end.max(0).min(i64::from(i32::MAX)); + if start >= end { + return set; + } + + { + let oplog = self.oplog().lock(); + let span = IdSpan::new(id.peer, start as i32, end as i32); + for op in oplog.iter_ops(span) { + let id = oplog.arena.get_container_id(op.container()).unwrap(); + set.insert(id); + } + } + set + }) + } + + pub fn delete_root_container(&self, cid: ContainerID) { + if !cid.is_root() { + return; + } + + // Do not treat "not in arena" as non-existence; consult state/kv + if !self.has_container(&cid) { + return; + } + + let Some(h) = self.get_handler(cid.clone()) else { + return; + }; + + self.config + .deleted_root_containers + .lock() + .insert(cid.clone()); + if let Err(e) = h.clear() { + self.config.deleted_root_containers.lock().remove(&cid); + eprintln!("Failed to clear handler: {:?}", e); + } + } + + pub fn set_hide_empty_root_containers(&self, hide: bool) { + self.config + .hide_empty_root_containers + .store(hide, std::sync::atomic::Ordering::Relaxed); + } +} + +fn find_last_delete_op(oplog: &OpLog, id: ID, idx: ContainerIdx) -> Option { + // Any delete op that covers `id` must have observed it, so its peer's counter + // at delete time was > id.counter. start_vv (the vv at `id`) is therefore a + // valid lower bound: changes at or before start_vv[peer] predate `id` and can + // be skipped. We scan peer-by-peer rather than using the DAG-ordered + // iter_changes_causally_rev, which is O(total changes). + // + // We choose the matching delete op with the greatest (op_lamport, peer, counter) + // ordering. op_lamport is the Lamport of the specific op within the change + // (change.lamport + op offset), not just the change's starting Lamport, so + // concurrent deletes with equal change Lamports are broken deterministically. + let start_vv = oplog + .dag + .frontiers_to_vv(&id.into()) + .unwrap_or_else(|| oplog.shallow_since_vv().to_vv()); + + // (op_lamport, peer) gives a deterministic total order for concurrent deletes. + // A single peer cannot produce two ops with the same lamport, so peer suffices + // as a tie-breaker. + let mut best: Option<((loro_common::Lamport, loro_common::PeerID), ID)> = None; + + for change in oplog.iter_changes_peer_by_peer(&start_vv, oplog.vv()) { + let peer = change.peer(); + for op in change.ops.iter() { + if op.container != idx { + continue; + } + if let InnerContent::List(InnerListOp::Delete(d)) = &op.content { + if d.id_start.to_span(d.atom_len()).contains(id) { + debug_assert!(op.counter >= change.id().counter); + let op_lamport = + change.lamport + (op.counter - change.id().counter) as loro_common::Lamport; + let key = (op_lamport, peer); + if best.is_none_or(|(bk, _)| key > bk) { + best = Some((key, ID::new(peer, op.counter))); + } + } + } + } + } + + best.map(|(_, op_id)| op_id) +} + +#[derive(Debug)] +pub struct CommitWhenDrop<'a> { + doc: &'a LoroDoc, + default_options: CommitOptions, +} + +impl Drop for CommitWhenDrop<'_> { + fn drop(&mut self) { + { + let mut guard = self.doc.txn.lock(); + if let Some(txn) = guard.as_mut() { + txn.set_default_options(std::mem::take(&mut self.default_options)); + }; + } + + self.doc.commit_then_renew(); + } +} + +/// Options for configuring a commit operation. +#[derive(Debug, Clone)] +pub struct CommitOptions { + /// Origin identifier for the commit event, used to track the source of changes. + /// It doesn't persist. + pub origin: Option, + + /// Whether to immediately start a new transaction after committing. + /// Defaults to true. + pub immediate_renew: bool, + + /// Custom timestamp for the commit in seconds since Unix epoch. + /// If None, the current time will be used. + pub timestamp: Option, + + /// Optional commit message to attach to the changes. It will be persisted. + pub commit_msg: Option>, +} + +impl CommitOptions { + /// Creates a new CommitOptions with default values. + pub fn new() -> Self { + Self { + origin: None, + immediate_renew: true, + timestamp: None, + commit_msg: None, + } + } + + /// Sets the origin identifier for this commit. + pub fn origin(mut self, origin: &str) -> Self { + self.origin = Some(origin.into()); + self + } + + /// Sets whether to immediately start a new transaction after committing. + pub fn immediate_renew(mut self, immediate_renew: bool) -> Self { + self.immediate_renew = immediate_renew; + self + } + + /// Set the timestamp of the commit. + /// + /// The timestamp is the number of **seconds** that have elapsed since 00:00:00 UTC on January 1, 1970. + pub fn timestamp(mut self, timestamp: Timestamp) -> Self { + self.timestamp = Some(timestamp); + self + } + + /// Sets a commit message to be attached to the changes. + pub fn commit_msg(mut self, commit_msg: &str) -> Self { + self.commit_msg = Some(commit_msg.into()); + self + } + + /// Sets the origin identifier for this commit. + pub fn set_origin(&mut self, origin: Option<&str>) { + self.origin = origin.map(|x| x.into()) + } + + /// Sets the timestamp for this commit. + pub fn set_timestamp(&mut self, timestamp: Option) { + self.timestamp = timestamp; + } +} + +impl Default for CommitOptions { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod test { + use std::{ + panic::AssertUnwindSafe, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + }; + + use crate::{ + cursor::PosType, + encoding::json_schema::json::{JsonOpContent, JsonSchema, ListOp}, + encoding::{fast_snapshot::EMPTY_MARK, EncodeMode}, + loro::ExportMode, + version::{Frontiers, VersionVector}, + LoroDoc, ToJson, TreeParentId, + }; + use bytes::{BufMut, Bytes}; + use loro_common::ID; + use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; + + const XXH_SEED: u32 = u32::from_le_bytes(*b"LORO"); + + fn encode_import_blob(mode: EncodeMode, body: &[u8]) -> Vec { + let mut ans = Vec::new(); + ans.extend_from_slice(b"loro"); + ans.extend_from_slice(&[0; 16]); + ans.extend_from_slice(&mode.to_bytes()); + ans.extend_from_slice(body); + let checksum = xxhash_rust::xxh32::xxh32(&ans[20..], XXH_SEED); + ans[16..20].copy_from_slice(&checksum.to_le_bytes()); + ans + } + + fn encode_fast_snapshot_import(oplog_bytes: &[u8]) -> Vec { + let mut body = Vec::new(); + body.put_u32_le(oplog_bytes.len() as u32); + body.extend_from_slice(oplog_bytes); + body.put_u32_le(EMPTY_MARK.len() as u32); + body.extend_from_slice(EMPTY_MARK); + body.put_u32_le(0); + encode_import_blob(EncodeMode::FastSnapshot, &body) + } + + fn sstable_with_huge_meta_block_count() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"LORO"); + bytes.push(0); + bytes.put_u32_le(10_000_000); + bytes.put_u32_le(xxhash_rust::xxh32::xxh32(&[], XXH_SEED)); + bytes.put_u32_le(5); + bytes + } + + fn snapshot_oplog_with_malformed_block() -> Vec { + let peer = 1; + let id = ID::new(peer, 0); + let vv = VersionVector::from_iter([(peer, 1)]); + let frontiers = Frontiers::from_id(id); + let mut store = MemKvStore::new(MemKvConfig::default()); + store.set(b"vv", vv.encode().into()); + store.set(b"fr", frontiers.encode().into()); + store.set(&id.to_bytes(), Bytes::from_static(&[0])); + store.export_all().to_vec() + } + + fn make_json_import_stress_doc(peer: u64) -> LoroDoc { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(peer).unwrap(); + + let text = doc.get_text("text"); + let mut text_pos = 0; + for i in 0..32 { + let chunk = format!("segment-{i}-abcdefghijklmnopqrstuvwxyz;"); + text.insert_unicode(text_pos, &chunk).unwrap(); + text_pos += chunk.chars().count(); + } + + let list = doc.get_list("list"); + for i in 0..32 { + list.insert(i, format!("item-{i}")).unwrap(); + } + + let map = doc.get_map("map"); + for i in 0..32 { + let key = format!("key-{i}"); + map.insert(&key, format!("value-{i}")).unwrap(); + } + + let tree = doc.get_tree("tree"); + let mut parent = TreeParentId::Root; + for i in 0..16 { + let node = tree.create(parent).unwrap(); + let meta = tree.get_meta(node).unwrap(); + meta.insert("name", format!("node-{i}")).unwrap(); + meta.insert("payload", format!("payload-{i}-{}", "x".repeat(16))) + .unwrap(); + parent = TreeParentId::Node(node); + } + + doc + } + + fn make_json_list_update_with_four_ops(peer: u64) -> (LoroDoc, JsonSchema) { + let doc = LoroDoc::new(); + doc.set_peer_id(peer).unwrap(); + let map = doc.get_map("map"); + let list = doc.get_list("list"); + let text = doc.get_text("text"); + + let mut txn = doc.txn().unwrap(); + map.insert_with_txn(&mut txn, "prefix", "map-value".into()) + .unwrap(); + list.insert_with_txn(&mut txn, 0, "seed".into()).unwrap(); + text.insert_with_txn(&mut txn, 0, "text-value", PosType::Unicode) + .unwrap(); + list.insert_with_txn(&mut txn, 1, "tail".into()).unwrap(); + txn.commit().unwrap(); + + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false); + assert_eq!(json.changes.len(), 1); + assert_eq!(json.changes[0].ops.len(), 4); + (doc, json) + } + + fn move_last_list_insert_far_out_of_bounds(json: &mut JsonSchema) { + let last_change = json.changes.last_mut().unwrap(); + let last_op = last_change.ops.last_mut().unwrap(); + match &mut last_op.content { + JsonOpContent::List(ListOp::Insert { pos, .. }) => { + *pos = 1_000; + } + other => panic!("expected list insert op, got {other:?}"), + } + } + + #[test] + fn test_sync() { + fn is_send_sync(_v: T) {} + let loro = super::LoroDoc::new(); + is_send_sync(loro) + } + + #[test] + fn import_rejects_huge_sstable_meta_block_count_without_panic() { + let bytes = encode_fast_snapshot_import(&sstable_with_huge_meta_block_count()); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| LoroDoc::new().import(&bytes))); + assert!(result.is_ok(), "malformed import should not panic"); + assert!(result.unwrap().is_err()); + } + + #[test] + fn import_rejects_malformed_change_block_without_panic() { + let bytes = encode_fast_snapshot_import(&snapshot_oplog_with_malformed_block()); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| LoroDoc::new().import(&bytes))); + assert!(result.is_ok(), "malformed import should not panic"); + assert!(result.unwrap().is_err()); + } + + #[test] + fn failed_import_rolls_back_oplog_and_arena() { + let src = LoroDoc::new(); + src.set_peer_id(1).unwrap(); + let text = src.get_text("text"); + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "hello", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let update = src.export(ExportMode::all_updates()).unwrap(); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let state_before_import = dst.get_deep_value(); + let err = dst + .import_with(&update, "__loro_fail_import_state_apply".into()) + .unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + + dst.import(&update).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_incremental_import_restores_previous_change_store_block() { + let src = LoroDoc::new(); + src.set_peer_id(1).unwrap(); + let text = src.get_text("text"); + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "a", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let first_update = src.export(ExportMode::all_updates()).unwrap(); + let first_vv = src.oplog_vv(); + + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 1, "b", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let second_update = src.export(ExportMode::updates(&first_vv)).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&first_update).unwrap(); + let vv_before_import = dst.oplog_vv(); + let state_before_import = dst.get_deep_value(); + dst.import_with(&second_update, "__loro_fail_import_state_apply".into()) + .unwrap_err(); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + + dst.import(&second_update).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_import_json_updates_rolls_back_complex_empty_doc() { + let src = make_json_import_stress_doc(11); + let json = src.export_json_updates(&Default::default(), &src.oplog_vv(), false); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + for _ in 0..3 { + crate::state::fail_next_import_state_apply_for_test(); + let err = dst.import_json_updates(json.clone()).unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + } + + dst.import_json_updates(json).unwrap(); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn failed_incremental_import_json_updates_restores_previous_change_store_block() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(12).unwrap(); + let text = src.get_text("text"); + text.insert_unicode(0, "a").unwrap(); + let list = src.get_list("list"); + list.push("seed").unwrap(); + let map = src.get_map("map"); + map.insert("seed", "value").unwrap(); + let tree = src.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + tree.get_meta(root).unwrap().insert("name", "root").unwrap(); + + let first_vv = src.oplog_vv(); + let first_json = src.export_json_updates(&Default::default(), &first_vv, false); + + let mut text_pos = text.len_unicode(); + for i in 0..64 { + let chunk = format!("chunk-{i};"); + text.insert_unicode(text_pos, &chunk).unwrap(); + text_pos += chunk.chars().count(); + } + for i in 0..32 { + list.push(format!("after-{i}")).unwrap(); + let key = format!("after-{i}"); + map.insert(&key, format!("value-{i}")).unwrap(); + } + let child = tree.create(TreeParentId::Node(root)).unwrap(); + tree.get_meta(child) + .unwrap() + .insert("name", "child") + .unwrap(); + + let second_json = src.export_json_updates(&first_vv, &src.oplog_vv(), false); + + let dst = LoroDoc::new(); + dst.import_json_updates(first_json).unwrap(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + + for _ in 0..2 { + crate::state::fail_next_import_state_apply_for_test(); + let err = dst.import_json_updates(second_json.clone()).unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + } + + dst.import_json_updates(second_json).unwrap(); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn malformed_later_import_json_update_rolls_back_after_valid_prefix_enters_oplog() { + let peer = 13; + let (src, good_json) = make_json_list_update_with_four_ops(peer); + let mut bad_json = good_json.clone(); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + + let good_dst = LoroDoc::new(); + good_dst.import_json_updates(good_json.clone()).unwrap(); + assert_eq!(good_dst.get_deep_value(), src.get_deep_value()); + + let last_op_counter = good_json.changes[0].ops.last().unwrap().counter; + let prefix_vv = VersionVector::from_iter([(peer, last_op_counter)]); + let prefix_json = src.export_json_updates(&Default::default(), &prefix_vv, false); + assert_eq!( + prefix_json.changes[0].ops.len(), + good_json.changes[0].ops.len() - 1 + ); + let good_suffix_json = src.export_json_updates(&prefix_vv, &src.oplog_vv(), false); + assert_eq!(good_suffix_json.changes[0].ops.len(), 1); + let mut bad_suffix_json = good_suffix_json.clone(); + move_last_list_insert_far_out_of_bounds(&mut bad_suffix_json); + + let prefix_dst = LoroDoc::new(); + prefix_dst.import_json_updates(prefix_json.clone()).unwrap(); + let vv_before_bad_suffix = prefix_dst.oplog_vv(); + let frontiers_before_bad_suffix = prefix_dst.oplog_frontiers(); + let state_before_bad_suffix = prefix_dst.get_deep_value(); + + let bad_suffix_json = serde_json::to_string(&bad_suffix_json).unwrap(); + let err = prefix_dst + .import_json_updates(&bad_suffix_json) + .unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(prefix_dst.oplog_vv(), vv_before_bad_suffix); + assert_eq!(prefix_dst.oplog_frontiers(), frontiers_before_bad_suffix); + assert_eq!(prefix_dst.get_deep_value(), state_before_bad_suffix); + + prefix_dst.import_json_updates(good_suffix_json).unwrap(); + assert_eq!(prefix_dst.get_deep_value(), src.get_deep_value()); + assert_eq!(prefix_dst.oplog_vv(), src.oplog_vv()); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + let bad_json = serde_json::to_string(&bad_json).unwrap(); + let err = dst.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + } + + #[test] + fn failed_import_restores_pending_changes_that_were_applied_during_import() { + let src = LoroDoc::new(); + src.set_peer_id(14).unwrap(); + let text = src.get_text("text"); + + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "a", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let first_update = src.export(ExportMode::all_updates()).unwrap(); + let first_vv = src.oplog_vv(); + + let mut txn = src.txn().unwrap(); + text.insert_with_txn(&mut txn, 1, "b", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let second_update = src.export(ExportMode::updates(&first_vv)).unwrap(); + + let dst = LoroDoc::new(); + let status = dst.import(&second_update).unwrap(); + assert!(status.success.is_empty()); + assert!(status.pending.is_some()); + let vv_before_dependency = dst.oplog_vv(); + let frontiers_before_dependency = dst.oplog_frontiers(); + let state_before_dependency = dst.get_deep_value(); + + crate::state::fail_next_import_state_apply_for_test(); + let err = dst.import(&first_update).unwrap_err(); + assert!(err.to_string().contains("state apply failpoint")); + assert_eq!(dst.oplog_vv(), vv_before_dependency); + assert_eq!(dst.oplog_frontiers(), frontiers_before_dependency); + assert_eq!(dst.get_deep_value(), state_before_dependency); + + dst.import(&first_update).unwrap(); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + // §perf-heaven T8.18: `warm_import_diff_calculator` convergence is validated + // by `flowstate-collab/tests/warm_import_calculator.rs` (a workspace member — + // this vendored crate cannot unit-test standalone: the isolated build lacks + // the `counter` feature + `dev_utils` dev-dependency the integrated build has). + + #[test] + fn failed_import_json_updates_does_not_emit_or_leave_events() { + let (src, good_json) = make_json_list_update_with_four_ops(15); + let mut bad_json = good_json.clone(); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + + let dst = LoroDoc::new(); + let event_count = Arc::new(AtomicUsize::new(0)); + let event_count_cloned = event_count.clone(); + let _sub = dst.subscribe_root(Arc::new(move |_| { + event_count_cloned.fetch_add(1, Ordering::SeqCst); + })); + + let bad_json = serde_json::to_string(&bad_json).unwrap(); + let err = dst.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(event_count.load(Ordering::SeqCst), 0); + assert!(dst.drop_pending_events().is_empty()); + assert!(dst.oplog().lock().is_empty()); + + dst.import_json_updates(good_json).unwrap(); + assert_eq!(event_count.load(Ordering::SeqCst), 1); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); + } + + #[test] + fn test_checkout() { + let loro = LoroDoc::new(); + loro.set_peer_id(1).unwrap(); + let text = loro.get_text("text"); + let map = loro.get_map("map"); + let list = loro.get_list("list"); + let mut txn = loro.txn().unwrap(); + for i in 0..10 { + map.insert_with_txn(&mut txn, "key", i.into()).unwrap(); + text.insert_with_txn(&mut txn, 0, &i.to_string(), PosType::Unicode) + .unwrap(); + list.insert_with_txn(&mut txn, 0, i.into()).unwrap(); + } + txn.commit().unwrap(); + let b = LoroDoc::new(); + b.import(&loro.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + loro.checkout(&Frontiers::default()).unwrap(); + { + let json = &loro.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"","list":[],"map":{}}) + ); + } + + b.checkout(&ID::new(1, 2).into()).unwrap(); + { + let json = &b.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"0","list":[0],"map":{"key":0}}) + ); + } + + loro.checkout(&ID::new(1, 3).into()).unwrap(); + { + let json = &loro.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"0","list":[0],"map":{"key":1}}) + ); + } + + b.checkout(&ID::new(1, 29).into()).unwrap(); + { + let json = &b.get_deep_value(); + assert_eq!( + json.to_json_value(), + serde_json::json!({"text":"9876543210","list":[9,8,7,6,5,4,3,2,1,0],"map":{"key":9}}) + ); + } + } + + #[test] + fn import_batch_err_181() { + let a = LoroDoc::new_auto_commit(); + let update_a = a.export(ExportMode::Snapshot); + let b = LoroDoc::new_auto_commit(); + b.import_batch(&[update_a.unwrap()]).unwrap(); + b.get_text("text") + .insert(0, "hello", PosType::Unicode) + .unwrap(); + b.commit_then_renew(); + let oplog = b.oplog().lock(); + drop(oplog); + b.export(ExportMode::all_updates()).unwrap(); + } + + #[test] + fn poisoned_mutex_keeps_follow_up_operations_failed() { + let doc = LoroDoc::new(); + let oplog = doc.oplog.clone(); + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _guard = oplog.lock(); + panic!("poison oplog"); + })); + + let err = std::panic::catch_unwind(AssertUnwindSafe(|| doc.oplog_vv())) + .expect_err("poisoned lock should continue to fail fast"); + let msg = if let Some(msg) = err.downcast_ref::<&str>() { + (*msg).to_string() + } else if let Some(msg) = err.downcast_ref::() { + msg.clone() + } else { + String::new() + }; + assert!(msg.contains("poisoned LoroMutex"), "{msg}"); + } +} + +/// §fidelity (flowstate vendor patch): restrict an undo diff's text RETAIN +/// attributes to the style keys the undone ops actually marked (per container). +/// +/// The checkout-based undo diff re-asserts the FULL resolved style set at every +/// affected position — including keys the undone op never touched (e.g. +/// `paragraph_style` at boundary sentinels when undoing a run-style highlight). +/// Applied to a state a CONCURRENT REMOTE peer has changed, that over-assertion +/// clobbers the peer's mark. Field bug (2026-07-08, two-peer hotpath): peer A's +/// local whole-doc highlight undo reverted peer B's concurrent whole-doc +/// paragraph-style tag on every paragraph. Keeping only the marked keys makes +/// the undo the exact inverse of the undone ops. Inserts (restored text and its +/// own styles) and containers with no undone mark ops pass through untouched. +fn restrict_undo_diff_to_marked_keys( + mut batch: crate::undo::DiffBatch, + keys_by_container: &FxHashMap>, +) -> crate::undo::DiffBatch { + use crate::event::{Diff, TextDiff}; + use generic_btree::rle::HasLength as _; + for (cid, diff) in batch.cid_to_events.iter_mut() { + let Some(keys) = keys_by_container.get(cid) else { + continue; + }; + let Diff::Text(text) = diff else { continue }; + let mut restricted: TextDiff = TextDiff::new(); + for item in text.iter() { + match item { + loro_delta::DeltaItem::Retain { len, attr } => { + let mut meta = attr.clone(); + meta.0.retain(|key, _| keys.contains(key)); + restricted.push_retain(*len, meta); + } + loro_delta::DeltaItem::Replace { value, attr, delete } => { + if *delete > 0 { + restricted.push_delete(*delete); + } + if value.rle_len() > 0 { + restricted.push_insert(value.clone(), attr.clone()); + } + } + } + } + *diff = Diff::Text(restricted); + } + batch +} + +/// Merge adjacent same-attribute insert runs inside every text diff of the +/// batch (see `undo_internal`). Non-text diffs pass through untouched. +fn compact_text_diff_batch(mut batch: crate::undo::DiffBatch) -> crate::undo::DiffBatch { + use crate::event::{Diff, TextDiff}; + use generic_btree::rle::HasLength as _; + for diff in batch.cid_to_events.values_mut() { + let Diff::Text(text) = diff else { continue }; + if text.iter().count() < 2 { + continue; + } + let mut compacted: TextDiff = TextDiff::new(); + let mut pending: Option<(String, crate::event::TextMeta)> = None; + fn flush(compacted: &mut TextDiff, pending: &mut Option<(String, crate::event::TextMeta)>) { + if let Some((text, meta)) = pending.take() { + compacted.push_insert(crate::utils::string_slice::StringSlice::from(text), meta); + } + } + for item in text.iter() { + match item { + loro_delta::DeltaItem::Retain { len, attr } => { + flush(&mut compacted, &mut pending); + compacted.push_retain(*len, attr.clone()); + } + loro_delta::DeltaItem::Replace { value, attr, delete } => { + if *delete > 0 { + flush(&mut compacted, &mut pending); + compacted.push_delete(*delete); + } + if value.rle_len() > 0 { + match &mut pending { + Some((buffer, meta)) if meta == attr => buffer.push_str(value.as_str()), + _ => { + flush(&mut compacted, &mut pending); + pending = Some((value.as_str().to_string(), attr.clone())); + } + } + } + } + } + } + flush(&mut compacted, &mut pending); + *diff = Diff::Text(compacted); + } + batch +} diff --git a/vendor/loro-internal/src/macros.rs b/vendor/loro-internal/src/macros.rs new file mode 100644 index 00000000..1fbd90af --- /dev/null +++ b/vendor/loro-internal/src/macros.rs @@ -0,0 +1,98 @@ +/// ```no_run +/// use rustc_hash::FxHashMap; +/// use loro_internal::fx_map; +/// +/// let mut expected = FxHashMap::default(); +/// expected.insert("test".to_string(), "test".to_string()); +/// expected.insert("test2".to_string(), "test2".to_string()); +/// let actual = fx_map!("test".into() => "test".into(), "test2".into() => "test2".into()); +/// assert_eq!(expected, actual); +/// ``` +#[macro_export] +macro_rules! fx_map { + ($($key:expr => $value:expr),*) => { + { + let mut m = rustc_hash::FxHashMap::default(); + $( + m.insert($key, $value); + )* + m + } + }; +} + +/// ```no_run +/// use loro_internal::vv; +/// +/// let v = vv!(1 => 2, 2 => 3); +/// assert_eq!(v.get(&1), Some(&2)); +/// assert_eq!(v.get(&2), Some(&3)); +/// ``` +#[macro_export] +macro_rules! vv { + ($($key:expr => $value:expr),*) => { + { + let mut m = $crate::version::VersionVector::default(); + $( + m.insert($key, $value); + )* + m + } + }; +} + +#[macro_export] +macro_rules! array_mut_ref { + ($arr:expr, [$a0:expr, $a1:expr]) => {{ + #[inline] + fn borrow_mut_ref(arr: &mut [T], a0: usize, a1: usize) -> (&mut T, &mut T) { + assert!(a0 != a1); + // SAFETY: this is safe because we know a0 != a1 + unsafe { + ( + &mut *(&mut arr[a0] as *mut _), + &mut *(&mut arr[a1] as *mut _), + ) + } + } + + borrow_mut_ref($arr, $a0, $a1) + }}; + ($arr:expr, [$a0:expr, $a1:expr, $a2:expr]) => {{ + #[inline] + fn borrow_mut_ref( + arr: &mut [T], + a0: usize, + a1: usize, + a2: usize, + ) -> (&mut T, &mut T, &mut T) { + assert!(a0 != a1 && a1 != a2 && a0 != a2); + // SAFETY: this is safe because we know there are not multiple mutable references to the same element + unsafe { + ( + &mut *(&mut arr[a0] as *mut _), + &mut *(&mut arr[a1] as *mut _), + &mut *(&mut arr[a2] as *mut _), + ) + } + } + + borrow_mut_ref($arr, $a0, $a1, $a2) + }}; +} + +#[cfg(test)] +mod test { + + #[test] + fn test_macro() { + let mut arr = vec![100, 101, 102, 103]; + let (a, b, _c) = array_mut_ref!(&mut arr, [1, 2, 3]); + assert_eq!(*a, 101); + assert_eq!(*b, 102); + *a = 50; + *b = 51; + assert!(arr[1] == 50); + assert!(arr[2] == 51); + } +} diff --git a/vendor/loro-internal/src/op.rs b/vendor/loro-internal/src/op.rs new file mode 100644 index 00000000..bbe8326f --- /dev/null +++ b/vendor/loro-internal/src/op.rs @@ -0,0 +1,646 @@ +use crate::{ + change::{Change, Lamport, Timestamp}, + container::{idx::ContainerIdx, ContainerID}, + estimated_size::EstimatedSize, + id::{Counter, PeerID, ID}, + oplog::BlockChangeRef, + span::{HasCounter, HasId, HasLamport}, +}; +use crate::{delta::DeltaValue, LoroValue}; +use either::Either; +use enum_as_inner::EnumAsInner; +use loro_common::{CompactIdLp, ContainerType, CounterSpan, IdFull, IdLp, IdSpan}; +use rle::{HasIndex, HasLength, Mergable, Sliceable}; +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, ops::Range}; + +mod content; +pub use content::*; + +/// Operation is a unit of change. +/// +/// A Op may have multiple atomic operations, since Op can be merged. +#[derive(Debug, Clone)] +pub struct Op { + pub(crate) counter: Counter, + pub(crate) container: ContainerIdx, + pub(crate) content: InnerContent, +} + +impl EstimatedSize for Op { + fn estimate_storage_size(&self) -> usize { + // TODO: use benchmark to get the optimal estimated size for each container type + self.content + .estimate_storage_size(self.container.get_type()) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct OpWithId { + pub peer: PeerID, + pub op: Op, +} + +impl OpWithId { + #[allow(unused)] + pub fn id_span(&self) -> IdSpan { + IdSpan::new( + self.peer, + self.op.counter, + self.op.counter + self.op.atom_len() as Counter, + ) + } +} + +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(feature = "wasm", derive(Serialize, Deserialize))] +pub struct RemoteOp<'a> { + pub(crate) counter: Counter, + pub(crate) container: ContainerID, + pub(crate) content: RawOpContent<'a>, +} + +/// This is used to propagate messages between inner module. +/// It's a temporary struct, and will be converted to Op when it's persisted. +#[derive(Debug, Clone)] +pub struct RawOp<'a> { + pub id: ID, + pub lamport: Lamport, + pub container: ContainerIdx, + pub content: RawOpContent<'a>, +} + +impl RawOp<'_> { + pub(crate) fn id_full(&self) -> loro_common::IdFull { + IdFull::new(self.id.peer, self.id.counter, self.lamport) + } + + pub(crate) fn idlp(&self) -> loro_common::IdLp { + IdLp::new(self.id.peer, self.lamport) + } +} + +/// RichOp includes lamport and timestamp info, which is used for conflict resolution. +#[derive(Debug, Clone)] +pub struct RichOp<'a> { + op: Cow<'a, Op>, + pub peer: PeerID, + lamport: Lamport, + pub timestamp: Timestamp, + pub start: usize, + pub end: usize, +} + +impl Op { + #[inline] + #[allow(unused)] + pub(crate) fn new(id: ID, content: InnerContent, container: ContainerIdx) -> Self { + Op { + counter: id.counter, + content, + container, + } + } + + /// If the estimated storage size of the content is greater than the given size, + /// return the length of the content that makes the estimated storage size equal to the given size. + /// Otherwise, return None. + pub(crate) fn check_whether_slice_content_to_fit_in_size(&self, size: usize) -> Option { + if self.estimate_storage_size() <= size { + return None; + } + + match &self.content { + InnerContent::List(l) => match l { + crate::container::list::list_op::InnerListOp::Insert { .. } => { + if matches!(self.container.get_type(), ContainerType::Text) { + Some(size.min(self.atom_len())) + } else { + Some((size / 4).min(self.atom_len())) + } + } + crate::container::list::list_op::InnerListOp::InsertText { .. } => { + Some(size.min(self.atom_len())) + } + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +impl RemoteOp<'_> { + #[allow(unused)] + pub(crate) fn into_static(self) -> RemoteOp<'static> { + RemoteOp { + counter: self.counter, + container: self.container, + content: self.content.to_static(), + } + } +} + +impl Mergable for Op { + fn is_mergable(&self, other: &Self, cfg: &()) -> bool { + self.counter + self.content_len() as Counter == other.counter + && self.container == other.container + && self.content.is_mergable(&other.content, cfg) + } + + fn merge(&mut self, other: &Self, cfg: &()) { + self.content.merge(&other.content, cfg) + } +} + +impl HasLength for Op { + fn content_len(&self) -> usize { + self.content.content_len() + } +} + +impl Sliceable for Op { + fn slice(&self, from: usize, to: usize) -> Self { + assert!(to > from, "{to} should be greater than {from}"); + assert!(to <= self.atom_len()); + let content: InnerContent = self.content.slice(from, to); + Op { + counter: (self.counter + from as Counter), + content, + container: self.container, + } + } +} + +impl Mergable for RemoteOp<'_> { + fn is_mergable(&self, _other: &Self, _cfg: &()) -> bool { + // don't merge remote op, because it's already merged. + false + } + + fn merge(&mut self, _other: &Self, _: &()) { + unreachable!() + } +} + +impl HasLength for RemoteOp<'_> { + fn content_len(&self) -> usize { + self.content.atom_len() + } +} + +impl HasIndex for Op { + type Int = Counter; + + fn get_start_index(&self) -> Self::Int { + self.counter + } +} + +impl HasIndex for RemoteOp<'_> { + type Int = Counter; + + fn get_start_index(&self) -> Self::Int { + self.counter + } +} + +impl HasCounter for Op { + fn ctr_start(&self) -> Counter { + self.counter + } +} + +impl HasCounter for RemoteOp<'_> { + fn ctr_start(&self) -> Counter { + self.counter + } +} + +impl HasId for RichOp<'_> { + fn id_start(&self) -> ID { + ID { + peer: self.peer, + counter: self.op.counter + self.start as Counter, + } + } +} + +impl HasLength for RichOp<'_> { + fn content_len(&self) -> usize { + self.end - self.start + } +} + +impl HasLamport for RichOp<'_> { + fn lamport(&self) -> Lamport { + self.lamport + self.start as Lamport + } +} + +impl<'a> RichOp<'a> { + pub fn new_by_change(change: &Change, op: &'a Op) -> Self { + let diff = op.counter - change.id.counter; + RichOp { + op: Cow::Borrowed(op), + peer: change.id.peer, + lamport: change.lamport + diff as Lamport, + timestamp: change.timestamp, + start: 0, + end: op.atom_len(), + } + } + + /// we want the overlap part of the op and change[start..end] + /// + /// op is contained in the change, but it's not necessary overlap with change[start..end] + pub fn new_by_slice_on_change(change: &Change, start: i32, end: i32, op: &'a Op) -> Self { + debug_assert!(end > start); + let op_index_in_change = op.counter - change.id.counter; + let op_slice_start = (start - op_index_in_change).clamp(0, op.atom_len() as i32); + let op_slice_end = (end - op_index_in_change).clamp(0, op.atom_len() as i32); + RichOp { + op: Cow::Borrowed(op), + peer: change.id.peer, + lamport: change.lamport + op_index_in_change as Lamport, + timestamp: change.timestamp, + start: op_slice_start as usize, + end: op_slice_end as usize, + } + } + + pub fn new_by_cnt_range(change: &Change, span: CounterSpan, op: &'a Op) -> Option { + let op_index_in_change = op.counter - change.id.counter; + let op_slice_start = (span.start - op.counter).clamp(0, op.atom_len() as i32); + let op_slice_end = (span.end - op.counter).clamp(0, op.atom_len() as i32); + if op_slice_start == op_slice_end { + return None; + } + Some(RichOp { + op: Cow::Borrowed(op), + peer: change.id.peer, + lamport: change.lamport + op_index_in_change as Lamport, + timestamp: change.timestamp, + start: op_slice_start as usize, + end: op_slice_end as usize, + }) + } + + pub(crate) fn new_iter_by_cnt_range( + change: BlockChangeRef, + span: CounterSpan, + ) -> RichOpBlockIter { + // Ops are counter-sorted within a change: start at the first op that + // can intersect `span` instead of scanning (and cloning) every op + // from index 0. A caller resolving many small spans inside one large + // change — e.g. the richtext diff calculator materializing an + // undo-restore of a whole imported document — was quadratic in the + // change size otherwise. + let op_index = change + .ops + .binary_search_by(|op| { + if op.counter + op.atom_len() as Counter <= span.start { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Greater + } + }) + .unwrap_or_else(|i| i); + RichOpBlockIter { + change, + span, + op_index, + } + } + + pub fn op(&self) -> Cow<'_, Op> { + if self.start == 0 && self.end == self.op.content_len() { + self.op.clone() + } else { + Cow::Owned(self.op.slice(self.start, self.end)) + } + } + + pub fn raw_op(&self) -> &Op { + &self.op + } + + pub fn client_id(&self) -> u64 { + self.peer + } + + pub fn container(&self) -> ContainerIdx { + self.op.container + } + + pub fn timestamp(&self) -> i64 { + self.timestamp + } + + pub fn start(&self) -> usize { + self.start + } + + pub fn end(&self) -> usize { + self.end + } + + pub fn counter(&self) -> Counter { + self.op.counter + self.start as Counter + } + + #[allow(unused)] + pub(crate) fn id(&self) -> ID { + ID { + peer: self.peer, + counter: self.op.counter + self.start as Counter, + } + } + + pub(crate) fn id_full(&self) -> IdFull { + IdFull::new( + self.peer, + self.op.counter + self.start as Counter, + self.lamport + self.start as Lamport, + ) + } + + pub fn idlp(&self) -> IdLp { + IdLp { + lamport: self.lamport + self.start as Lamport, + peer: self.peer, + } + } +} + +pub(crate) struct RichOpBlockIter { + change: BlockChangeRef, + span: CounterSpan, + op_index: usize, +} + +impl Iterator for RichOpBlockIter { + type Item = RichOp<'static>; + + fn next(&mut self) -> Option { + loop { + let op = self.change.ops.get(self.op_index)?; + if op.counter >= self.span.end { + // Counter-sorted: no later op can intersect the span. + return None; + } + let op_slice_start = (self.span.start - op.counter).clamp(0, op.atom_len() as i32); + let op_slice_end = (self.span.end - op.counter).clamp(0, op.atom_len() as i32); + if op_slice_start == op_slice_end { + self.op_index += 1; + continue; + } + // Clone only ops that are actually yielded (the old code cloned + // every skipped op too, and recursed per skip). + let op = op.clone(); + self.op_index += 1; + let op_offset_in_change = op.counter - self.change.id.counter; + return Some(RichOp { + op: Cow::Owned(op), + peer: self.change.id.peer, + lamport: self.change.lamport + op_offset_in_change as Lamport, + timestamp: self.change.timestamp, + start: op_slice_start as usize, + end: op_slice_end as usize, + }); + } + } +} + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(PartialEq, Debug, EnumAsInner, Clone, Serialize, Deserialize)] +pub enum ListSlice<'a> { + RawData(Cow<'a, [LoroValue]>), + RawStr { + str: Cow<'a, str>, + unicode_len: usize, + }, +} + +impl<'a> ListSlice<'a> { + pub fn from_borrowed_str(str: &'a str) -> Self { + Self::RawStr { + str: Cow::Borrowed(str), + unicode_len: str.chars().count(), + } + } +} + +#[repr(transparent)] +#[derive(PartialEq, Eq, Debug, Clone, Serialize)] +pub struct SliceRange(pub Range); + +const UNKNOWN_START: u32 = u32::MAX / 2; +impl SliceRange { + #[inline(always)] + pub fn is_unknown(&self) -> bool { + self.0.start == UNKNOWN_START + } + + #[inline(always)] + pub fn new_unknown(size: u32) -> Self { + Self(UNKNOWN_START..UNKNOWN_START + size) + } + + #[inline(always)] + pub fn new(range: Range) -> Self { + Self(range) + } + + #[inline(always)] + pub fn to_range(&self) -> Range { + self.0.start as usize..self.0.end as usize + } +} + +impl From> for SliceRange { + fn from(a: Range) -> Self { + SliceRange(a) + } +} + +impl HasLength for SliceRange { + fn content_len(&self) -> usize { + self.0.len() + } +} + +impl Sliceable for SliceRange { + fn slice(&self, from: usize, to: usize) -> Self { + if self.is_unknown() { + Self::new_unknown((to - from) as u32) + } else { + SliceRange(self.0.start + from as u32..self.0.start + to as u32) + } + } +} + +impl Mergable for SliceRange { + fn merge(&mut self, other: &Self, _: &()) { + if self.is_unknown() { + self.0.end += other.0.end - other.0.start; + } else { + self.0.end = other.0.end; + } + } + + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + (self.is_unknown() && other.is_unknown()) || self.0.end == other.0.start + } +} + +impl ListSlice<'_> { + #[inline(always)] + pub fn unknown_range(len: usize) -> SliceRange { + let start = UNKNOWN_START; + let end = len as u32 + UNKNOWN_START; + SliceRange(start..end) + } + + #[inline(always)] + pub fn is_unknown(range: &SliceRange) -> bool { + range.is_unknown() + } + + pub fn to_static(&self) -> ListSlice<'static> { + match self { + ListSlice::RawData(x) => ListSlice::RawData(Cow::Owned(x.to_vec())), + ListSlice::RawStr { str, unicode_len } => ListSlice::RawStr { + str: Cow::Owned(str.to_string()), + unicode_len: *unicode_len, + }, + } + } +} + +impl HasLength for ListSlice<'_> { + fn content_len(&self) -> usize { + match self { + ListSlice::RawStr { unicode_len, .. } => *unicode_len, + ListSlice::RawData(x) => x.len(), + } + } +} + +impl Sliceable for ListSlice<'_> { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + ListSlice::RawStr { + str, + unicode_len: _, + } => { + let ans = str.chars().skip(from).take(to - from).collect::(); + ListSlice::RawStr { + str: Cow::Owned(ans), + unicode_len: to - from, + } + } + ListSlice::RawData(x) => match x { + Cow::Borrowed(x) => ListSlice::RawData(Cow::Borrowed(&x[from..to])), + Cow::Owned(x) => ListSlice::RawData(Cow::Owned(x[from..to].into())), + }, + } + } +} + +impl Mergable for ListSlice<'_> { + fn is_mergable(&self, _other: &Self, _: &()) -> bool { + false + } +} + +#[derive(Debug, Clone)] +pub struct SliceWithId { + pub values: Either, + /// This field is no-none when diff calculating movable list that need to handle_unknown + pub elem_id: Option, + pub id: IdFull, +} + +impl DeltaValue for SliceWithId { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + if self.id.peer != other.id.peer { + return Err(other); + } + + if self.id.counter + self.length() as Counter != other.id.counter { + return Err(other); + } + + if self.id.lamport + self.length() as Lamport != other.id.lamport { + return Err(other); + } + + match (&mut self.values, &other.values) { + (Either::Left(left_range), Either::Left(right_range)) + if left_range.0.end == right_range.0.start => + { + left_range.0.end = right_range.0.end; + Ok(()) + } + _ => { + // If one is SliceRange and the other is LoroValue, we can't merge + Err(other) + } + } + } + + fn take(&mut self, target_len: usize) -> Self { + match &mut self.values { + Either::Left(range) => { + let ans = range.slice(0, target_len); + let this = range.slice(target_len, range.atom_len()); + *range = this; + let old_id = self.id; + self.id = self.id.inc(target_len as i32); + Self { + id: old_id, + values: Either::Left(ans), + elem_id: None, + } + } + Either::Right(_) => unimplemented!(), + } + } + + fn length(&self) -> usize { + match &self.values { + Either::Left(r) => r.atom_len(), + Either::Right(_) => 1, + } + } +} + +#[cfg(test)] +mod test { + use crate::LoroValue; + + use super::ListSlice; + + #[test] + fn fix_fields_order() { + let list_slice = vec![ + ListSlice::RawData(vec![LoroValue::Bool(true)].into()), + ListSlice::RawStr { + str: "".into(), + unicode_len: 0, + }, + ]; + let list_slice_buf = vec![2, 0, 1, 1, 1, 1, 0, 0]; + assert_eq!( + &postcard::to_allocvec(&list_slice).unwrap(), + &list_slice_buf + ); + assert_eq!( + postcard::from_bytes::>(&list_slice_buf).unwrap(), + list_slice + ); + } +} diff --git a/vendor/loro-internal/src/op/content.rs b/vendor/loro-internal/src/op/content.rs new file mode 100644 index 00000000..919b6534 --- /dev/null +++ b/vendor/loro-internal/src/op/content.rs @@ -0,0 +1,476 @@ +use std::sync::Arc; + +use enum_as_inner::EnumAsInner; +use loro_common::{ContainerID, ContainerType, LoroValue}; +use rle::{HasLength, Mergable, Sliceable}; +#[cfg(feature = "wasm")] +use serde::{Deserialize, Serialize}; + +use crate::{ + arena::SharedArena, + container::{ + list::list_op::{InnerListOp, ListOp}, + map::MapSet, + tree::tree_op::TreeOp, + }, + encoding::OwnedValue, +}; + +#[derive(EnumAsInner, Debug, Clone)] +pub enum InnerContent { + List(InnerListOp), + Map(MapSet), + Tree(Arc), + // The future content should not use any encoded arena context. + Future(FutureInnerContent), +} + +impl InnerContent { + pub fn visit_created_children(&self, arena: &SharedArena, f: &mut dyn FnMut(&ContainerID)) { + match self { + InnerContent::List(l) => match l { + InnerListOp::Insert { slice, .. } => { + for v in arena.iter_value_slice(slice.to_range()) { + if let LoroValue::Container(c) = v { + f(&c); + } + } + } + InnerListOp::Set { value, .. } => { + if let LoroValue::Container(c) = value { + f(c); + } + } + + InnerListOp::Move { .. } => {} + InnerListOp::InsertText { .. } => {} + InnerListOp::Delete(_) => {} + InnerListOp::StyleStart { .. } => {} + InnerListOp::StyleEnd => {} + }, + crate::op::InnerContent::Map(m) => { + if let Some(LoroValue::Container(c)) = &m.value { + f(c); + } + } + crate::op::InnerContent::Tree(t) => { + if let TreeOp::Create { target, .. } = t.as_ref() { + let id = target.associated_meta_container(); + f(&id); + } + } + crate::op::InnerContent::Future(f) => match &f { + #[cfg(feature = "counter")] + crate::op::FutureInnerContent::Counter(_) => {} + crate::op::FutureInnerContent::Unknown { .. } => {} + }, + } + } +} + +impl InnerContent { + pub fn estimate_storage_size(&self, kind: ContainerType) -> usize { + match self { + InnerContent::List(l) => l.estimate_storage_size(kind), + InnerContent::Map(_) => 3, + InnerContent::Tree(_) => 8, + InnerContent::Future(f) => f.estimate_storage_size(), + } + } +} + +#[derive(EnumAsInner, Debug, Clone)] +pub enum FutureInnerContent { + #[cfg(feature = "counter")] + Counter(f64), + Unknown { + prop: i32, + value: Box, + }, +} +impl FutureInnerContent { + fn estimate_storage_size(&self) -> usize { + match self { + #[cfg(feature = "counter")] + FutureInnerContent::Counter(_) => 4, + FutureInnerContent::Unknown { .. } => 6, + } + } +} + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. +#[derive(EnumAsInner, Debug, PartialEq)] +#[cfg_attr(feature = "wasm", derive(Serialize, Deserialize,))] +pub enum RawOpContent<'a> { + Map(MapSet), + List(ListOp<'a>), + Tree(Arc), + #[cfg(feature = "counter")] + Counter(f64), + Unknown { + prop: i32, + value: OwnedValue, + }, +} + +impl Clone for RawOpContent<'_> { + fn clone(&self) -> Self { + match self { + Self::Map(arg0) => Self::Map(arg0.clone()), + Self::List(arg0) => Self::List(arg0.clone()), + Self::Tree(arg0) => Self::Tree(arg0.clone()), + #[cfg(feature = "counter")] + Self::Counter(x) => Self::Counter(*x), + Self::Unknown { prop, value } => Self::Unknown { + prop: *prop, + value: value.clone(), + }, + } + } +} + +impl RawOpContent<'_> { + pub fn to_static(&self) -> RawOpContent<'static> { + match self { + Self::Map(arg0) => RawOpContent::Map(arg0.clone()), + Self::List(arg0) => match arg0 { + ListOp::Insert { slice, pos } => RawOpContent::List(ListOp::Insert { + slice: slice.to_static(), + pos: *pos, + }), + ListOp::Delete(x) => RawOpContent::List(ListOp::Delete(*x)), + ListOp::StyleStart { + start, + end, + key, + value, + info, + } => RawOpContent::List(ListOp::StyleStart { + start: *start, + end: *end, + key: key.clone(), + value: value.clone(), + info: *info, + }), + ListOp::StyleEnd => RawOpContent::List(ListOp::StyleEnd), + ListOp::Move { + from, + to, + elem_id: from_id, + } => RawOpContent::List(ListOp::Move { + from: *from, + to: *to, + elem_id: *from_id, + }), + ListOp::Set { elem_id, value } => RawOpContent::List(ListOp::Set { + elem_id: *elem_id, + value: value.clone(), + }), + }, + Self::Tree(arg0) => RawOpContent::Tree(arg0.clone()), + #[cfg(feature = "counter")] + Self::Counter(x) => RawOpContent::Counter(*x), + Self::Unknown { prop, value } => RawOpContent::Unknown { + prop: *prop, + value: value.clone(), + }, + } + } +} + +impl HasLength for RawOpContent<'_> { + fn content_len(&self) -> usize { + match self { + RawOpContent::Map(x) => x.content_len(), + RawOpContent::List(x) => x.content_len(), + RawOpContent::Tree(x) => x.content_len(), + #[cfg(feature = "counter")] + RawOpContent::Counter(_) => 1, + RawOpContent::Unknown { .. } => 1, + } + } +} + +impl Mergable for RawOpContent<'_> { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match (self, other) { + (RawOpContent::List(x), RawOpContent::List(y)) => x.is_mergable(y, &()), + (RawOpContent::Tree(x), RawOpContent::Tree(y)) => x.is_mergable(y, &()), + _ => false, + } + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + match self { + RawOpContent::List(x) => match _other { + RawOpContent::List(y) => x.merge(y, &()), + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +impl HasLength for InnerContent { + fn content_len(&self) -> usize { + match self { + InnerContent::List(list) => list.atom_len(), + InnerContent::Map(_) => 1, + InnerContent::Tree(_) => 1, + InnerContent::Future(_) => 1, + } + } +} + +impl Sliceable for InnerContent { + fn slice(&self, from: usize, to: usize) -> Self { + match self { + a @ InnerContent::Map(_) => { + assert!(from == 0 && to == 1); + a.clone() + } + a @ InnerContent::Tree(_) => { + assert!(from == 0 && to == 1); + a.clone() + } + InnerContent::List(x) => InnerContent::List(x.slice(from, to)), + InnerContent::Future(f) => { + assert!(from == 0 && to == 1); + InnerContent::Future(f.clone()) + } + } + } +} + +impl Mergable for InnerContent { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + match (self, other) { + (InnerContent::List(x), InnerContent::List(y)) => x.is_mergable(y, &()), + _ => false, + } + } + + fn merge(&mut self, _other: &Self, _conf: &()) + where + Self: Sized, + { + match self { + InnerContent::List(x) => match _other { + InnerContent::List(y) => x.merge(y, &()), + _ => unreachable!(), + }, + _ => unreachable!(), + } + } +} + +#[cfg(test)] +mod tests { + use std::{borrow::Cow, sync::Arc}; + + use fractional_index::FractionalIndex; + use loro_common::{ContainerID, ContainerType, TreeID}; + use rle::{HasLength, Mergable, Sliceable}; + + use super::*; + use crate::{ + container::{ + list::list_op::{DeleteSpanWithId, ListOp}, + map::MapSet, + tree::tree_op::TreeOp, + }, + op::{ListSlice, SliceRange}, + InternalString, LoroValue, ID, + }; + + #[test] + fn raw_op_content_reports_lengths_and_static_content_without_borrowing() { + let raw_insert = RawOpContent::List(ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned(vec![1_i64.into(), 2_i64.into()])), + pos: 3, + }); + assert_eq!(raw_insert.content_len(), 2); + match raw_insert.to_static() { + RawOpContent::List(ListOp::Insert { slice, pos }) => { + assert_eq!(pos, 3); + assert_eq!( + slice, + ListSlice::RawData(Cow::Owned(vec![1_i64.into(), 2_i64.into()])) + ); + } + other => panic!("expected list insert, got {other:?}"), + } + + let raw_map = RawOpContent::Map(MapSet { + key: InternalString::from("title"), + value: Some("draft".into()), + }); + assert_eq!(raw_map.content_len(), 1); + assert_eq!(raw_map.to_static(), raw_map); + + let raw_tree = RawOpContent::Tree(Arc::new(TreeOp::Delete { + target: TreeID::new(2, 9), + })); + assert_eq!(raw_tree.content_len(), 1); + assert_eq!(raw_tree.to_static(), raw_tree); + + let raw_unknown = RawOpContent::Unknown { + prop: 7, + value: OwnedValue::I64(11), + }; + assert_eq!(raw_unknown.content_len(), 1); + assert_eq!(raw_unknown.to_static(), raw_unknown); + + #[cfg(feature = "counter")] + { + let raw_counter = RawOpContent::Counter(1.25); + assert_eq!(raw_counter.content_len(), 1); + assert_eq!(raw_counter.to_static(), raw_counter); + } + } + + #[test] + fn raw_op_content_merges_only_mergeable_list_operations() { + let mut left = + RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new(ID::new(1, 0), 0, 1))); + let right = RawOpContent::List(ListOp::Delete(DeleteSpanWithId::new(ID::new(1, 1), 0, 1))); + assert!(left.is_mergable(&right, &())); + left.merge(&right, &()); + assert_eq!(left.content_len(), 2); + + let map = RawOpContent::Map(MapSet { + key: InternalString::from("k"), + value: Some(1_i64.into()), + }); + assert!(!left.is_mergable(&map, &())); + } + + #[test] + fn inner_content_lengths_storage_estimates_slices_and_merges_follow_inner_ops() { + let mut left = InnerContent::List(InnerListOp::Insert { + slice: SliceRange(0..2), + pos: 0, + }); + let right = InnerContent::List(InnerListOp::Insert { + slice: SliceRange(2..4), + pos: 2, + }); + assert_eq!(left.content_len(), 2); + assert_eq!(left.estimate_storage_size(ContainerType::List), 8); + assert!(left.is_mergable(&right, &())); + left.merge(&right, &()); + assert_eq!(left.content_len(), 4); + + let sliced = left.slice(1, 3); + match sliced { + InnerContent::List(InnerListOp::Insert { slice, pos }) => { + assert_eq!(slice, SliceRange(1..3)); + assert_eq!(pos, 1); + } + other => panic!("expected sliced list insert, got {other:?}"), + } + + let map = InnerContent::Map(MapSet { + key: InternalString::from("flag"), + value: Some(true.into()), + }); + assert_eq!(map.content_len(), 1); + assert_eq!(map.estimate_storage_size(ContainerType::Map), 3); + assert!(matches!(map.slice(0, 1), InnerContent::Map(_))); + + let tree = InnerContent::Tree(Arc::new(TreeOp::Delete { + target: TreeID::new(4, 5), + })); + assert_eq!(tree.content_len(), 1); + assert_eq!(tree.estimate_storage_size(ContainerType::Tree), 8); + assert!(matches!(tree.slice(0, 1), InnerContent::Tree(_))); + + let future = InnerContent::Future(FutureInnerContent::Unknown { + prop: 99, + value: Box::new(OwnedValue::Null), + }); + assert_eq!(future.content_len(), 1); + assert_eq!(future.estimate_storage_size(ContainerType::Unknown(3)), 6); + assert!(matches!(future.slice(0, 1), InnerContent::Future(_))); + + #[cfg(feature = "counter")] + { + let counter = InnerContent::Future(FutureInnerContent::Counter(2.0)); + assert_eq!(counter.estimate_storage_size(ContainerType::Counter), 4); + } + } + + #[test] + fn visit_created_children_reports_containers_created_by_list_map_and_tree_content() { + let arena = SharedArena::new(); + let list_child = ContainerID::new_normal(ID::new(1, 1), ContainerType::List); + let text_child = ContainerID::new_normal(ID::new(1, 2), ContainerType::Text); + let values = arena.alloc_values( + vec![ + LoroValue::Container(list_child.clone()), + "plain".into(), + LoroValue::Container(text_child.clone()), + ] + .into_iter(), + ); + let list = InnerContent::List(InnerListOp::Insert { + slice: SliceRange(values.start as u32..values.end as u32), + pos: 0, + }); + let mut children = Vec::new(); + list.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert_eq!(children, vec![list_child.clone(), text_child.clone()]); + + let map_child = ContainerID::new_normal(ID::new(2, 1), ContainerType::Map); + let map = InnerContent::Map(MapSet { + key: InternalString::from("child"), + value: Some(LoroValue::Container(map_child.clone())), + }); + let mut children = Vec::new(); + map.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert_eq!(children, vec![map_child]); + + let tree_target = TreeID::new(3, 7); + let tree = InnerContent::Tree(Arc::new(TreeOp::Create { + target: tree_target, + parent: None, + position: FractionalIndex::default(), + })); + let mut children = Vec::new(); + tree.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert_eq!(children, vec![tree_target.associated_meta_container()]); + + for tree in [ + TreeOp::Move { + target: tree_target, + parent: None, + position: FractionalIndex::default(), + }, + TreeOp::Delete { + target: tree_target, + }, + ] { + let tree = InnerContent::Tree(Arc::new(tree)); + let mut children = Vec::new(); + tree.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert!(children.is_empty()); + } + + let future = InnerContent::Future(FutureInnerContent::Unknown { + prop: 1, + value: Box::new(OwnedValue::False), + }); + let mut children = Vec::new(); + future.visit_created_children(&arena, &mut |id| children.push(id.clone())); + assert!(children.is_empty()); + } +} diff --git a/vendor/loro-internal/src/oplog.rs b/vendor/loro-internal/src/oplog.rs new file mode 100644 index 00000000..d776c484 --- /dev/null +++ b/vendor/loro-internal/src/oplog.rs @@ -0,0 +1,990 @@ +mod change_store; +pub(crate) mod loro_dag; +mod pending_changes; + +use crate::sync::{AtomicUsize, Mutex}; +use bytes::Bytes; +use std::borrow::Cow; +use std::cell::RefCell; +use std::cmp::Ordering; +use std::rc::Rc; +use std::sync::Arc; +use tracing::trace_span; + +use self::change_store::iter::MergedChangeIter; +use self::pending_changes::{PendingChanges, PendingChangesRollback}; +use super::arena::{SharedArena, SharedArenaRollback}; +use crate::change::{get_sys_timestamp, Change, Lamport, Timestamp}; +use crate::configure::Configure; +use crate::container::list::list_op; +use crate::dag::{Dag, DagUtils}; +use crate::diff_calc::DiffMode; +use crate::encoding::decode_oplog; +use crate::encoding::{ImportStatus, ParsedHeaderAndBody}; +use crate::history_cache::ContainerHistoryCache; +use crate::id::{Counter, PeerID, ID}; +use crate::op::{FutureInnerContent, ListSlice, RawOpContent, RemoteOp, RichOp}; +use crate::span::{HasCounterSpan, HasLamportSpan}; +use crate::version::{Frontiers, ImVersionVector, VersionVector}; +use crate::LoroError; +use change_store::{BlockOpRef, ChangeStoreRollback}; +use loro_common::{ContainerType, HasIdSpan, IdLp, IdSpan}; +use rle::{HasLength, RleVec, Sliceable}; +use smallvec::SmallVec; + +pub use self::loro_dag::{AppDag, AppDagNode, FrontiersNotIncluded}; +pub use change_store::{BlockChangeRef, ChangeStore}; + +/// [OpLog] store all the ops i.e. the history. +/// It allows multiple [AppState] to attach to it. +/// So you can derive different versions of the state from the [OpLog]. +/// It allows us to build a version control system. +/// +/// The causal graph should always be a DAG and complete. So we can always find the LCA. +/// If deps are missing, we can't import the change. It will be put into the `pending_changes`. +pub struct OpLog { + pub(crate) dag: AppDag, + pub(crate) arena: SharedArena, + visible_op_count: Arc, + change_store: ChangeStore, + history_cache: Mutex, + /// Pending changes that haven't been applied to the dag. + /// A change can be imported only when all its deps are already imported. + /// Key is the ID of the missing dep + pub(crate) pending_changes: PendingChanges, + /// Whether we are importing a batch of changes. + /// If so the Dag's frontiers won't be updated until the batch is finished. + pub(crate) batch_importing: bool, + pub(crate) configure: Configure, + /// The uncommitted change, it's a placeholder for the change + /// that is being edited in pre-commit callback. + pub(crate) uncommitted_change: Option, + pub(crate) import_rollback: Option, +} + +pub(crate) struct ImportRollback { + old_vv: VersionVector, + arena: SharedArenaRollback, + change_store: ChangeStoreRollback, + pending: PendingChangesRollback, +} + +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct ImportChangesPreflight { + pub applies_to_dag: bool, + pub has_deps_before_shallow_root: bool, + pub needs_state_apply_rollback: bool, +} + +impl std::fmt::Debug for OpLog { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OpLog") + .field("dag", &self.dag) + .field("pending_changes", &self.pending_changes) + .finish() + } +} + +impl OpLog { + #[inline] + pub(crate) fn new(visible_op_count: Arc) -> Self { + let arena = SharedArena::new(); + let cfg = Configure::default(); + let change_store = ChangeStore::new_mem(&arena, cfg.merge_interval_in_s.clone()); + Self { + visible_op_count, + history_cache: Mutex::new(ContainerHistoryCache::new(change_store.clone(), None)), + dag: AppDag::new(change_store.clone()), + change_store, + arena, + pending_changes: Default::default(), + batch_importing: false, + configure: cfg, + uncommitted_change: None, + import_rollback: None, + } + } + + #[inline] + fn calc_visible_op_count(&self) -> usize { + let total = self.dag.vv().values().sum::() as usize; + let shallow = self + .dag + .shallow_since_vv() + .iter() + .map(|(_, ops)| *ops) + .sum::() as usize; + total - shallow + } + + #[inline] + pub(crate) fn visible_op_count_exact(&self) -> usize { + self.calc_visible_op_count() + } + + #[inline] + pub(crate) fn refresh_visible_op_count(&self) -> usize { + let count = self.calc_visible_op_count(); + self.visible_op_count + .store(count, std::sync::atomic::Ordering::Release); + count + } + + #[inline] + pub fn dag(&self) -> &AppDag { + &self.dag + } + + pub fn change_store(&self) -> &ChangeStore { + &self.change_store + } + + /// Get the change with the given peer and lamport. + /// + /// If not found, return the change with the greatest lamport that is smaller than the given lamport. + pub fn get_change_with_lamport_lte( + &self, + peer: PeerID, + lamport: Lamport, + ) -> Option { + let ans = self + .change_store + .get_change_by_lamport_lte(IdLp::new(peer, lamport))?; + debug_assert!(ans.lamport <= lamport); + Some(ans) + } + + pub fn get_timestamp_of_version(&self, f: &Frontiers) -> Timestamp { + let mut timestamp = Timestamp::default(); + for id in f.iter() { + if let Some(change) = self.lookup_change(id) { + timestamp = timestamp.max(change.timestamp); + } + } + + timestamp + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.dag.is_empty() && self.arena.can_import_snapshot() + } + + /// This is the **only** place to update the `OpLog.changes` + pub(crate) fn insert_new_change(&mut self, change: Change, from_local: bool) { + let s = trace_span!( + "insert_new_change", + id = ?change.id, + lamport = change.lamport, + deps = ?change.deps + ); + let _enter = s.enter(); + let rollback_old_vv = self + .import_rollback + .as_ref() + .and_then(|x| (!x.old_vv.is_empty()).then_some(&x.old_vv)); + self.dag + .handle_new_change(&change, from_local, rollback_old_vv); + self.history_cache + .lock() + .insert_by_new_change(&change, true, true); + self.register_container_and_parent_link(&change); + if let Some(rollback) = self.import_rollback.as_mut() { + self.change_store.insert_change_with_rollback( + change, + true, + from_local, + &mut rollback.change_store, + ); + } else { + self.change_store.insert_change(change, true, from_local); + } + self.refresh_visible_op_count(); + } + + pub(crate) fn begin_import_rollback(&mut self) { + let arena = self.arena.checkpoint_for_rollback(); + self.begin_import_rollback_with_arena(arena); + } + + pub(crate) fn begin_import_rollback_with_arena(&mut self, arena: SharedArenaRollback) { + debug_assert!(self.import_rollback.is_none()); + let old_vv = self.vv().clone(); + self.dag.begin_import_rollback(); + self.import_rollback = Some(ImportRollback { + old_vv: old_vv.clone(), + arena, + change_store: ChangeStoreRollback::new(old_vv), + pending: Default::default(), + }); + } + + pub(crate) fn commit_import_rollback(&mut self) { + self.dag.commit_import_rollback(); + self.import_rollback = None; + } + + pub(crate) fn preflight_import_changes(&self, changes: &[Change]) -> ImportChangesPreflight { + let mut ans = ImportChangesPreflight::default(); + let pending_needs_state_apply_rollback = + self.pending_changes.has_state_apply_rollback_ops(); + for change in changes { + if change.ctr_end() <= self.vv().get(&change.id.peer).copied().unwrap_or(0) { + continue; + } + + if self.dag.is_before_shallow_root(&change.deps) { + ans.has_deps_before_shallow_root = true; + continue; + } + + if self + .dag + .get_change_lamport_from_deps(&change.deps) + .is_none() + { + continue; + } + + ans.applies_to_dag = true; + if change.ops.iter().any(|op| { + matches!( + op.container.get_type(), + ContainerType::List | ContainerType::Tree + ) + }) { + ans.needs_state_apply_rollback = true; + } + } + + // Any newly applied change can unlock pending changes whose ops are not + // visible in `changes`, so include pending in the rollback decision. + // Keep this narrow: text/map-only pending changes cannot return a + // state-apply error, and forcing rollback there adds lock traffic to + // small sync/import workloads. + if ans.applies_to_dag && pending_needs_state_apply_rollback { + ans.needs_state_apply_rollback = true; + } + + #[cfg(test)] + if ans.applies_to_dag { + ans.needs_state_apply_rollback = true; + } + + ans + } + + pub(crate) fn rollback_import(&mut self) { + let Some(rollback) = self.import_rollback.take() else { + return; + }; + + self.change_store.rollback_import(rollback.change_store); + self.dag.rollback_import(); + rollback.pending.rollback(&mut self.pending_changes); + self.history_cache.lock().free_all(); + self.arena.rollback(rollback.arena); + self.refresh_visible_op_count(); + } + + pub(crate) fn reset_to_empty_for_failed_snapshot_import( + &mut self, + arena_checkpoint: SharedArenaRollback, + ) { + let arena = self.arena.clone(); + let configure = self.configure.clone(); + arena.rollback(arena_checkpoint); + let change_store = ChangeStore::new_mem(&arena, configure.merge_interval_in_s.clone()); + self.history_cache = Mutex::new(ContainerHistoryCache::new(change_store.clone(), None)); + self.dag = AppDag::new(change_store.clone()); + self.change_store = change_store; + self.pending_changes = Default::default(); + self.batch_importing = false; + self.configure = configure; + self.uncommitted_change = None; + self.import_rollback = None; + self.visible_op_count + .store(0, std::sync::atomic::Ordering::Release); + } + + #[inline(always)] + pub(crate) fn with_history_cache(&self, f: F) -> R + where + F: FnOnce(&mut ContainerHistoryCache) -> R, + { + let mut history_cache = self.history_cache.lock(); + f(&mut history_cache) + } + + pub fn has_history_cache(&self) -> bool { + self.history_cache.lock().has_cache() + } + + pub fn free_history_cache(&self) { + let mut history_cache = self.history_cache.lock(); + history_cache.free(); + } + + #[cfg(test)] + #[allow(dead_code)] + pub(crate) fn pending_changes_len(&self) -> usize { + self.pending_changes.len() + } + + /// Import a change. + /// + /// Pending changes that haven't been applied to the dag. + /// A change can be imported only when all its deps are already imported. + /// Key is the ID of the missing dep + /// + /// # Err + /// + /// - Return Err(LoroError::UsedOpID) when the change's id is occupied + /// - Return Err(LoroError::DecodeError) when the change's deps are missing + pub(crate) fn import_local_change(&mut self, change: Change) -> Result<(), LoroError> { + self.insert_new_change(change, true); + Ok(()) + } + + /// Trim the known part of change + pub(crate) fn trim_the_known_part_of_change(&self, change: Change) -> Option { + let Some(&end) = self.dag.vv().get(&change.id.peer) else { + return Some(change); + }; + + if change.id.counter >= end { + return Some(change); + } + + if change.ctr_end() <= end { + return None; + } + + let offset = (end - change.id.counter) as usize; + Some(change.slice(offset, change.atom_len())) + } + + #[allow(unused)] + fn check_id_is_not_duplicated(&self, id: ID) -> Result<(), LoroError> { + let cur_end = self.dag.vv().get(&id.peer).cloned().unwrap_or(0); + if cur_end > id.counter { + return Err(LoroError::UsedOpID { id }); + } + + Ok(()) + } + + /// Ensure the new change is greater than the last peer's id and the counter is continuous. + /// + /// It can be false when users use detached editing mode and use a custom peer id. + // This method might be slow and can be optimized if needed in the future. + pub(crate) fn check_change_greater_than_last_peer_id( + &self, + peer: PeerID, + counter: Counter, + deps: &Frontiers, + ) -> Result<(), LoroError> { + if counter == 0 { + return Ok(()); + } + + if !self.configure.detached_editing() { + return Ok(()); + } + + let mut max_last_counter = -1; + for dep in deps.iter() { + let dep_vv = self + .dag + .get_vv(dep) + .ok_or(LoroError::FrontiersNotFound(dep))?; + max_last_counter = max_last_counter.max(dep_vv.get(&peer).cloned().unwrap_or(0) - 1); + } + + if counter != max_last_counter + 1 { + return Err(LoroError::ConcurrentOpsWithSamePeerID { + peer, + last_counter: max_last_counter, + current: counter, + }); + } + + Ok(()) + } + + pub(crate) fn next_id(&self, peer: PeerID) -> ID { + let cnt = self.dag.vv().get(&peer).copied().unwrap_or(0); + ID::new(peer, cnt) + } + + pub(crate) fn vv(&self) -> &VersionVector { + self.dag.vv() + } + + pub(crate) fn frontiers(&self) -> &Frontiers { + self.dag.frontiers() + } + + /// - Ordering::Less means self is less than target or parallel + /// - Ordering::Equal means versions equal + /// - Ordering::Greater means self's version is greater than target + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + self.dag.cmp_with_frontiers(other) + } + + /// Compare two [Frontiers] causally. + /// + /// If one of the [Frontiers] are not included, it will return [FrontiersNotIncluded]. + #[inline] + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + self.dag.cmp_frontiers(a, b) + } + + pub(crate) fn get_min_lamport_at(&self, id: ID) -> Lamport { + self.get_change_at(id).map(|c| c.lamport).unwrap_or(0) + } + + pub(crate) fn get_lamport_at(&self, id: ID) -> Option { + self.get_change_at(id) + .map(|c| c.lamport + (id.counter - c.id.counter) as Lamport) + } + + pub(crate) fn iter_ops(&self, id_span: IdSpan) -> impl Iterator> + '_ { + let change_iter = self.change_store.iter_changes(id_span); + change_iter.flat_map(move |c| RichOp::new_iter_by_cnt_range(c, id_span.counter)) + } + + pub(crate) fn get_max_lamport_at(&self, id: ID) -> Lamport { + self.get_change_at(id) + .map(|c| { + let change_counter = c.id.counter as u32; + c.lamport + c.ops().last().map(|op| op.counter).unwrap_or(0) as u32 - change_counter + }) + .unwrap_or(Lamport::MAX) + } + + pub fn get_change_at(&self, id: ID) -> Option { + self.change_store.get_change(id) + } + + pub(crate) fn set_uncommitted_change(&mut self, change: Change) { + self.uncommitted_change = Some(change); + } + + pub(crate) fn get_uncommitted_change_in_span( + &self, + id_span: IdSpan, + ) -> Option> { + self.uncommitted_change.as_ref().and_then(|c| { + if c.id_span() == id_span { + Some(Cow::Borrowed(c)) + } else if let Some((start, end)) = id_span.get_slice_range_on(&c.id_span()) { + Some(Cow::Owned(c.slice(start, end))) + } else { + None + } + }) + } + + pub fn get_deps_of(&self, id: ID) -> Option { + self.get_change_at(id).map(|c| { + if c.id.counter == id.counter { + c.deps.clone() + } else { + Frontiers::from_id(id.inc(-1)) + } + }) + } + + pub fn get_remote_change_at(&self, id: ID) -> Option>> { + let change = self.get_change_at(id)?; + Some(convert_change_to_remote(&self.arena, &change)) + } + + pub(crate) fn import_unknown_lamport_pending_changes( + &mut self, + remote_changes: Vec, + ) -> Result<(), LoroError> { + self.extend_pending_changes_with_unknown_lamport(remote_changes) + } + + /// lookup change by id. + /// + /// if id does not included in this oplog, return None + pub(crate) fn lookup_change(&self, id: ID) -> Option { + self.change_store.get_change(id) + } + + #[inline(always)] + pub(crate) fn export_change_store_from(&self, vv: &VersionVector, f: &Frontiers) -> Bytes { + self.change_store + .export_from(vv, f, self.vv(), self.frontiers()) + } + + #[inline(always)] + pub(crate) fn export_change_store_in_range( + &self, + vv: &VersionVector, + f: &Frontiers, + to_vv: &VersionVector, + to_frontiers: &Frontiers, + ) -> Bytes { + self.change_store.export_from(vv, f, to_vv, to_frontiers) + } + + #[inline(always)] + pub(crate) fn export_blocks_from(&self, vv: &VersionVector, w: &mut W) { + self.change_store + .export_blocks_from(vv, self.shallow_since_vv(), self.vv(), w) + } + + #[inline(always)] + pub(crate) fn export_blocks_in_range(&self, spans: &[IdSpan], w: &mut W) { + self.change_store.export_blocks_in_range(spans, w) + } + + pub(crate) fn fork_changes_up_to(&self, frontiers: &Frontiers) -> Option { + let vv = self.dag.frontiers_to_vv(frontiers)?; + Some( + self.change_store + .fork_changes_up_to(self.dag.shallow_since_vv(), frontiers, &vv), + ) + } + + #[inline(always)] + pub(crate) fn decode(&mut self, data: ParsedHeaderAndBody) -> Result { + decode_oplog(self, data) + } + + /// iterates over all changes between LCA(common ancestors) to the merged version of (`from` and `to`) causally + /// + /// Tht iterator will include a version vector when the change is applied + /// + /// returns: (common_ancestor_vv, iterator) + /// + /// Note: the change returned by the iterator may include redundant ops at the beginning, you should trim it by yourself. + /// You can trim it by the provided counter value. It should start with the counter. + /// + /// If frontiers are provided, it will be faster (because we don't need to calculate it from version vector + /// §flowstate vendor patch #21: `cap` bounds the iteration to `lca..cap` + /// instead of `lca..(from ∪ to)` — the diff calculator's gap pre-feed + /// walks only the common prefix up to the meet (`cap ⊇ lca` is the + /// caller's obligation; the meet always dominates the common ancestor). + #[allow(clippy::type_complexity)] + pub(crate) fn iter_from_lca_causally( + &self, + from: &VersionVector, + from_frontiers: &Frontiers, + to: &VersionVector, + to_frontiers: &Frontiers, + cap: Option<&VersionVector>, + ) -> ( + VersionVector, + DiffMode, + impl Iterator< + Item = ( + BlockChangeRef, + (Counter, Counter), + Rc>, + ), + > + '_, + ) { + self.iter_from_lca_causally_floored(from, from_frontiers, to, to_frontiers, cap, None) + } + + /// §flowstate vendor patch #24b (act-twelve A12.2.1): like + /// [`Self::iter_from_lca_causally`], but when `floor` is given the + /// ENUMERATION starts at that causally-closed cut instead of the + /// dominator LCA — the returned lca/mode are still computed from the true + /// common ancestor. Sound whenever the caller would SKIP every change at + /// or below the floor anyway (the #21 meet clamp): the yielded set of + /// above-floor changes is identical, calculators that turn out to need + /// deeper history still declare `needs_full` and get the unclamped + /// phase-2 walk, and nothing else observes the enumeration. On a + /// dirty-history undo checkout the dominator walk enumerated 64.5M + /// op-units to skip nearly all of them (~0.5s per checkout, ×2 per undo). + pub(crate) fn iter_from_lca_causally_floored( + &self, + from: &VersionVector, + from_frontiers: &Frontiers, + to: &VersionVector, + to_frontiers: &Frontiers, + cap: Option<&VersionVector>, + floor: Option<&VersionVector>, + ) -> ( + VersionVector, + DiffMode, + impl Iterator< + Item = ( + BlockChangeRef, + (Counter, Counter), + Rc>, + ), + > + '_, + ) { + let mut merged_vv = from.clone(); + merged_vv.merge(to); + if let Some(cap) = cap { + merged_vv = cap.clone(); + } + loro_common::debug!("to_frontiers={:?} vv={:?}", &to_frontiers, to); + let (common_ancestors, mut diff_mode) = + self.dag.find_common_ancestor(from_frontiers, to_frontiers); + if diff_mode == DiffMode::Checkout && to > from { + diff_mode = DiffMode::Import; + } + + let common_ancestors_vv = self.dag.frontiers_to_vv(&common_ancestors).unwrap(); + let (walk_start, walk_start_vv) = match floor { + Some(floor) if floor.includes_vv(&common_ancestors_vv) => { + (self.dag.vv_to_frontiers(floor), floor.clone()) + } + _ => (common_ancestors.clone(), common_ancestors_vv.clone()), + }; + // go from the walk start (lca, or the higher floor) to merged_vv + let diff = walk_start_vv.diff(&merged_vv).forward; + let mut iter = self.dag.iter_causal(walk_start, diff); + let mut node = iter.next(); + let mut cur_cnt = 0; + let vv = Rc::new(RefCell::new(VersionVector::default())); + ( + common_ancestors_vv.clone(), + diff_mode, + std::iter::from_fn(move || { + if let Some(inner) = &node { + let mut inner_vv = vv.borrow_mut(); + // FIXME: PERF: it looks slow for large vv, like 10000+ entries + inner_vv.clear(); + self.dag.ensure_vv_for(&inner.data); + inner_vv.extend_to_include_vv(inner.data.vv.get().unwrap().iter()); + let peer = inner.data.peer; + let cnt = inner + .data + .cnt + .max(cur_cnt) + .max(common_ancestors_vv.get(&peer).copied().unwrap_or(0)); + let dag_node_end = (inner.data.cnt + inner.data.len as Counter) + .min(merged_vv.get(&peer).copied().unwrap_or(0)); + let change = self.change_store.get_change(ID::new(peer, cnt)).unwrap(); + + if change.ctr_end() < dag_node_end { + cur_cnt = change.ctr_end(); + } else { + node = iter.next(); + cur_cnt = 0; + } + + inner_vv.extend_to_include_end_id(change.id); + + Some((change, (cnt, dag_node_end), vv.clone())) + } else { + None + } + }), + ) + } + + pub fn len_changes(&self) -> usize { + self.change_store.change_num() + } + + pub fn diagnose_size(&self) -> SizeInfo { + let mut total_changes = 0; + let mut total_ops = 0; + let mut total_atom_ops = 0; + let total_dag_node = self.dag.total_parsed_dag_node(); + self.change_store.visit_all_changes(&mut |change| { + total_changes += 1; + total_ops += change.ops.len(); + total_atom_ops += change.atom_len(); + }); + + println!("total changes: {}", total_changes); + println!("total ops: {}", total_ops); + println!("total atom ops: {}", total_atom_ops); + println!("total dag node: {}", total_dag_node); + SizeInfo { + total_changes, + total_ops, + total_atom_ops, + total_dag_node, + } + } + + pub(crate) fn iter_changes_peer_by_peer<'a>( + &'a self, + from: &VersionVector, + to: &VersionVector, + ) -> impl Iterator + 'a { + let spans: Vec<_> = from.diff_iter(to).1.collect(); + spans + .into_iter() + .flat_map(move |span| self.change_store.iter_changes(span)) + } + + #[allow(dead_code)] + pub(crate) fn iter_changes_causally_rev<'a>( + &'a self, + from: &VersionVector, + to: &VersionVector, + ) -> impl Iterator + 'a { + MergedChangeIter::new_change_iter_rev(self, from, to) + } + + pub fn get_timestamp_for_next_txn(&self) -> Timestamp { + if self.configure.record_timestamp() { + get_timestamp_now_txn() + } else { + 0 + } + } + + #[inline(never)] + pub(crate) fn idlp_to_id(&self, id: loro_common::IdLp) -> Option { + let change = self.change_store.get_change_by_lamport_lte(id)?; + + if change.lamport > id.lamport || change.lamport_end() <= id.lamport { + return None; + } + + Some(ID::new( + change.id.peer, + (id.lamport - change.lamport) as Counter + change.id.counter, + )) + } + + #[allow(unused)] + pub(crate) fn id_to_idlp(&self, id_start: ID) -> IdLp { + let change = self.get_change_at(id_start).unwrap(); + let lamport = change.lamport + (id_start.counter - change.id.counter) as Lamport; + let peer = id_start.peer; + loro_common::IdLp { peer, lamport } + } + + /// NOTE: This may return a op that includes the given id, not necessarily start with the given id + pub(crate) fn get_op_that_includes(&self, id: ID) -> Option { + let change = self.get_change_at(id)?; + change.get_op_with_counter(id.counter) + } + + pub(crate) fn split_span_based_on_deps(&self, id_span: IdSpan) -> Vec<(IdSpan, Frontiers)> { + let peer = id_span.peer; + let mut counter = id_span.counter.min(); + let span_end = id_span.counter.norm_end(); + let mut ans = Vec::new(); + + while counter < span_end { + let id = ID::new(peer, counter); + let node = self.dag.get(id).unwrap(); + + let f = if node.cnt == counter { + node.deps.clone() + } else if counter > 0 { + id.inc(-1).into() + } else { + unreachable!() + }; + + let cur_end = node.cnt + node.len as Counter; + let len = cur_end.min(span_end) - counter; + ans.push((id.to_span(len as usize), f)); + counter += len; + } + + ans + } + + #[inline] + pub fn compact_change_store(&mut self) { + self.change_store + .flush_and_compact(self.dag.vv(), self.dag.frontiers()); + } + + #[inline] + pub fn change_store_kv_size(&self) -> usize { + self.change_store.kv_size() + } + + pub fn encode_change_store(&self) -> bytes::Bytes { + self.change_store + .encode_all(self.dag.vv(), self.dag.frontiers()) + } + + pub fn check_dag_correctness(&self) { + self.dag.check_dag_correctness(); + } + + pub fn shallow_since_vv(&self) -> &ImVersionVector { + self.dag.shallow_since_vv() + } + + pub fn shallow_since_frontiers(&self) -> &Frontiers { + self.dag.shallow_since_frontiers() + } + + pub fn is_shallow(&self) -> bool { + !self.dag.shallow_since_vv().is_empty() + } + + pub fn get_greatest_timestamp(&self, frontiers: &Frontiers) -> Timestamp { + let mut max_timestamp = Timestamp::default(); + for id in frontiers.iter() { + let change = self.get_change_at(id).unwrap(); + if change.timestamp > max_timestamp { + max_timestamp = change.timestamp; + } + } + + max_timestamp + } +} + +#[derive(Debug)] +pub struct SizeInfo { + pub total_changes: usize, + pub total_ops: usize, + pub total_atom_ops: usize, + pub total_dag_node: usize, +} + +pub(crate) fn convert_change_to_remote( + arena: &SharedArena, + change: &Change, +) -> Change> { + let mut ops = RleVec::new(); + for op in change.ops.iter() { + for op in local_op_to_remote(arena, op) { + ops.push(op); + } + } + + Change { + ops, + id: change.id, + deps: change.deps.clone(), + lamport: change.lamport, + timestamp: change.timestamp, + commit_msg: change.commit_msg.clone(), + } +} + +pub(crate) fn local_op_to_remote( + arena: &SharedArena, + op: &crate::op::Op, +) -> SmallVec<[RemoteOp<'static>; 1]> { + let container = arena.get_container_id(op.container).unwrap(); + let mut contents: SmallVec<[_; 1]> = SmallVec::new(); + match &op.content { + crate::op::InnerContent::List(list) => match list { + list_op::InnerListOp::Insert { slice, pos } => match container.container_type() { + loro_common::ContainerType::Text => { + let str = arena + .slice_str_by_unicode_range(slice.0.start as usize..slice.0.end as usize); + contents.push(RawOpContent::List(list_op::ListOp::Insert { + slice: ListSlice::RawStr { + unicode_len: str.chars().count(), + str: Cow::Owned(str), + }, + pos: *pos, + })); + } + loro_common::ContainerType::List | loro_common::ContainerType::MovableList => { + contents.push(RawOpContent::List(list_op::ListOp::Insert { + slice: ListSlice::RawData(Cow::Owned( + arena.get_values(slice.0.start as usize..slice.0.end as usize), + )), + pos: *pos, + })) + } + _ => unreachable!(), + }, + list_op::InnerListOp::InsertText { + slice, + unicode_len: len, + unicode_start: _, + pos, + } => match container.container_type() { + loro_common::ContainerType::Text => { + contents.push(RawOpContent::List(list_op::ListOp::Insert { + slice: ListSlice::RawStr { + unicode_len: *len as usize, + str: Cow::Owned(std::str::from_utf8(slice).unwrap().to_owned()), + }, + pos: *pos as usize, + })); + } + _ => unreachable!(), + }, + list_op::InnerListOp::Delete(del) => { + contents.push(RawOpContent::List(list_op::ListOp::Delete(*del))) + } + list_op::InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => contents.push(RawOpContent::List(list_op::ListOp::StyleStart { + start: *start, + end: *end, + key: key.clone(), + value: value.clone(), + info: *info, + })), + list_op::InnerListOp::StyleEnd => { + contents.push(RawOpContent::List(list_op::ListOp::StyleEnd)) + } + list_op::InnerListOp::Move { + from, + elem_id: from_id, + to, + } => contents.push(RawOpContent::List(list_op::ListOp::Move { + from: *from, + elem_id: *from_id, + to: *to, + })), + list_op::InnerListOp::Set { elem_id, value } => { + contents.push(RawOpContent::List(list_op::ListOp::Set { + elem_id: *elem_id, + value: value.clone(), + })) + } + }, + crate::op::InnerContent::Map(map) => { + let value = map.value.clone(); + contents.push(RawOpContent::Map(crate::container::map::MapSet { + key: map.key.clone(), + value, + })) + } + crate::op::InnerContent::Tree(tree) => contents.push(RawOpContent::Tree(tree.clone())), + crate::op::InnerContent::Future(f) => match f { + #[cfg(feature = "counter")] + crate::op::FutureInnerContent::Counter(c) => contents.push(RawOpContent::Counter(*c)), + FutureInnerContent::Unknown { prop, value } => { + contents.push(crate::op::RawOpContent::Unknown { + prop: *prop, + value: (**value).clone(), + }) + } + }, + }; + + let mut ans = SmallVec::with_capacity(contents.len()); + for content in contents { + ans.push(RemoteOp { + container: container.clone(), + content, + counter: op.counter, + }) + } + ans +} + +pub(crate) fn get_timestamp_now_txn() -> Timestamp { + (get_sys_timestamp() as Timestamp + 500) / 1000 +} diff --git a/vendor/loro-internal/src/oplog/change_store.rs b/vendor/loro-internal/src/oplog/change_store.rs new file mode 100644 index 00000000..750a8d7c --- /dev/null +++ b/vendor/loro-internal/src/oplog/change_store.rs @@ -0,0 +1,1788 @@ +use self::block_encode::{decode_block, decode_header, encode_block, ChangesBlockHeader}; +use super::{loro_dag::AppDagNodeInner, AppDagNode}; +use crate::sync::Mutex; +use crate::{ + arena::SharedArena, + change::Change, + estimated_size::EstimatedSize, + kv_store::KvStore, + op::Op, + parent::register_container_and_parent_link, + version::{Frontiers, ImVersionVector}, + VersionVector, +}; +use block_encode::decode_block_range; +use bytes::Bytes; +use itertools::Itertools; +use loro_common::{ + Counter, HasCounterSpan, HasId, HasIdSpan, HasLamportSpan, IdLp, IdSpan, Lamport, LoroError, + LoroResult, PeerID, ID, +}; +use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; +use once_cell::sync::OnceCell; +use rle::{HasLength, Mergable, RlePush, RleVec, Sliceable}; +use std::sync::atomic::AtomicI64; +use std::{ + cmp::Ordering, + collections::{BTreeMap, VecDeque}, + ops::{Bound, Deref}, + sync::Arc, +}; +use tracing::{info_span, warn}; +mod block_encode; +mod block_meta_encode; +pub(super) mod iter; + +#[cfg(not(test))] +const MAX_BLOCK_SIZE: usize = 1024 * 4; +#[cfg(test)] +const MAX_BLOCK_SIZE: usize = 128; + +/// # Invariance +/// +/// - We don't allow holes in a block or between two blocks with the same peer id. +/// The [Change] should be continuous for each peer. +/// - However, the first block of a peer can have counter > 0 so that we can trim the history. +/// +/// # Encoding Schema +/// +/// It's based on the underlying KV store. +/// +/// The entries of the KV store is made up of the following fields +/// +/// |Key |Value | +/// |:-- |:---- | +/// |b"vv" |VersionVector | +/// |b"fr" |Frontiers | +/// |b"sv" |Shallow VV | +/// |b"sf" |Shallow Frontiers | +/// |12 bytes PeerID + Counter |Encoded Block | +#[derive(Debug, Clone)] +pub struct ChangeStore { + inner: Arc>, + arena: SharedArena, + /// A change may be in external_kv or in the mem_parsed_kv. + /// mem_parsed_kv is more up-to-date. + /// + /// We cannot directly write into the external_kv except from the initial load + external_kv: Arc>, + /// The version vector of the external kv store. + external_vv: Arc>, + merge_interval: Arc, +} + +#[derive(Debug, Clone)] +struct ChangeStoreInner { + /// The start version vector of the first block for each peer. + /// It allows us to trim the history + start_vv: ImVersionVector, + /// The last version of the shallow history. + start_frontiers: Frontiers, + /// It's more like a parsed cache for binary_kv. + mem_parsed_kv: BTreeMap>, +} + +#[derive(Debug)] +pub(crate) struct ChangeStoreRollback { + old_vv: VersionVector, + blocks_before_mutation: BTreeMap>, +} + +impl ChangeStoreRollback { + pub(crate) fn new(old_vv: VersionVector) -> Self { + Self { + old_vv, + blocks_before_mutation: BTreeMap::new(), + } + } + + fn record_block_before_mutation(&mut self, id: ID, block: Arc) { + let old_end = self.old_vv.get(&id.peer).copied().unwrap_or(0); + if id.counter >= old_end { + return; + } + + self.blocks_before_mutation.entry(id).or_insert(block); + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ChangesBlock { + peer: PeerID, + counter_range: (Counter, Counter), + lamport_range: (Lamport, Lamport), + /// Estimated size of the block in bytes + estimated_size: usize, + flushed: bool, + content: ChangesBlockContent, +} + +#[derive(Clone)] +pub(crate) enum ChangesBlockContent { + Changes(Arc>), + Bytes(ChangesBlockBytes), + Both(Arc>, ChangesBlockBytes), +} + +/// It's cheap to clone this struct because it's cheap to clone the bytes +#[derive(Clone)] +pub(crate) struct ChangesBlockBytes { + bytes: Bytes, + header: OnceCell>, +} + +pub const START_VV_KEY: &[u8] = b"sv"; +pub const START_FRONTIERS_KEY: &[u8] = b"sf"; +pub const VV_KEY: &[u8] = b"vv"; +pub const FRONTIERS_KEY: &[u8] = b"fr"; + +impl ChangeStore { + pub fn new_mem(a: &SharedArena, merge_interval: Arc) -> Self { + Self { + inner: Arc::new(Mutex::new(ChangeStoreInner { + start_vv: ImVersionVector::new(), + start_frontiers: Frontiers::default(), + mem_parsed_kv: BTreeMap::new(), + })), + arena: a.clone(), + external_vv: Arc::new(Mutex::new(VersionVector::new())), + external_kv: Arc::new(Mutex::new(MemKvStore::new(MemKvConfig::default()))), + // external_kv: Arc::new(Mutex::new(BTreeMap::default())), + merge_interval, + } + } + + #[cfg(test)] + fn new_for_test() -> Self { + Self::new_mem(&SharedArena::new(), Arc::new(AtomicI64::new(0))) + } + + pub(super) fn encode_all(&self, vv: &VersionVector, frontiers: &Frontiers) -> Bytes { + self.flush_and_compact(vv, frontiers); + let mut kv = self.external_kv.lock(); + kv.export_all() + } + + #[tracing::instrument(skip(self), level = "debug")] + pub(super) fn export_from( + &self, + start_vv: &VersionVector, + start_frontiers: &Frontiers, + latest_vv: &VersionVector, + latest_frontiers: &Frontiers, + ) -> Bytes { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for span in latest_vv.sub_iter(start_vv) { + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + let end = ((latest_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + + if start == end { + continue; + } + + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + loro_common::debug!( + "start_vv={:?} start_frontiers={:?}", + &start_vv, + start_frontiers + ); + new_store.encode_from(start_vv, start_frontiers, latest_vv, latest_frontiers) + } + + pub(super) fn export_blocks_in_range(&self, spans: &[IdSpan], w: &mut W) { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for span in spans { + let mut span = *span; + span.normalize_(); + if span.counter.end <= 0 { + continue; + } + + span.counter.start = span.counter.start.max(0); + span.counter.end = span.counter.end.max(0); + if span.counter.start >= span.counter.end { + continue; + } + + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((span.counter.start - c.id.counter).max(0) as usize).min(c.atom_len()); + let end = ((span.counter.end - c.id.counter).max(0) as usize).min(c.atom_len()); + if start == end { + continue; + } + + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + encode_blocks_in_store(new_store, &self.arena, w); + } + + fn encode_from( + &self, + start_vv: &VersionVector, + start_frontiers: &Frontiers, + latest_vv: &VersionVector, + latest_frontiers: &Frontiers, + ) -> Bytes { + { + let mut store = self.external_kv.lock(); + store.set(START_VV_KEY, start_vv.encode().into()); + store.set(START_FRONTIERS_KEY, start_frontiers.encode().into()); + let mut inner = self.inner.lock(); + inner.start_frontiers = start_frontiers.clone(); + inner.start_vv = ImVersionVector::from_vv(start_vv); + } + self.flush_and_compact(latest_vv, latest_frontiers); + self.external_kv.lock().export_all() + } + + pub(crate) fn decode_snapshot_for_updates( + bytes: Bytes, + arena: &SharedArena, + self_vv: &VersionVector, + ) -> Result, LoroError> { + let change_store = ChangeStore::new_mem(arena, Arc::new(AtomicI64::new(0))); + let _ = change_store.import_all(bytes)?; + let mut changes = Vec::new(); + change_store.visit_all_changes(&mut |c| { + let cnt_threshold = self_vv.get(&c.id.peer).copied().unwrap_or(0); + if c.id.counter >= cnt_threshold { + changes.push(c.clone()); + return; + } + + let change_end = c.ctr_end(); + if change_end > cnt_threshold { + changes.push(c.slice((cnt_threshold - c.id.counter) as usize, c.atom_len())); + } + }); + + Ok(changes) + } + + pub(crate) fn decode_block_bytes( + bytes: Bytes, + arena: &SharedArena, + self_vv: &VersionVector, + ) -> LoroResult> { + let mut ans = ChangesBlockBytes::new(bytes).parse(arena)?; + if ans.is_empty() { + return Ok(ans); + } + + let start = self_vv.get(&ans[0].peer()).copied().unwrap_or(0); + ans.retain_mut(|c| { + if c.id.counter >= start { + true + } else if c.ctr_end() > start { + *c = c.slice((start - c.id.counter) as usize, c.atom_len()); + true + } else { + false + } + }); + + Ok(ans) + } + + pub(crate) fn rollback_import(&self, rollback: ChangeStoreRollback) { + let mut inner = self.inner.lock(); + inner.mem_parsed_kv.retain(|id, _| { + let old_end = rollback.old_vv.get(&id.peer).copied().unwrap_or(0); + id.counter < old_end + }); + + for (id, block) in rollback.blocks_before_mutation { + inner.mem_parsed_kv.insert(id, block); + } + } + + pub fn get_dag_nodes_that_contains(&self, id: ID) -> Option> { + let block = self.get_block_that_contains(id)?; + Some(block.content.iter_dag_nodes()) + } + + pub fn get_last_dag_nodes_for_peer(&self, peer: PeerID) -> Option> { + let block = self.get_the_last_block_of_peer(peer)?; + Some(block.content.iter_dag_nodes()) + } + + pub fn visit_all_changes(&self, f: &mut dyn FnMut(&Change)) { + self.ensure_block_loaded_in_range(Bound::Unbounded, Bound::Unbounded); + let mut inner = self.inner.lock(); + for (_, block) in inner.mem_parsed_kv.iter_mut() { + block + .ensure_changes(&self.arena) + .expect("Parse block error"); + for c in block.content.try_changes().unwrap() { + f(c); + } + } + } + + pub(crate) fn iter_blocks(&self, id_span: IdSpan) -> Vec<(Arc, usize, usize)> { + if id_span.counter.start == id_span.counter.end { + return vec![]; + } + + assert!(id_span.counter.start < id_span.counter.end); + self.ensure_block_loaded_in_range( + Bound::Included(id_span.id_start()), + Bound::Excluded(id_span.id_end()), + ); + let mut inner = self.inner.lock(); + let next_back = inner.mem_parsed_kv.range(..=id_span.id_start()).next_back(); + match next_back { + None => { + return vec![]; + } + Some(next_back) => { + if next_back.0.peer != id_span.peer { + return vec![]; + } + } + } + let start_counter = next_back.map(|(id, _)| id.counter).unwrap_or(0); + let ans = inner + .mem_parsed_kv + .range_mut( + ID::new(id_span.peer, start_counter)..ID::new(id_span.peer, id_span.counter.end), + ) + .filter_map(|(_id, block)| { + if block.counter_range.1 < id_span.counter.start { + return None; + } + + block + .ensure_changes(&self.arena) + .expect("Parse block error"); + let changes = block.content.try_changes().unwrap(); + let start; + let end; + if id_span.counter.start <= block.counter_range.0 + && id_span.counter.end >= block.counter_range.1 + { + start = 0; + end = changes.len(); + } else { + start = block + .get_change_index_by_counter(id_span.counter.start) + .unwrap_or_else(|x| x); + + match block.get_change_index_by_counter(id_span.counter.end - 1) { + Ok(e) => { + end = e + 1; + } + Err(0) => return None, + Err(e) => { + end = e; + } + } + } + if start == end { + return None; + } + + Some((block.clone(), start, end)) + }) + // TODO: PERF avoid alloc + .collect_vec(); + + ans + } + + pub fn iter_changes(&self, id_span: IdSpan) -> impl Iterator + '_ { + let v = self.iter_blocks(id_span); + #[cfg(debug_assertions)] + { + if !v.is_empty() { + assert_eq!(v[0].0.peer, id_span.peer); + assert_eq!(v.last().unwrap().0.peer, id_span.peer); + { + // Test start + let (block, start, _end) = v.first().unwrap(); + let changes = block.content.try_changes().unwrap(); + assert!(changes[*start].id.counter <= id_span.counter.start); + } + { + // Test end + let (block, _start, end) = v.last().unwrap(); + let changes = block.content.try_changes().unwrap(); + assert!(changes[*end - 1].ctr_end() >= id_span.counter.end); + assert!(changes[*end - 1].id.counter < id_span.counter.end); + } + } + } + + v.into_iter().flat_map(move |(block, start, end)| { + (start..end).map(move |i| BlockChangeRef { + change_index: i, + block: block.clone(), + }) + }) + } + + #[allow(dead_code)] + pub(crate) fn get_blocks_in_range(&self, id_span: IdSpan) -> VecDeque> { + let mut inner = self.inner.lock(); + let start_counter = inner + .mem_parsed_kv + .range(..=id_span.id_start()) + .next_back() + .map(|(id, _)| id.counter) + .unwrap_or(0); + let vec = inner + .mem_parsed_kv + .range_mut( + ID::new(id_span.peer, start_counter)..ID::new(id_span.peer, id_span.counter.end), + ) + .filter_map(|(_id, block)| { + if block.counter_range.1 < id_span.counter.start { + return None; + } + + block + .ensure_changes(&self.arena) + .expect("Parse block error"); + Some(block.clone()) + }) + // TODO: PERF avoid alloc + .collect(); + vec + } + + pub(crate) fn get_block_that_contains(&self, id: ID) -> Option> { + self.ensure_block_loaded_in_range(Bound::Included(id), Bound::Included(id)); + let inner = self.inner.lock(); + let block = inner + .mem_parsed_kv + .range(..=id) + .next_back() + .filter(|(_, block)| { + block.peer == id.peer + && block.counter_range.0 <= id.counter + && id.counter < block.counter_range.1 + }) + .map(|(_, block)| block.clone()); + + block + } + + pub(crate) fn get_the_last_block_of_peer(&self, peer: PeerID) -> Option> { + let end_id = ID::new(peer, Counter::MAX); + self.ensure_id_lte(end_id); + let inner = self.inner.lock(); + let block = inner + .mem_parsed_kv + .range(..=end_id) + .next_back() + .filter(|(_, block)| block.peer == peer) + .map(|(_, block)| block.clone()); + + block + } + + pub fn change_num(&self) -> usize { + self.ensure_block_loaded_in_range(Bound::Unbounded, Bound::Unbounded); + let mut inner = self.inner.lock(); + inner + .mem_parsed_kv + .iter_mut() + .map(|(_, block)| block.change_num()) + .sum() + } + + pub fn fork( + &self, + arena: SharedArena, + merge_interval: Arc, + vv: &VersionVector, + frontiers: &Frontiers, + ) -> Self { + self.flush_and_compact(vv, frontiers); + let inner = self.inner.lock(); + Self { + inner: Arc::new(Mutex::new(ChangeStoreInner { + start_vv: inner.start_vv.clone(), + start_frontiers: inner.start_frontiers.clone(), + mem_parsed_kv: BTreeMap::new(), + })), + arena, + external_vv: Arc::new(Mutex::new(self.external_vv.lock().clone())), + external_kv: self.external_kv.lock().clone_store(), + merge_interval, + } + } + + pub fn kv_size(&self) -> usize { + self.external_kv + .lock() + .scan(Bound::Unbounded, Bound::Unbounded) + .map(|(k, v)| k.len() + v.len()) + .sum() + } + + pub(crate) fn export_blocks_from( + &self, + start_vv: &VersionVector, + shallow_since_vv: &ImVersionVector, + latest_vv: &VersionVector, + w: &mut W, + ) { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for mut span in latest_vv.sub_iter(start_vv) { + let counter_lower_bound = shallow_since_vv.get(&span.peer).copied().unwrap_or(0); + span.counter.start = span.counter.start.max(counter_lower_bound); + span.counter.end = span.counter.end.max(counter_lower_bound); + if span.counter.start >= span.counter.end { + continue; + } + + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + let end = ((latest_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + + assert_ne!(start, end); + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + let arena = &self.arena; + encode_blocks_in_store(new_store, arena, w); + } + + pub(crate) fn fork_changes_up_to( + &self, + start_vv: &ImVersionVector, + frontiers: &Frontiers, + vv: &VersionVector, + ) -> Bytes { + let new_store = ChangeStore::new_mem(&self.arena, self.merge_interval.clone()); + for mut span in vv.sub_iter_im(start_vv) { + let counter_lower_bound = start_vv.get(&span.peer).copied().unwrap_or(0); + span.counter.start = span.counter.start.max(counter_lower_bound); + span.counter.end = span.counter.end.max(counter_lower_bound); + if span.counter.start >= span.counter.end { + continue; + } + + // PERF: this can be optimized by reusing the current encoded blocks + // In the current method, it needs to parse and re-encode the blocks + for c in self.iter_changes(span) { + let start = ((start_vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + let end = ((vv.get(&c.id.peer).copied().unwrap_or(0) - c.id.counter).max(0) + as usize) + .min(c.atom_len()); + + assert_ne!(start, end); + let ch = c.slice(start, end); + new_store.insert_change(ch, false, false); + } + } + + new_store.encode_all(vv, frontiers) + } +} + +fn encode_blocks_in_store( + new_store: ChangeStore, + arena: &SharedArena, + w: &mut W, +) { + let mut inner = new_store.inner.lock(); + for (_id, block) in inner.mem_parsed_kv.iter_mut() { + let bytes = block.to_bytes(arena); + leb128::write::unsigned(w, bytes.bytes.len() as u64).unwrap(); + w.write_all(&bytes.bytes).unwrap(); + } +} + +mod mut_external_kv { + //! Only this module contains the code that mutate the external kv store + //! All other modules should only read from the external kv store + use super::*; + + impl ChangeStore { + #[tracing::instrument(skip_all, level = "debug", name = "change_store import_all")] + pub(crate) fn import_all(&self, bytes: Bytes) -> Result { + let mut kv_store = self.external_kv.lock(); + assert!( + // 2 because there are vv and frontiers + kv_store.len() <= 2, + "kv store should be empty when using decode_all" + ); + kv_store + .import_all(bytes) + .map_err(|e| LoroError::DecodeError(e.into_boxed_str()))?; + drop(kv_store); + self.validate_imported_change_blocks()?; + let vv_bytes = self.external_kv.lock().get(VV_KEY).unwrap_or_default(); + let vv = VersionVector::decode(&vv_bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let start_vv_bytes = self + .external_kv + .lock() + .get(START_VV_KEY) + .unwrap_or_default(); + let start_vv = if start_vv_bytes.is_empty() { + Default::default() + } else { + VersionVector::decode(&start_vv_bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)? + }; + + #[cfg(test)] + { + // This is for tests + for (peer, cnt) in vv.iter() { + self.get_change(ID::new(*peer, *cnt - 1)) + .ok_or(LoroError::DecodeDataCorruptionError)?; + } + } + + *self.external_vv.lock() = vv.clone(); + let frontiers_bytes = self + .external_kv + .lock() + .get(FRONTIERS_KEY) + .unwrap_or_default(); + let frontiers = Frontiers::decode(&frontiers_bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let start_frontiers = self + .external_kv + .lock() + .get(START_FRONTIERS_KEY) + .unwrap_or_default(); + let start_frontiers = if start_frontiers.is_empty() { + Default::default() + } else { + Frontiers::decode(&start_frontiers) + .map_err(|_| LoroError::DecodeDataCorruptionError)? + }; + + let mut max_lamport = None; + let mut max_timestamp = 0; + for id in frontiers.iter() { + let c = self + .get_change(id) + .ok_or(LoroError::DecodeDataCorruptionError)?; + debug_assert_ne!(c.atom_len(), 0); + let l = c.lamport_last(); + if let Some(x) = max_lamport { + if l > x { + max_lamport = Some(l); + } + } else { + max_lamport = Some(l); + } + + let t = c.timestamp; + if t > max_timestamp { + max_timestamp = t; + } + } + + Ok(BatchDecodeInfo { + vv, + frontiers, + start_version: if start_vv.is_empty() { + None + } else { + let mut inner = self.inner.lock(); + inner.start_frontiers = start_frontiers.clone(); + inner.start_vv = ImVersionVector::from_vv(&start_vv); + Some((start_vv, start_frontiers)) + }, + }) + } + + fn validate_imported_change_blocks(&self) -> LoroResult<()> { + let blocks: Vec<(ID, Bytes)> = { + let kv_store = self.external_kv.lock(); + kv_store + .scan(Bound::Unbounded, Bound::Unbounded) + .filter(|(id, _)| id.len() == 12) + .map(|(id, bytes)| (ID::from_bytes(&id), bytes)) + .collect() + }; + + for (_id, bytes) in blocks { + let mut block = Arc::new(ChangesBlock::from_bytes(bytes)?); + block.ensure_changes(&self.arena)?; + } + + Ok(()) + } + + /// Flush the cached change to kv_store + pub(crate) fn flush_and_compact(&self, vv: &VersionVector, frontiers: &Frontiers) { + let mut inner = self.inner.lock(); + let mut store = self.external_kv.lock(); + let mut external_vv = self.external_vv.lock(); + for (id, block) in inner.mem_parsed_kv.iter_mut() { + if !block.flushed { + let id_bytes = id.to_bytes(); + let counter_start = external_vv.get(&id.peer).copied().unwrap_or(0); + assert!( + counter_start < block.counter_range.1, + "Peer={} Block Counter Range={:?}, counter_start={}", + id.peer, + &block.counter_range, + counter_start + ); + if counter_start > block.counter_range.0 { + assert!(store.get(&id_bytes).is_some()); + } + external_vv.insert(id.peer, block.counter_range.1); + let bytes = block.to_bytes(&self.arena); + store.set(&id_bytes, bytes.bytes); + Arc::make_mut(block).flushed = true; + } + } + + if inner.start_vv.is_empty() { + assert_eq!(&*external_vv, vv); + } else { + #[cfg(debug_assertions)] + { + // TODO: makes some assertions here? + } + } + let vv_bytes = vv.encode(); + let frontiers_bytes = frontiers.encode(); + store.set(VV_KEY, vv_bytes.into()); + store.set(FRONTIERS_KEY, frontiers_bytes.into()); + } + } +} + +mod mut_inner_kv { + //! Only this module contains the code that mutate the internal kv store + //! All other modules should only read from the internal kv store + + use super::*; + impl ChangeStore { + /// This method is the **only place** that push a new change into the change store + /// + /// The new change either merges with the previous block or is put into a new block. + /// This method only updates the internal kv store. + pub fn insert_change(&self, change: Change, split_when_exceeds: bool, is_local: bool) { + self.insert_change_inner(change, split_when_exceeds, is_local, None); + } + + pub(crate) fn insert_change_with_rollback( + &self, + change: Change, + split_when_exceeds: bool, + is_local: bool, + rollback: &mut ChangeStoreRollback, + ) { + self.insert_change_inner(change, split_when_exceeds, is_local, Some(rollback)); + } + + fn insert_change_inner( + &self, + mut change: Change, + split_when_exceeds: bool, + is_local: bool, + mut rollback: Option<&mut ChangeStoreRollback>, + ) { + #[cfg(debug_assertions)] + { + let vv = self.external_vv.lock(); + assert!(vv.get(&change.id.peer).copied().unwrap_or(0) <= change.id.counter); + } + + let s = info_span!("change_store insert_change", id = ?change.id); + let _e = s.enter(); + let estimated_size = change.estimate_storage_size(); + if estimated_size > MAX_BLOCK_SIZE && split_when_exceeds { + self.split_change_then_insert(change, rollback.as_deref_mut()); + return; + } + + let id = change.id; + let mut inner = self.inner.lock(); + + // try to merge with previous block + if let Some((_id, block)) = inner.mem_parsed_kv.range_mut(..id).next_back() { + if block.peer == change.id.peer { + if block.counter_range.1 != change.id.counter { + panic!("counter should be continuous") + } + + if let Some(rollback) = &mut rollback { + rollback.record_block_before_mutation(*_id, block.clone()); + } + + match block.push_change( + change, + estimated_size, + if is_local { + // local change should try to merge with previous change when + // the timestamp interval <= the `merge_interval` + self.merge_interval + .load(std::sync::atomic::Ordering::Acquire) + } else { + 0 + }, + &self.arena, + ) { + Ok(_) => { + drop(inner); + debug_assert!(self.get_change(id).is_some()); + return; + } + Err(c) => change = c, + } + } + } + + inner + .mem_parsed_kv + .insert(id, Arc::new(ChangesBlock::new(change, &self.arena))); + drop(inner); + debug_assert!(self.get_change(id).is_some()); + } + + pub fn get_change(&self, id: ID) -> Option { + let block = self.get_parsed_block(id)?; + Some(BlockChangeRef { + change_index: block.get_change_index_by_counter(id.counter).unwrap(), + block: block.clone(), + }) + } + + /// Get the change with the given peer and lamport. + /// + /// If not found, return the change with the greatest lamport that is smaller than the given lamport. + pub fn get_change_by_lamport_lte(&self, idlp: IdLp) -> Option { + // This method is complicated because we impl binary search on top of the range api + // It can be simplified + let mut inner = self.inner.lock(); + let mut iter = inner + .mem_parsed_kv + .range_mut(ID::new(idlp.peer, 0)..ID::new(idlp.peer, i32::MAX)); + + // This won't change, we only adjust upper_bound + let mut lower_bound = 0; + let mut upper_bound = i32::MAX; + let mut is_binary_searching = false; + loop { + match iter.next_back() { + Some((&id, block)) => { + if block.lamport_range.0 <= idlp.lamport + && (!is_binary_searching || idlp.lamport < block.lamport_range.1) + { + if !is_binary_searching + && upper_bound != i32::MAX + && upper_bound != block.counter_range.1 + { + warn!( + "There is a hole between the last block and the current block" + ); + // There is hole between the last block and the current block + // We need to load it from the kv store + break; + } + + // Found the block + block + .ensure_changes(&self.arena) + .expect("Parse block error"); + let index = block.get_change_index_by_lamport_lte(idlp.lamport)?; + return Some(BlockChangeRef { + change_index: index, + block: block.clone(), + }); + } + + if is_binary_searching { + let mid_bound = (lower_bound + upper_bound) / 2; + if block.lamport_range.1 <= idlp.lamport { + // Target is larger than the current block (pointed by mid_bound) + lower_bound = mid_bound; + } else { + debug_assert!( + idlp.lamport < block.lamport_range.0, + "{} {:?}", + idlp, + &block.lamport_range + ); + // Target is smaller than the current block (pointed by mid_bound) + upper_bound = mid_bound; + } + + let mid_bound = (lower_bound + upper_bound) / 2; + iter = inner + .mem_parsed_kv + .range_mut(ID::new(idlp.peer, 0)..ID::new(idlp.peer, mid_bound)); + } else { + // Test whether we need to switch to binary search by measuring the gap + if block.lamport_range.0 - idlp.lamport > MAX_BLOCK_SIZE as Lamport * 8 + { + // Use binary search to find the block + upper_bound = id.counter; + let mid_bound = (lower_bound + upper_bound) / 2; + iter = inner.mem_parsed_kv.range_mut( + ID::new(idlp.peer, 0)..ID::new(idlp.peer, mid_bound), + ); + is_binary_searching = true; + } + + upper_bound = id.counter; + } + } + None => { + if !is_binary_searching { + break; + } + + let mid_bound = (lower_bound + upper_bound) / 2; + lower_bound = mid_bound; + if upper_bound - lower_bound <= MAX_BLOCK_SIZE as i32 { + // If they are too close, we can just scan the range + iter = inner.mem_parsed_kv.range_mut( + ID::new(idlp.peer, lower_bound)..ID::new(idlp.peer, upper_bound), + ); + is_binary_searching = false; + } else { + let mid_bound = (lower_bound + upper_bound) / 2; + iter = inner + .mem_parsed_kv + .range_mut(ID::new(idlp.peer, 0)..ID::new(idlp.peer, mid_bound)); + } + } + } + } + + let counter_end = upper_bound; + let scan_end = ID::new(idlp.peer, counter_end).to_bytes(); + + let (id, bytes) = 'block_scan: { + let kv_store = &self.external_kv.lock(); + let iter = kv_store + .scan( + Bound::Included(&ID::new(idlp.peer, 0).to_bytes()), + Bound::Excluded(&scan_end), + ) + .rev(); + + for (id, bytes) in iter { + let mut block = ChangesBlockBytes::new(bytes.clone()); + let (lamport_start, _lamport_end) = block.lamport_range(); + if lamport_start <= idlp.lamport { + break 'block_scan (id, bytes); + } + } + + return None; + }; + + let block_id = ID::from_bytes(&id); + let mut block = Arc::new( + ChangesBlock::from_bytes(bytes) + .expect("validated external change block should decode"), + ); + block + .ensure_changes(&self.arena) + .expect("Parse block error"); + inner.mem_parsed_kv.insert(block_id, block.clone()); + let index = block.get_change_index_by_lamport_lte(idlp.lamport)?; + Some(BlockChangeRef { + change_index: index, + block, + }) + } + + fn split_change_then_insert( + &self, + change: Change, + mut rollback: Option<&mut ChangeStoreRollback>, + ) { + let original_len = change.atom_len(); + let mut new_change = Change { + ops: RleVec::new(), + deps: change.deps, + id: change.id, + lamport: change.lamport, + timestamp: change.timestamp, + commit_msg: change.commit_msg.clone(), + }; + + let mut total_len = 0; + let mut estimated_size = new_change.estimate_storage_size(); + 'outer: for mut op in change.ops.into_iter() { + if op.estimate_storage_size() >= MAX_BLOCK_SIZE - estimated_size { + new_change = self._insert_splitted_change( + new_change, + &mut total_len, + &mut estimated_size, + rollback.as_deref_mut(), + ); + } + + while let Some(end) = + op.check_whether_slice_content_to_fit_in_size(MAX_BLOCK_SIZE - estimated_size) + { + // The new op can take the rest of the room + let new = op.slice(0, end); + new_change.ops.push(new); + new_change = self._insert_splitted_change( + new_change, + &mut total_len, + &mut estimated_size, + rollback.as_deref_mut(), + ); + + if end < op.atom_len() { + op = op.slice(end, op.atom_len()); + } else { + continue 'outer; + } + } + + estimated_size += op.estimate_storage_size(); + if estimated_size > MAX_BLOCK_SIZE && !new_change.ops.is_empty() { + new_change = self._insert_splitted_change( + new_change, + &mut total_len, + &mut estimated_size, + rollback.as_deref_mut(), + ); + new_change.ops.push(op); + } else { + new_change.ops.push(op); + } + } + + if !new_change.ops.is_empty() { + total_len += new_change.atom_len(); + self.insert_change_inner(new_change, false, false, rollback); + } + + assert_eq!(total_len, original_len); + } + + fn _insert_splitted_change( + &self, + new_change: Change, + total_len: &mut usize, + estimated_size: &mut usize, + rollback: Option<&mut ChangeStoreRollback>, + ) -> Change { + if new_change.atom_len() == 0 { + return new_change; + } + + let ctr_end = new_change.id.counter + new_change.atom_len() as Counter; + let next_lamport = new_change.lamport + new_change.atom_len() as Lamport; + *total_len += new_change.atom_len(); + let ans = Change { + ops: RleVec::new(), + deps: ID::new(new_change.id.peer, ctr_end - 1).into(), + id: ID::new(new_change.id.peer, ctr_end), + lamport: next_lamport, + timestamp: new_change.timestamp, + commit_msg: new_change.commit_msg.clone(), + }; + + self.insert_change_inner(new_change, false, false, rollback); + *estimated_size = ans.estimate_storage_size(); + ans + } + + fn get_parsed_block(&self, id: ID) -> Option> { + let mut inner = self.inner.lock(); + if let Some((_id, block)) = inner.mem_parsed_kv.range_mut(..=id).next_back() { + if block.peer == id.peer && block.counter_range.1 > id.counter { + block + .ensure_changes(&self.arena) + .expect("Parse block error"); + return Some(block.clone()); + } + } + + let store = self.external_kv.lock(); + let mut iter = store + .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) + .filter(|(id, _)| id.len() == 12); + + // println!( + // "\nkeys {:?}", + // store + // .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) + // .filter(|(id, _)| id.len() == 12) + // .map(|(k, _v)| ID::from_bytes(&k)) + // .count() + // ); + // println!("id {:?}", id); + + let (b_id, b_bytes) = iter.next_back()?; + let block_id: ID = ID::from_bytes(&b_id[..]); + let block = ChangesBlock::from_bytes(b_bytes) + .expect("validated external change block should decode"); + if block_id.peer == id.peer + && block_id.counter <= id.counter + && block.counter_range.1 > id.counter + { + let mut arc_block = Arc::new(block); + arc_block + .ensure_changes(&self.arena) + .expect("Parse block error"); + inner.mem_parsed_kv.insert(block_id, arc_block.clone()); + return Some(arc_block); + } + + None + } + + /// Load all the blocks that have overlapped with the given ID range into `inner_mem_parsed_kv` + /// + /// This is fast because we don't actually parse the content. + // TODO: PERF: This method feels slow. + pub(super) fn ensure_block_loaded_in_range(&self, start: Bound, end: Bound) { + let mut whether_need_scan_backward = match start { + Bound::Included(id) => Some(id), + Bound::Excluded(id) => Some(id.inc(1)), + Bound::Unbounded => None, + }; + + { + let start = start.map(|id| id.to_bytes()); + let end = end.map(|id| id.to_bytes()); + let kv = self.external_kv.lock(); + let mut inner = self.inner.lock(); + for (id, bytes) in kv + .scan( + start.as_ref().map(|x| x.as_slice()), + end.as_ref().map(|x| x.as_slice()), + ) + .filter(|(id, _)| id.len() == 12) + { + let id = ID::from_bytes(&id); + if let Some(expected_start_id) = whether_need_scan_backward { + if id == expected_start_id { + whether_need_scan_backward = None; + } + } + + if inner.mem_parsed_kv.contains_key(&id) { + continue; + } + + let block = ChangesBlock::from_bytes(bytes.clone()) + .expect("validated external change block should decode"); + inner.mem_parsed_kv.insert(id, Arc::new(block)); + } + } + + if let Some(start_id) = whether_need_scan_backward { + self.ensure_id_lte(start_id); + } + } + + pub(super) fn ensure_id_lte(&self, id: ID) { + let kv = self.external_kv.lock(); + let mut inner = self.inner.lock(); + let Some((next_back_id, next_back_bytes)) = kv + .scan(Bound::Unbounded, Bound::Included(&id.to_bytes())) + .rfind(|(id, _)| id.len() == 12) + else { + return; + }; + + let next_back_id = ID::from_bytes(&next_back_id); + if next_back_id.peer == id.peer { + if inner.mem_parsed_kv.contains_key(&next_back_id) { + return; + } + + let block = ChangesBlock::from_bytes(next_back_bytes) + .expect("validated external change block should decode"); + inner.mem_parsed_kv.insert(next_back_id, Arc::new(block)); + } + } + } +} + +#[must_use] +#[derive(Clone, Debug)] +pub(crate) struct BatchDecodeInfo { + pub vv: VersionVector, + pub frontiers: Frontiers, + pub start_version: Option<(VersionVector, Frontiers)>, +} + +#[derive(Clone, Debug)] +pub struct BlockChangeRef { + block: Arc, + change_index: usize, +} + +impl Deref for BlockChangeRef { + type Target = Change; + fn deref(&self) -> &Change { + &self.block.content.try_changes().unwrap()[self.change_index] + } +} + +impl BlockChangeRef { + pub(crate) fn get_op_with_counter(&self, counter: Counter) -> Option { + if counter >= self.ctr_end() { + return None; + } + + let index = self.ops.search_atom_index(counter); + Some(BlockOpRef { + block: self.block.clone(), + change_index: self.change_index, + op_index: index, + }) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct BlockOpRef { + pub block: Arc, + pub change_index: usize, + pub op_index: usize, +} + +impl Deref for BlockOpRef { + type Target = Op; + + fn deref(&self) -> &Op { + &self.block.content.try_changes().unwrap()[self.change_index].ops[self.op_index] + } +} + +impl BlockOpRef { + pub fn lamport(&self) -> Lamport { + let change = &self.block.content.try_changes().unwrap()[self.change_index]; + let op = &change.ops[self.op_index]; + (op.counter - change.id.counter) as Lamport + change.lamport + } +} + +impl ChangesBlock { + fn from_bytes(bytes: Bytes) -> LoroResult { + let len = bytes.len(); + let bytes = ChangesBlockBytes::new(bytes); + bytes.ensure_header()?; + let header = bytes + .header + .get() + .expect("header should be initialized after ensure_header"); + let peer = header.peer; + let counter_range = ( + header.counter, + *header.counters.last().ok_or_else(|| { + LoroError::DecodeError("Decode block error: missing counters".into()) + })?, + ); + let lamport_range = ( + *header.lamports.first().ok_or_else(|| { + LoroError::DecodeError("Decode block error: missing lamports".into()) + })?, + *header.lamports.last().ok_or_else(|| { + LoroError::DecodeError("Decode block error: missing lamports".into()) + })?, + ); + let content = ChangesBlockContent::Bytes(bytes); + Ok(Self { + peer, + estimated_size: len, + counter_range, + lamport_range, + flushed: true, + content, + }) + } + + #[allow(dead_code)] + pub(crate) fn content(&self) -> &ChangesBlockContent { + &self.content + } + + fn new(change: Change, _a: &SharedArena) -> Self { + let atom_len = change.atom_len(); + let counter_range = (change.id.counter, change.id.counter + atom_len as Counter); + let lamport_range = (change.lamport, change.lamport + atom_len as Lamport); + let estimated_size = change.estimate_storage_size(); + let peer = change.id.peer; + let content = ChangesBlockContent::Changes(Arc::new(vec![change])); + Self { + peer, + counter_range, + lamport_range, + estimated_size, + content, + flushed: false, + } + } + + #[allow(unused)] + fn cmp_id(&self, id: ID) -> Ordering { + self.peer.cmp(&id.peer).then_with(|| { + if self.counter_range.0 > id.counter { + Ordering::Greater + } else if self.counter_range.1 <= id.counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + } + + #[allow(unused)] + fn cmp_idlp(&self, idlp: (PeerID, Lamport)) -> Ordering { + self.peer.cmp(&idlp.0).then_with(|| { + if self.lamport_range.0 > idlp.1 { + Ordering::Greater + } else if self.lamport_range.1 <= idlp.1 { + Ordering::Less + } else { + Ordering::Equal + } + }) + } + + #[allow(unused)] + fn is_full(&self) -> bool { + self.estimated_size > MAX_BLOCK_SIZE + } + + #[allow(clippy::result_large_err)] + fn push_change( + self: &mut Arc, + change: Change, + new_change_size: usize, + merge_interval: i64, + a: &SharedArena, + ) -> Result<(), Change> { + if self.counter_range.1 != change.id.counter { + return Err(change); + } + + let atom_len = change.atom_len(); + let next_lamport = change.lamport + atom_len as Lamport; + let next_counter = change.id.counter + atom_len as Counter; + + let is_full = new_change_size + self.estimated_size > MAX_BLOCK_SIZE; + let this = Arc::make_mut(self); + let changes = this.content.changes_mut(a).unwrap(); + let changes = Arc::make_mut(changes); + match changes.last_mut() { + Some(last) + if last.can_merge_right(&change, merge_interval) + && (!is_full + || (change.ops.len() == 1 + && last.ops.last().unwrap().is_mergable(&change.ops[0], &()))) => + { + for op in change.ops.into_iter() { + let size = op.estimate_storage_size(); + if !last.ops.push(op) { + this.estimated_size += size; + } + } + } + _ => { + if is_full { + return Err(change); + } else { + this.estimated_size += new_change_size; + changes.push(change); + } + } + } + + this.flushed = false; + this.counter_range.1 = next_counter; + this.lamport_range.1 = next_lamport; + Ok(()) + } + + fn to_bytes(self: &mut Arc, a: &SharedArena) -> ChangesBlockBytes { + match &self.content { + ChangesBlockContent::Bytes(bytes) => bytes.clone(), + ChangesBlockContent::Both(_, bytes) => { + let bytes = bytes.clone(); + let this = Arc::make_mut(self); + this.content = ChangesBlockContent::Bytes(bytes.clone()); + bytes + } + ChangesBlockContent::Changes(changes) => { + let bytes = ChangesBlockBytes::serialize(changes, a); + let this = Arc::make_mut(self); + this.content = ChangesBlockContent::Bytes(bytes.clone()); + bytes + } + } + } + + fn ensure_changes(self: &mut Arc, a: &SharedArena) -> LoroResult<()> { + match &self.content { + ChangesBlockContent::Changes(_) => Ok(()), + ChangesBlockContent::Both(_, _) => Ok(()), + ChangesBlockContent::Bytes(bytes) => { + let changes = bytes.parse(a)?; + let b = bytes.clone(); + let this = Arc::make_mut(self); + this.content = ChangesBlockContent::Both(Arc::new(changes), b); + Ok(()) + } + } + } + + fn get_change_index_by_counter(&self, counter: Counter) -> Result { + let changes = self.content.try_changes().unwrap(); + changes.binary_search_by(|c| { + if c.id.counter > counter { + Ordering::Greater + } else if (c.id.counter + c.content_len() as Counter) <= counter { + Ordering::Less + } else { + Ordering::Equal + } + }) + } + + fn get_change_index_by_lamport_lte(&self, lamport: Lamport) -> Option { + let changes = self.content.try_changes().unwrap(); + let r = changes.binary_search_by(|c| { + if c.lamport > lamport { + Ordering::Greater + } else if (c.lamport + c.content_len() as Lamport) <= lamport { + Ordering::Less + } else { + Ordering::Equal + } + }); + + match r { + Ok(found) => Some(found), + Err(idx) => { + if idx == 0 { + None + } else { + Some(idx - 1) + } + } + } + } + + #[allow(unused)] + fn get_changes(&mut self, a: &SharedArena) -> LoroResult<&Vec> { + self.content.changes(a) + } + + #[allow(unused)] + fn id(&self) -> ID { + ID::new(self.peer, self.counter_range.0) + } + + pub fn change_num(&self) -> usize { + match &self.content { + ChangesBlockContent::Changes(c) => c.len(), + ChangesBlockContent::Bytes(b) => b.len_changes(), + ChangesBlockContent::Both(c, _) => c.len(), + } + } +} + +impl ChangesBlockContent { + // TODO: PERF: We can use Iter to replace Vec + pub fn iter_dag_nodes(&self) -> Vec { + let mut dag_nodes = Vec::new(); + match self { + ChangesBlockContent::Changes(c) | ChangesBlockContent::Both(c, _) => { + for change in c.iter() { + let new_node = AppDagNodeInner { + peer: change.id.peer, + cnt: change.id.counter, + lamport: change.lamport, + deps: change.deps.clone(), + vv: OnceCell::new(), + has_succ: false, + len: change.atom_len(), + } + .into(); + + dag_nodes.push_rle_element(new_node); + } + } + ChangesBlockContent::Bytes(b) => { + b.ensure_header().unwrap(); + let header = b.header.get().unwrap(); + let n = header.n_changes; + for i in 0..n { + let new_node = AppDagNodeInner { + peer: header.peer, + cnt: header.counters[i], + lamport: header.lamports[i], + deps: header.deps_groups[i].clone(), + vv: OnceCell::new(), + has_succ: false, + len: (header.counters[i + 1] - header.counters[i]) as usize, + } + .into(); + + dag_nodes.push_rle_element(new_node); + } + } + } + + dag_nodes + } + + #[allow(unused)] + pub fn changes(&mut self, a: &SharedArena) -> LoroResult<&Vec> { + match self { + ChangesBlockContent::Changes(changes) => Ok(changes), + ChangesBlockContent::Both(changes, _) => Ok(changes), + ChangesBlockContent::Bytes(bytes) => { + let changes = bytes.parse(a)?; + *self = ChangesBlockContent::Both(Arc::new(changes), bytes.clone()); + self.changes(a) + } + } + } + + /// Note that this method will invalidate the stored bytes + fn changes_mut(&mut self, a: &SharedArena) -> LoroResult<&mut Arc>> { + match self { + ChangesBlockContent::Changes(changes) => Ok(changes), + ChangesBlockContent::Both(changes, _) => { + *self = ChangesBlockContent::Changes(std::mem::take(changes)); + self.changes_mut(a) + } + ChangesBlockContent::Bytes(bytes) => { + let changes = bytes.parse(a)?; + *self = ChangesBlockContent::Changes(Arc::new(changes)); + self.changes_mut(a) + } + } + } + + pub(crate) fn try_changes(&self) -> Option<&Vec> { + match self { + ChangesBlockContent::Changes(changes) => Some(changes), + ChangesBlockContent::Both(changes, _) => Some(changes), + ChangesBlockContent::Bytes(_) => None, + } + } + + #[allow(dead_code)] + pub(crate) fn len_changes(&self) -> usize { + match self { + ChangesBlockContent::Changes(changes) => changes.len(), + ChangesBlockContent::Both(changes, _) => changes.len(), + ChangesBlockContent::Bytes(bytes) => bytes.len_changes(), + } + } +} + +impl std::fmt::Debug for ChangesBlockContent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ChangesBlockContent::Changes(changes) => f + .debug_tuple("ChangesBlockContent::Changes") + .field(changes) + .finish(), + ChangesBlockContent::Bytes(_bytes) => { + f.debug_tuple("ChangesBlockContent::Bytes").finish() + } + ChangesBlockContent::Both(changes, _bytes) => f + .debug_tuple("ChangesBlockContent::Both") + .field(changes) + .finish(), + } + } +} + +impl ChangesBlockBytes { + fn new(bytes: Bytes) -> Self { + Self { + header: OnceCell::new(), + bytes, + } + } + + fn ensure_header(&self) -> LoroResult<()> { + self.header + .get_or_try_init(|| decode_header(&self.bytes).map(Arc::new))?; + Ok(()) + } + + fn parse(&self, a: &SharedArena) -> LoroResult> { + self.ensure_header()?; + let ans: Vec = decode_block(&self.bytes, a, self.header.get().map(|h| h.as_ref()))?; + for c in ans.iter() { + // PERF: This can be made faster (low priority) + register_container_and_parent_link(a, c) + } + + Ok(ans) + } + + fn serialize(changes: &[Change], a: &SharedArena) -> Self { + let bytes = encode_block(changes, a); + // TODO: Perf we can calculate header directly without parsing the bytes + let bytes = ChangesBlockBytes::new(Bytes::from(bytes)); + bytes.ensure_header().unwrap(); + bytes + } + + fn lamport_range(&mut self) -> (Lamport, Lamport) { + if let Some(header) = self.header.get() { + (header.lamports[0], *header.lamports.last().unwrap()) + } else { + decode_block_range(&self.bytes).unwrap().1 + } + } + + /// Length of the changes + fn len_changes(&self) -> usize { + self.ensure_header().unwrap(); + self.header.get().unwrap().n_changes + } +} + +#[cfg(test)] +mod test { + use crate::cursor::PosType; + use crate::{ + loro::ExportMode, oplog::convert_change_to_remote, state::TreeParentId, ListHandler, + LoroDoc, MovableListHandler, TextHandler, TreeHandler, + }; + + use super::*; + + fn test_encode_decode(doc: LoroDoc) { + doc.commit_then_renew(); + let oplog = doc.oplog().lock(); + let bytes = oplog + .change_store + .encode_all(oplog.vv(), oplog.dag.frontiers()); + let store = ChangeStore::new_for_test(); + let _ = store.import_all(bytes.clone()).unwrap(); + assert_eq!(store.external_kv.lock().export_all(), bytes); + let mut changes_parsed = Vec::new(); + let a = store.arena.clone(); + store.visit_all_changes(&mut |c| { + changes_parsed.push(convert_change_to_remote(&a, c)); + }); + let mut changes = Vec::new(); + oplog.change_store.visit_all_changes(&mut |c| { + changes.push(convert_change_to_remote(&oplog.arena, c)); + }); + assert_eq!(changes_parsed, changes); + } + + #[test] + fn test_change_store() { + let doc = LoroDoc::new_auto_commit(); + doc.set_record_timestamp(true); + let t = doc.get_text("t"); + t.insert(0, "hello", PosType::Unicode).unwrap(); + doc.commit_then_renew(); + let t = doc.get_list("t"); + t.insert(0, "hello").unwrap(); + test_encode_decode(doc); + } + + #[test] + fn test_synced_doc() -> LoroResult<()> { + let doc_a = LoroDoc::new_auto_commit(); + let doc_b = LoroDoc::new_auto_commit(); + let doc_c = LoroDoc::new_auto_commit(); + + { + // A: Create initial structure + let map = doc_a.get_map("root"); + map.insert_container("text", TextHandler::new_detached())?; + map.insert_container("list", ListHandler::new_detached())?; + map.insert_container("tree", TreeHandler::new_detached())?; + } + + { + // Sync initial state to B and C + let initial_state = doc_a.export(ExportMode::all_updates()).unwrap(); + doc_b.import(&initial_state)?; + doc_c.import(&initial_state)?; + } + + { + // B: Edit text and list + let map = doc_b.get_map("root"); + let text = map + .insert_container("text", TextHandler::new_detached()) + .unwrap(); + text.insert(0, "Hello, ", PosType::Unicode)?; + + let list = map + .insert_container("list", ListHandler::new_detached()) + .unwrap(); + list.push("world")?; + } + + { + // C: Edit tree and movable list + let map = doc_c.get_map("root"); + let tree = map + .insert_container("tree", TreeHandler::new_detached()) + .unwrap(); + let node_id = tree.create(TreeParentId::Root)?; + tree.get_meta(node_id)?.insert("key", "value")?; + let node_b = tree.create(TreeParentId::Root)?; + tree.move_to(node_b, TreeParentId::Root, 0).unwrap(); + + let movable_list = map + .insert_container("movable", MovableListHandler::new_detached()) + .unwrap(); + movable_list.push("item1".into())?; + movable_list.push("item2".into())?; + movable_list.mov(0, 1)?; + } + + // Sync B's changes to A + let b_changes = doc_b + .export(ExportMode::updates(&doc_a.oplog_vv())) + .unwrap(); + doc_a.import(&b_changes)?; + + // Sync C's changes to A + let c_changes = doc_c + .export(ExportMode::updates(&doc_a.oplog_vv())) + .unwrap(); + doc_a.import(&c_changes)?; + + test_encode_decode(doc_a); + Ok(()) + } +} diff --git a/vendor/loro-internal/src/oplog/change_store/block_encode.rs b/vendor/loro-internal/src/oplog/change_store/block_encode.rs new file mode 100644 index 00000000..1d733f29 --- /dev/null +++ b/vendor/loro-internal/src/oplog/change_store/block_encode.rs @@ -0,0 +1,822 @@ +//! # Encode +//! +//! ```log +//! ≈4KB after compression +//! +//! N = Number of Changes +//! +//! Peer_1 = This Peer +//! +//! +//! ┌────────────────────────┬─────────────────────────────────────┐ +//! │ LEB Counter Start&Len │ LEB Lamport Start&Len │◁───┐ +//! └────────────────────────┴─────────────────────────────────────┘ │ +//! ┌──────────────┬──────────────┬────────────────────────────────┐ │ +//! │ LEB N │ LEB Peer Num │ 8B Peer_1,...,Peer_x │◁───┤ +//! └──────────────┴──────────────┴────────────────────────────────┘ │ +//! ┌──────────────────────────────────────────────────────────────┐ │ +//! │ N LEB128 Change AtomLen │◁───┼───── Important metadata +//! └──────────────────────────────────────────────────────────────┘ │ +//! ┌───────────────────┬────────────────────────┬─────────────────┐ │ +//! │N DepOnSelf BoolRle│ N Delta Rle Deps Lens │ Dep IDs │◁───┤ +//! └───────────────────┴────────────────────────┴─────────────────┘ │ +//! ┌──────────────────────────────────────────────────────────────┐ │ +//! │ N LEB128 Delta Lamports │◁───┘ +//! └──────────────────────────────────────────────────────────────┘ +//! ┌──────────────────────────────────────────────────────────────┐ +//! │ N LEB128 DeltaOfDelta Timestamps │ +//! └──────────────────────────────────────────────────────────────┘ +//! ┌────────────────────────────────┬─────────────────────────────┐ +//! │ N Rle Commit Msg Lengths │ Commit Messages │ +//! └────────────────────────────────┴─────────────────────────────┘ +//! +//! ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ Encoded Operations ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ +//! +//! ┌────────────────────┬─────────────────────────────────────────┐ +//! │ ContainerIDs Size │ ContainerIDs │ +//! └────────────────────┴─────────────────────────────────────────┘ +//! ┌────────────────────┬─────────────────────────────────────────┐ +//! │ Key Strings Size │ Key Strings │ +//! └────────────────────┴─────────────────────────────────────────┘ +//! ┌────────────────────┬─────────────────────────────────────────┐ +//! │ Position Size │ Position │ +//! └────────────────────┴─────────────────────────────────────────┘ +//! ┌────────┬──────────┬──────────┬───────┬───────────────────────┐ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! │ Ops │ LEB128 │ RLE │ Delta │ │ +//! │ Size │ Lengths │Containers│ RLE │ ValueType │ +//! │ │ │ │ Props │ │ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! │ │ │ │ │ │ +//! └────────┴──────────┴──────────┴───────┴───────────────────────┘ +//! ┌──────────────────────────────────────────────────────────────┐ +//! │ (Encoded with Ops By serde_columnar) │ +//! │ Delete Start IDs │ +//! │ │ +//! └──────────────────────────────────────────────────────────────┘ +//! ┌────────────────┬─────────────────────────────────────────────┐ +//! │ │ │ +//! │Value Bytes Size│ Value Bytes │ +//! │ │ │ +//! └────────────────┴─────────────────────────────────────────────┘ +//! ``` + +use std::borrow::Cow; +use std::collections::BTreeSet; +use std::io::Write; +use std::sync::Arc; + +use fractional_index::FractionalIndex; +use loro_common::{ + ContainerID, Counter, HasCounterSpan, HasLamportSpan, InternalString, Lamport, LoroError, + LoroResult, PeerID, TreeID, ID, +}; +use once_cell::sync::OnceCell; +use rle::HasLength; +use serde::{Deserialize, Serialize}; +use serde_columnar::{columnar, AnyRleDecoder, DeltaOfDeltaDecoder, Itertools}; +use tracing::info; + +use super::block_meta_encode::decode_changes_header; +use crate::arena::SharedArena; +use crate::change::{Change, Timestamp}; +use crate::container::tree::tree_op; +use crate::encoding::arena::{ContainerArena, PositionArena}; +use crate::encoding::value_register::ValueRegister; +use crate::encoding::{ + self, decode_op, encode_op, get_op_prop, EncodedDeleteStartId, IterableEncodedDeleteStartId, +}; +use crate::op::Op; + +#[derive(Debug, Serialize, Deserialize)] +struct EncodedBlock<'a> { + counter_start: u32, + counter_len: u32, + lamport_start: u32, + lamport_len: u32, + n_changes: u32, + #[serde(borrow)] + header: Cow<'a, [u8]>, + // timestamp and commit messages + #[serde(borrow)] + change_meta: Cow<'a, [u8]>, + // ---------------------- Ops ---------------------- + #[serde(borrow)] + cids: Cow<'a, [u8]>, + #[serde(borrow)] + keys: Cow<'a, [u8]>, + #[serde(borrow)] + positions: Cow<'a, [u8]>, + #[serde(borrow)] + ops: Cow<'a, [u8]>, + #[serde(borrow)] + delete_start_ids: Cow<'a, [u8]>, + #[serde(borrow)] + values: Cow<'a, [u8]>, +} + +fn diagnose_block(block: &EncodedBlock) { + info!("Diagnosing EncodedBlock:"); + info!(" header {} bytes", block.header.len()); + info!(" change_meta {} bytes", block.change_meta.len()); + info!(" cids: {} bytes", block.cids.len()); + info!(" keys: {} bytes", block.keys.len()); + info!(" positions: {} bytes", block.positions.len()); + info!(" ops: {} bytes", block.ops.len()); + info!(" delete_id_starts: {} bytes", block.delete_start_ids.len()); + info!(" values: {} bytes", block.values.len()); +} + +const VERSION: u16 = 0; + +// MARK: encode_block +/// It's assume that every change in the block share the same peer. +pub fn encode_block(block: &[Change], arena: &SharedArena) -> Vec { + if block.is_empty() { + panic!("Empty block") + } + + let mut peer_register: ValueRegister = ValueRegister::new(); + let peer = block[0].peer(); + peer_register.register(&peer); + + let cid_register: ValueRegister = ValueRegister::new(); + + let mut encoded_ops = Vec::new(); + let mut registers = Registers { + peer_register, + key_register: ValueRegister::new(), + cid_register, + position_register: ValueRegister::new(), + }; + + { + // Init position register, making it ordered by fractional index + let mut position_set = BTreeSet::default(); + for c in block { + for op in c.ops().iter() { + if let crate::op::InnerContent::Tree(tree_op) = &op.content { + match &**tree_op { + tree_op::TreeOp::Create { position, .. } => { + position_set.insert(position.clone()); + } + tree_op::TreeOp::Move { position, .. } => { + position_set.insert(position.clone()); + } + tree_op::TreeOp::Delete { .. } => {} + } + } + } + } + + for position in position_set { + registers.position_register.register(&position); + } + } + + let mut del_starts: Vec<_> = Vec::new(); + let mut value_writer = ValueWriter::new(); + for c in block { + for op in c.ops().iter() { + let cid = arena.get_container_id(op.container).unwrap(); + let cidx = registers.cid_register.register(&cid); + let prop = get_op_prop(op, &mut registers); + let value_kind = encode_op( + op, + arena, + &mut del_starts, + &mut value_writer, + &mut registers, + ); + encoded_ops.push(EncodedOp { + container_index: cidx as u32, + prop, + value_type: value_kind.to_u8(), + len: op.atom_len() as u32, + }) + } + } + + let cids = registers.cid_register.unwrap_vec(); + let container_arena = ContainerArena::from_containers( + cids, + &mut registers.peer_register, + &mut registers.key_register, + ); + + // Write to output + + // ┌────────────────────┬─────────────────────────────────────────┐ + // │ Key Strings Size │ Key Strings │ + // └────────────────────┴─────────────────────────────────────────┘ + let keys = registers.key_register.unwrap_vec(); + let keys_bytes = encode_keys(keys); + + // ┌────────────────────┬─────────────────────────────────────────┐ + // │ Position Size │ Position │ + // └────────────────────┴─────────────────────────────────────────┘ + let position_vec = registers.position_register.unwrap_vec(); + let positions = PositionArena::from_positions(position_vec.iter().map(|p| p.as_bytes())); + let position_bytes = positions.encode_v2(); + + // ┌──────────┬──────────┬───────┬────────────────────────────────┐ + // │ │ │ │ │ + // │ │ │ │ │ + // │ │ │ │ │ + // │ LEB128 │ RLE │ Delta │ │ + // │ Lengths │Containers│ RLE │ ValueType │ + // │ │ │ Props │ │ + // │ │ │ │ │ + // │ │ │ │ │ + // │ │ │ │ │ + // └──────────┴──────────┴───────┴────────────────────────────────┘ + + let ops_bytes = serde_columnar::to_vec(&EncodedOps { ops: encoded_ops }).unwrap(); + + let delete_id_starts_bytes = if del_starts.is_empty() { + Vec::new() + } else { + serde_columnar::to_vec(&EncodedDeleteStartIds { + delete_start_ids: del_starts, + }) + .unwrap() + }; + // ┌────────────────┬─────────────────────────────────────────────┐ + // │Value Bytes Size│ Value Bytes │ + // └────────────────┴─────────────────────────────────────────────┘ + + // PeerIDs + let mut peer_register = registers.peer_register; + // .unwrap_vec(); + // let peer_bytes: Vec = peers.iter().flat_map(|p| p.to_le_bytes()).collect(); + + // Change meta + let (header, change_meta) = encode_changes(block, &mut peer_register); + + let value_bytes = value_writer.finish(); + let out = EncodedBlock { + counter_start: block[0].id.counter as u32, + counter_len: (block.last().unwrap().ctr_end() - block[0].id.counter) as u32, + lamport_start: block[0].lamport(), + lamport_len: block.last().unwrap().lamport_end() - block[0].lamport(), + n_changes: block.len() as u32, + header: header.into(), + change_meta: change_meta.into(), + cids: container_arena.encode().into(), + keys: keys_bytes.into(), + positions: position_bytes.into(), + ops: ops_bytes.into(), + delete_start_ids: delete_id_starts_bytes.into(), + values: value_bytes.into(), + }; + + diagnose_block(&out); + postcard::to_allocvec(&out).unwrap() +} + +fn encode_keys(keys: Vec) -> Vec { + let mut keys_bytes = Vec::new(); + for key in keys { + let bytes = key.as_bytes(); + leb128::write::unsigned(&mut keys_bytes, bytes.len() as u64).unwrap(); + keys_bytes.write_all(bytes).unwrap(); + } + keys_bytes +} + +fn decode_keys(mut bytes: &[u8]) -> LoroResult> { + let mut keys = Vec::new(); + while !bytes.is_empty() { + let len = leb128::read::unsigned(&mut bytes) + .map_err(|_| LoroError::DecodeDataCorruptionError)? as usize; + if bytes.len() < len { + return Err(LoroError::DecodeDataCorruptionError); + } + let key = + std::str::from_utf8(&bytes[..len]).map_err(|_| LoroError::DecodeDataCorruptionError)?; + keys.push(key.into()); + bytes = &bytes[len..]; + } + + Ok(keys) +} + +struct Registers { + peer_register: ValueRegister, + key_register: ValueRegister, + cid_register: ValueRegister, + position_register: ValueRegister, +} + +use crate::encoding::value::{ + RawTreeMove, Value, ValueDecodedArenasTrait, ValueEncodeRegister, ValueKind, ValueReader, + ValueWriter, +}; +use crate::oplog::change_store::block_meta_encode::encode_changes; +use crate::version::Frontiers; +impl ValueEncodeRegister for Registers { + fn key_mut(&mut self) -> &mut ValueRegister { + &mut self.key_register + } + + fn peer_mut(&mut self) -> &mut ValueRegister { + &mut self.peer_register + } + + fn encode_tree_op(&mut self, op: &tree_op::TreeOp) -> encoding::value::Value<'static> { + match op { + tree_op::TreeOp::Create { + target, + parent, + position, + } => encoding::value::Value::RawTreeMove(RawTreeMove { + subject_peer_idx: self.peer_register.register(&target.peer), + subject_cnt: target.counter, + is_parent_null: parent.is_none(), + parent_peer_idx: parent.map_or(0, |p| self.peer_register.register(&p.peer)), + parent_cnt: parent.map_or(0, |p| p.counter), + position_idx: self.position_register.register(position), + }), + tree_op::TreeOp::Move { + target, + parent, + position, + } => encoding::value::Value::RawTreeMove(RawTreeMove { + subject_peer_idx: self.peer_register.register(&target.peer), + subject_cnt: target.counter, + is_parent_null: parent.is_none(), + parent_peer_idx: parent.map_or(0, |p| self.peer_register.register(&p.peer)), + parent_cnt: parent.map_or(0, |p| p.counter), + position_idx: self.position_register.register(position), + }), + tree_op::TreeOp::Delete { target } => { + let parent = TreeID::delete_root(); + encoding::value::Value::RawTreeMove(RawTreeMove { + subject_peer_idx: self.peer_register.register(&target.peer), + subject_cnt: target.counter, + is_parent_null: false, + parent_peer_idx: self.peer_register.register(&parent.peer), + parent_cnt: parent.counter, + position_idx: 0, + }) + } + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct ChangesBlockHeader { + pub peer: PeerID, + pub counter: Counter, + pub n_changes: usize, + pub peers: Vec, + /// This has n + 1 elements, where counters[n] is the end counter of the + /// last change in the block. + /// + /// You can infer the size of kth change by taking counters[k + 1] - counters[k] + pub counters: Vec, + /// This has n elements + pub lamports: Vec, + pub deps_groups: Vec, + pub keys: OnceCell>, + pub cids: OnceCell>, +} + +pub fn decode_header(m_bytes: &[u8]) -> LoroResult { + let doc = postcard::from_bytes(m_bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))?; + + decode_header_from_doc(&doc) +} + +// MARK: decode_block_from_doc +fn decode_header_from_doc(doc: &EncodedBlock) -> Result { + let EncodedBlock { + n_changes, + header, + counter_len, + counter_start, + lamport_len, + lamport_start, + .. + } = doc; + let ans: ChangesBlockHeader = decode_changes_header( + header, + *n_changes as usize, + *counter_start as Counter, + *counter_len as Counter, + *lamport_start, + *lamport_len, + )?; + Ok(ans) +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +struct EncodedOp { + #[columnar(strategy = "DeltaRle")] + container_index: u32, + #[columnar(strategy = "DeltaRle")] + prop: i32, + #[columnar(strategy = "Rle")] + value_type: u8, + #[columnar(strategy = "Rle")] + len: u32, +} + +#[columnar(ser, de)] +struct EncodedOps { + #[columnar(class = "vec", iter = "EncodedOp")] + ops: Vec, +} + +#[columnar(ser, de)] +struct EncodedDeleteStartIds { + #[columnar(class = "vec", iter = "EncodedDeleteStartId")] + delete_start_ids: Vec, +} + +struct ValueDecodeArena<'a> { + peers: &'a [PeerID], + keys: &'a [InternalString], +} + +impl ValueDecodedArenasTrait for ValueDecodeArena<'_> { + fn keys(&self) -> &[InternalString] { + self.keys + } + + fn peers(&self) -> &[PeerID] { + self.peers + } + + fn decode_tree_op( + &self, + _positions: &[Vec], + _op: encoding::value::EncodedTreeMove, + _id: ID, + ) -> LoroResult { + Err(LoroError::DecodeDataCorruptionError) + } +} + +pub fn decode_block_range( + mut bytes: &[u8], +) -> LoroResult<((Counter, Counter), (Lamport, Lamport))> { + let version = leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Failed to read version: {e}").into_boxed_str()) + })?; + + if version as u16 != VERSION { + return Err(LoroError::DecodeError( + "Version mismatch".to_string().into_boxed_str(), + )); + } + + let counter_start = leb128::read::unsigned(&mut bytes).unwrap() as Counter; + let counter_len = leb128::read::unsigned(&mut bytes).unwrap() as Counter; + let lamport_start = leb128::read::unsigned(&mut bytes).unwrap() as Lamport; + let lamport_len = leb128::read::unsigned(&mut bytes).unwrap() as Lamport; + Ok(( + (counter_start, counter_start + counter_len), + (lamport_start, lamport_start + lamport_len), + )) +} + +/// Ensure the cids in header are decoded +#[allow(unused)] +pub fn decode_cids( + bytes: &[u8], + header: Option, +) -> LoroResult { + let doc = postcard::from_bytes(bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))?; + let header = if let Some(h) = header { + h + } else { + let doc = postcard::from_bytes(bytes).map_err(|e| { + LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()) + })?; + decode_header_from_doc(&doc)? + }; + + if header.cids.get().is_some() { + return Ok(header); + } + + let EncodedBlock { cids, keys, .. } = doc; + let keys = header.keys.get_or_try_init(|| decode_keys(&keys))?; + let decode_arena = ValueDecodeArena { + peers: &header.peers, + keys, + }; + + header.cids.get_or_try_init(|| { + ContainerArena::decode(&cids)? + .iter() + .map(|x| x.as_container_id(&decode_arena)) + .try_collect() + })?; + Ok(header) +} + +// MARK: decode_block +pub fn decode_block( + m_bytes: &[u8], + shared_arena: &SharedArena, + header: Option<&ChangesBlockHeader>, +) -> LoroResult> { + let doc = postcard::from_bytes(m_bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))?; + let header_on_stack; + let header = if let Some(header) = header { + header + } else { + header_on_stack = decode_header_from_doc(&doc)?; + &header_on_stack + }; + let EncodedBlock { + n_changes, + counter_start: first_counter, + change_meta, + cids, + keys, + ops, + delete_start_ids, + values, + positions, + .. + } = doc; + let n_changes = n_changes as usize; + let mut changes = Vec::with_capacity(n_changes); + let timestamp_decoder = DeltaOfDeltaDecoder::::new(&change_meta) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let (timestamps, bytes) = timestamp_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let commit_msg_len_decoder = AnyRleDecoder::::new(bytes); + let (commit_msg_lens, commit_msgs) = commit_msg_len_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeDataCorruptionError)?; + let mut commit_msg_index = 0u32; + let keys = header.keys.get_or_try_init(|| decode_keys(&keys))?; + let decode_arena = ValueDecodeArena { + peers: &header.peers, + keys, + }; + let positions = PositionArena::decode_v2(&positions)?; + let positions = positions.try_parse_to_positions()?; + let cids: &Vec = header.cids.get_or_try_init(|| { + ContainerArena::decode(&cids)? + .iter() + .map(|x| x.as_container_id(&decode_arena)) + .try_collect() + })?; + let mut value_reader = ValueReader::new(&values); + let encoded_ops_iters = serde_columnar::iter_from_bytes::(&ops)?; + let op_iter = encoded_ops_iters.ops; + let encoded_delete_id_starts: EncodedDeleteStartIds = if delete_start_ids.is_empty() { + EncodedDeleteStartIds { + delete_start_ids: Vec::new(), + } + } else { + serde_columnar::from_bytes(&delete_start_ids)? + }; + let mut del_iter = encoded_delete_id_starts + .delete_start_ids + .into_iter() + .map(Ok); + for i in 0..n_changes { + let commit_msg: Option> = { + let len = *commit_msg_lens + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)?; + if len == 0 { + None + } else { + let end = commit_msg_index + .checked_add(len) + .ok_or(LoroError::DecodeDataCorruptionError)?; + if commit_msgs.len() < end as usize { + return Err(LoroError::DecodeDataCorruptionError); + } + match std::str::from_utf8(&commit_msgs[commit_msg_index as usize..end as usize]) { + Ok(s) => { + commit_msg_index = end; + Some(Arc::from(s)) + } + Err(_) => { + tracing::error!("Invalid UTF8 String"); + return LoroResult::Err(LoroError::DecodeDataCorruptionError); + } + } + } + }; + changes.push(Change { + ops: Default::default(), + deps: header + .deps_groups + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)? + .clone(), + id: ID::new( + header.peer, + *header + .counters + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)?, + ), + lamport: *header + .lamports + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)?, + timestamp: *timestamps + .get(i) + .ok_or(LoroError::DecodeDataCorruptionError)? as Timestamp, + commit_msg, + }) + } + + let mut counter = first_counter as Counter; + let mut change_index = 0; + let peer = header.peer; + for op in op_iter { + let EncodedOp { + container_index, + prop, + value_type, + len, + } = op?; + let value = Value::decode( + ValueKind::from_u8(value_type), + &mut value_reader, + &decode_arena, + ID::new(peer, counter), + )?; + + let cid = cids + .get(container_index as usize) + .ok_or(LoroError::DecodeDataCorruptionError)?; + let content = decode_op( + cid, + value, + &mut del_iter, + shared_arena, + &decode_arena, + &positions, + prop, + ID::new(peer, counter), + )?; + + let c_idx = shared_arena.register_container(cid); + let op = Op { + counter, + container: c_idx, + content, + }; + + changes + .get_mut(change_index) + .ok_or(LoroError::DecodeDataCorruptionError)? + .ops + .push(op); + counter = counter + .checked_add(len as Counter) + .ok_or(LoroError::DecodeDataCorruptionError)?; + let next_counter = header + .counters + .get(change_index + 1) + .ok_or(LoroError::DecodeDataCorruptionError)?; + if counter >= *next_counter { + change_index += 1; + } + } + Ok(changes) +} + +#[cfg(test)] +mod test { + use std::{borrow::Cow, panic::AssertUnwindSafe}; + + use loro_common::ContainerType; + + use crate::encoding::value::{EncodedTreeMove, Value}; + use crate::{cursor::PosType, delta::DeltaValue, loro::ExportMode, LoroDoc}; + + use super::*; + + #[test] + fn decode_block_rejects_corrupt_payload_without_panic() { + let doc = LoroDoc::new_auto_commit(); + doc.get_map("map").insert("x", 100).unwrap(); + doc.commit_then_renew(); + + let oplog = doc.oplog.lock(); + let mut changes = Vec::new(); + oplog + .change_store() + .visit_all_changes(&mut |change| changes.push(change.clone())); + let block_bytes = encode_block(&changes, &oplog.arena); + let mut encoded: EncodedBlock = postcard::from_bytes(&block_bytes).unwrap(); + encoded.change_meta = Cow::Owned(vec![0x80]); + let corrupt = postcard::to_allocvec(&encoded).unwrap(); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + decode_block(&corrupt, &oplog.arena, None) + })); + assert!(result.is_ok(), "corrupt block payload should not panic"); + assert!(result.unwrap().is_err()); + } + + #[test] + fn tree_move_payload_is_rejected_without_panic() { + let arena = SharedArena::new(); + let value_arena = ValueDecodeArena { + peers: &[1], + keys: &[], + }; + let cid = ContainerID::Root { + name: "tree".into(), + container_type: ContainerType::Tree, + }; + let value = Value::TreeMove(EncodedTreeMove { + target_idx: 0, + is_parent_null: true, + parent_idx: 0, + position: 0, + }); + let mut deletes = std::iter::empty(); + + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + decode_op( + &cid, + value, + &mut deletes, + &arena, + &value_arena, + &[], + 0, + ID::new(1, 0), + ) + })); + + assert!(result.is_ok(), "tree move payload should not panic"); + assert!(matches!( + result.unwrap(), + Err(LoroError::DecodeDataCorruptionError) + )); + } + + #[test] + pub fn encode_single_text_edit() { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + doc.get_map("map").insert("x", 100).unwrap(); + // doc.get_text("text").insert(0, "Hi").unwrap(); + // let node = doc.get_tree("tree").create(None).unwrap(); + // doc.get_tree("tree").create(node).unwrap(); + diagnose(&doc); + doc.get_map("map").insert("y", 20).unwrap(); + diagnose(&doc); + doc.get_map("map").insert("z", 1000).unwrap(); + diagnose(&doc); + doc.get_text("text") + .insert(0, "Hello", PosType::Unicode) + .unwrap(); + diagnose(&doc); + doc.get_text("text") + .insert(2, "He", PosType::Unicode) + .unwrap(); + diagnose(&doc); + doc.get_text("text").delete(1, 4, PosType::Unicode).unwrap(); + diagnose(&doc); + doc.get_text("text").delete(0, 2, PosType::Unicode).unwrap(); + diagnose(&doc); + } + + fn diagnose(doc: &LoroDoc) { + let bytes = doc.export(ExportMode::all_updates()).unwrap(); + println!("Old Update bytes {:?}", dev_utils::ByteSize(bytes.length())); + + let bytes = doc.export(ExportMode::all_updates()).unwrap(); + println!("Update bytes {:?}", dev_utils::ByteSize(bytes.length())); + // assert!(bytes.len() < 30); + + let bytes = doc.export(ExportMode::Snapshot).unwrap(); + println!("Snapshot bytes {:?}", dev_utils::ByteSize(bytes.length())); + // assert!(bytes.len() < 30); + + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), true); + let json_string = serde_json::to_string(&json.changes).unwrap(); + println!( + "JSON string bytes {:?}", + dev_utils::ByteSize(json_string.len()) + ); + let bytes = postcard::to_allocvec(&json.changes).unwrap(); + println!("JSON update bytes {:?}", dev_utils::ByteSize(bytes.len())); + println!("\n\n") + } +} diff --git a/vendor/loro-internal/src/oplog/change_store/block_meta_encode.rs b/vendor/loro-internal/src/oplog/change_store/block_meta_encode.rs new file mode 100644 index 00000000..ef9695f7 --- /dev/null +++ b/vendor/loro-internal/src/oplog/change_store/block_meta_encode.rs @@ -0,0 +1,247 @@ +use loro_common::{Counter, Lamport, LoroError, LoroResult, PeerID, ID}; +use once_cell::sync::OnceCell; +use rle::HasLength; +use serde_columnar::{ + AnyRleDecoder, AnyRleEncoder, BoolRleDecoder, BoolRleEncoder, DeltaOfDeltaDecoder, + DeltaOfDeltaEncoder, +}; + +use crate::{change::Change, encoding::value_register::ValueRegister, version::Frontiers}; + +use super::block_encode::ChangesBlockHeader; + +pub(crate) fn encode_changes( + block: &[Change], + peer_register: &mut ValueRegister, +) -> (Vec, Vec) { + let peer = block[0].peer(); + let mut timestamp_encoder = DeltaOfDeltaEncoder::new(); + let mut lamport_encoder = DeltaOfDeltaEncoder::new(); + let mut commit_msg_len_encoder = AnyRleEncoder::::new(); + let mut commit_msgs = String::new(); + let mut dep_self_encoder = BoolRleEncoder::new(); + let mut dep_len_encoder = AnyRleEncoder::::new(); + let mut encoded_deps = EncodedDeps { + peer_idx: AnyRleEncoder::new(), + counter: DeltaOfDeltaEncoder::new(), + }; + // First Counter + Change Len + let mut lengths_bytes = Vec::new(); + let mut counter = vec![]; + + for (i, c) in block.iter().enumerate() { + counter.push(c.id.counter); + let is_last = i == block.len() - 1; + if !is_last { + leb128::write::unsigned(&mut lengths_bytes, c.atom_len() as u64).unwrap(); + lamport_encoder.append(c.lamport() as i64).unwrap(); + } + timestamp_encoder.append(c.timestamp()).unwrap(); + if let Some(msg) = c.commit_msg.as_ref() { + commit_msg_len_encoder.append(msg.len() as u32).unwrap(); + commit_msgs.push_str(msg); + } else { + commit_msg_len_encoder.append(0).unwrap(); + } + + let mut dep_on_self = false; + for dep in c.deps().iter() { + if dep.peer == peer { + dep_on_self = true; + } else { + let peer_idx = peer_register.register(&dep.peer); + encoded_deps.peer_idx.append(peer_idx as u32).unwrap(); + encoded_deps.counter.append(dep.counter as i64).unwrap(); + } + } + + dep_self_encoder.append(dep_on_self).unwrap(); + dep_len_encoder + .append(if dep_on_self { + c.deps().len() - 1 + } else { + c.deps().len() + }) + .unwrap(); + } + + // TODO: capacity + let mut ans = Vec::with_capacity(block.len() * 15); + let _ = leb128::write::unsigned(&mut ans, peer_register.vec().len() as u64); + ans.extend(peer_register.vec().iter().flat_map(|p| p.to_le_bytes())); + ans.append(&mut lengths_bytes); + ans.append(&mut dep_self_encoder.finish().unwrap()); + ans.append(&mut dep_len_encoder.finish().unwrap()); + ans.append(&mut encoded_deps.peer_idx.finish().unwrap()); + ans.append(&mut encoded_deps.counter.finish().unwrap()); + ans.append(&mut lamport_encoder.finish().unwrap()); + + let mut t = timestamp_encoder.finish().unwrap(); + let mut cml = commit_msg_len_encoder.finish().unwrap(); + let mut cms = commit_msgs.into_bytes(); + let mut meta = Vec::with_capacity(t.len() + cml.len() + cms.len()); + meta.append(&mut t); + meta.append(&mut cml); + meta.append(&mut cms); + + (ans, meta) +} + +pub(crate) fn decode_changes_header( + mut bytes: &[u8], + n_changes: usize, + first_counter: Counter, + counter_len: Counter, + lamport_start: Lamport, + lamport_len: Lamport, +) -> LoroResult { + if n_changes == 0 { + return Err(LoroError::DecodeError( + "Decode block error: empty change block".into(), + )); + } + + let mut this_counter = first_counter; + let peer_num = leb128::read::unsigned(&mut bytes) + .map_err(|e| LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()))? + as usize; + let peers_bytes_len = peer_num + .checked_mul(8) + .ok_or_else(|| LoroError::DecodeError("Decode block error: peer length overflow".into()))?; + if peer_num == 0 || bytes.len() < peers_bytes_len { + return Err(LoroError::DecodeError( + "Decode block error: invalid peer table".into(), + )); + } + let mut peers = Vec::with_capacity(peer_num); + for i in 0..peer_num { + let start = 8 * i; + let peer_id = PeerID::from_le_bytes( + bytes[start..start + 8] + .try_into() + .expect("peer byte range should be checked"), + ); + peers.push(peer_id); + } + let mut bytes = &bytes[peers_bytes_len..]; + + // ┌───────────────────┬──────────────────────────────────────────┐ │ + // │ LEB First Counter │ N LEB128 Change AtomLen │◁───┼───── Important metadata + // └───────────────────┴──────────────────────────────────────────┘ │ + + let mut lengths = Vec::with_capacity(n_changes); + for _ in 0..n_changes - 1 { + lengths.push(leb128::read::unsigned(&mut bytes).map_err(|e| { + LoroError::DecodeError(format!("Decode block error {e}").into_boxed_str()) + })? as Counter); + } + let known_len = lengths.iter().try_fold(0i32, |acc, len| { + acc.checked_add(*len).ok_or_else(|| { + LoroError::DecodeError("Decode block error: counter length overflow".into()) + }) + })?; + lengths.push(counter_len.checked_sub(known_len).ok_or_else(|| { + LoroError::DecodeError("Decode block error: invalid counter length".into()) + })?); + + // ┌───────────────────┬────────────────────────┬─────────────────┐ │ + // │N DepOnSelf BoolRle│ N Delta Rle Deps Lens │ N Dep IDs │◁───┘ + // └───────────────────┴────────────────────────┴─────────────────┘ + + let dep_self_decoder = BoolRleDecoder::new(bytes); + let (dep_self, bytes) = dep_self_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let dep_len_decoder = AnyRleDecoder::::new(bytes); + let (deps_len, bytes) = dep_len_decoder + .take_n_finalize(n_changes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let other_dep_num = deps_len.iter().sum::(); + let dep_peer_decoder = AnyRleDecoder::::new(bytes); + let (dep_peers, bytes) = dep_peer_decoder + .take_n_finalize(other_dep_num) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let mut deps_peers_iter = dep_peers.into_iter(); + let dep_counter_decoder = DeltaOfDeltaDecoder::::new(bytes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let (dep_counters, bytes) = dep_counter_decoder + .take_n_finalize(other_dep_num) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let mut deps_counters_iter = dep_counters.into_iter(); + let mut deps = Vec::with_capacity(n_changes); + for i in 0..n_changes { + let mut f = Frontiers::default(); + if dep_self[i] { + let dep_counter = this_counter.checked_sub(1).ok_or_else(|| { + LoroError::DecodeError("Decode block error: invalid self dependency".into()) + })?; + f.push(ID::new(peers[0], dep_counter)) + } + + let len = deps_len[i]; + for _ in 0..len { + let peer_idx = deps_peers_iter + .next() + .ok_or_else(|| LoroError::DecodeError("Decode block error: invalid deps".into()))?; + let peer = peers.get(peer_idx).copied().ok_or_else(|| { + LoroError::DecodeError("Decode block error: invalid peer index".into()) + })?; + let counter = deps_counters_iter + .next() + .ok_or_else(|| LoroError::DecodeError("Decode block error: invalid deps".into()))? + as Counter; + f.push(ID::new(peer, counter)); + } + + deps.push(f); + this_counter = this_counter + .checked_add(lengths[i]) + .ok_or_else(|| LoroError::DecodeError("Decode block error: counter overflow".into()))?; + } + let mut counters = Vec::with_capacity(n_changes); + let mut last = first_counter; + for len in lengths.iter() { + counters.push(last); + last = last + .checked_add(*len) + .ok_or_else(|| LoroError::DecodeError("Decode block error: counter overflow".into()))?; + } + + let lamport_decoder = DeltaOfDeltaDecoder::new(bytes) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid lamport".into()))?; + let (mut lamports, rest) = lamport_decoder + .take_n_finalize(n_changes.saturating_sub(1)) + .map_err(|_| LoroError::DecodeError("Decode block error: invalid lamport".into()))?; + // the last lamport + let last_len = *lengths.last().unwrap_or(&0) as u32; + let last_lamport = lamport_start + .checked_add(lamport_len) + .and_then(|end| end.checked_sub(last_len)) + .ok_or_else(|| LoroError::DecodeError("Decode block error: invalid lamport".into()))?; + lamports.push(last_lamport as Lamport); + + // we need counter range, so encode + counters.push( + first_counter + .checked_add(counter_len) + .ok_or_else(|| LoroError::DecodeError("Decode block error: counter overflow".into()))?, + ); + debug_assert!(rest.is_empty()); + + Ok(ChangesBlockHeader { + peer: peers[0], + counter: first_counter, + n_changes, + peers, + counters, + deps_groups: deps, + lamports, + keys: OnceCell::new(), + cids: OnceCell::new(), + }) +} + +struct EncodedDeps { + peer_idx: AnyRleEncoder, + counter: DeltaOfDeltaEncoder, +} diff --git a/vendor/loro-internal/src/oplog/change_store/iter.rs b/vendor/loro-internal/src/oplog/change_store/iter.rs new file mode 100644 index 00000000..153eaf0a --- /dev/null +++ b/vendor/loro-internal/src/oplog/change_store/iter.rs @@ -0,0 +1,139 @@ +#![allow(dead_code)] + +use std::{ + collections::{BinaryHeap, VecDeque}, + sync::Arc, +}; + +use loro_common::{CounterSpan, HasIdSpan}; + +use crate::{OpLog, VersionVector}; + +use super::{BlockChangeRef, ChangesBlock}; + +pub(crate) struct PeerChangesIter { + blocks: VecDeque>, + change_index: usize, + counter_range: CounterSpan, +} + +impl PeerChangesIter { + fn new_change_iter_rev(mut changes: VecDeque>, counter: CounterSpan) -> Self { + let mut index = changes + .back() + .map(|x| x.content().len_changes().saturating_sub(1)) + .unwrap_or(0); + + while let Some(block) = changes.back() { + if let Some(change) = block.content().try_changes().unwrap().get(index) { + if change.id.counter < counter.end { + break; + } + } else if index == 0 { + changes.pop_back(); + } else { + index -= 1; + } + } + + PeerChangesIter { + blocks: changes, + change_index: index, + counter_range: counter, + } + } + + fn current_weight(&self) -> i32 { + if self.blocks.is_empty() { + return 0; + } + + self.blocks + .back() + .map(|x| { + x.content() + .try_changes() + .unwrap() + .get(self.change_index) + .unwrap() + .lamport as i32 + }) + .unwrap_or(0) + } +} + +impl Ord for PeerChangesIter { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.current_weight().cmp(&other.current_weight()) + } +} + +impl PartialOrd for PeerChangesIter { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for PeerChangesIter { + fn eq(&self, other: &Self) -> bool { + self.current_weight() == other.current_weight() + } +} + +impl Eq for PeerChangesIter {} + +impl Iterator for PeerChangesIter { + type Item = BlockChangeRef; + + fn next(&mut self) -> Option { + if self.blocks.is_empty() { + return None; + } + + let c = BlockChangeRef { + block: self.blocks.back().unwrap().clone(), + change_index: self.change_index, + }; + + if c.id_last().counter < self.counter_range.start { + return None; + } + + let ans = Some(c); + if self.change_index == 0 { + self.blocks.pop_back(); + } else { + self.change_index -= 1; + } + + ans + } +} + +pub(crate) struct MergedChangeIter { + heap: BinaryHeap, +} + +impl Iterator for MergedChangeIter { + type Item = BlockChangeRef; + + fn next(&mut self) -> Option { + let mut iter = self.heap.pop()?; + let ans = iter.next(); + self.heap.push(iter); + ans + } +} + +impl MergedChangeIter { + pub fn new_change_iter_rev(oplog: &OpLog, from: &VersionVector, to: &VersionVector) -> Self { + let mut heap = BinaryHeap::new(); + for span in to.sub_iter(from) { + let blocks = oplog.change_store.get_blocks_in_range(span); + let iter = PeerChangesIter::new_change_iter_rev(blocks, span.counter); + heap.push(iter); + } + + Self { heap } + } +} diff --git a/vendor/loro-internal/src/oplog/loro_dag.rs b/vendor/loro-internal/src/oplog/loro_dag.rs new file mode 100644 index 00000000..d71a08e5 --- /dev/null +++ b/vendor/loro-internal/src/oplog/loro_dag.rs @@ -0,0 +1,1388 @@ +use crate::change::{Change, Lamport}; +use crate::dag::{Dag, DagNode}; +use crate::id::{Counter, ID}; +use crate::span::{HasId, HasLamport}; +use crate::sync::Mutex; +use crate::version::{shrink_frontiers, Frontiers, ImVersionVector, VersionVector}; +use loro_common::{HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, PeerID}; +use once_cell::sync::OnceCell; +use rle::{HasIndex, HasLength, Mergable, Sliceable}; +use rustc_hash::FxHashSet; +use smallvec::SmallVec; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet, BinaryHeap}; +use std::fmt::Display; +use std::ops::{ControlFlow, Deref}; +use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering}; +use std::sync::Arc; +use tracing::instrument; + +use super::change_store::BatchDecodeInfo; +use super::ChangeStore; + +/// [AppDag] maintains the causal graph of the app. +/// It's faster to answer the question like what's the LCA version +#[derive(Debug)] +pub struct AppDag { + change_store: ChangeStore, + /// It only contains nodes that are already parsed. + /// + /// - All the unparsed op ids must be included in `unparsed_vv`. + /// - All the parsed and unparsed op ids must be included in `vv`. + map: Mutex>, + /// The latest known frontiers + frontiers: Frontiers, + /// The latest known version vector + vv: VersionVector, + /// The earliest known frontiers + shallow_since_frontiers: Frontiers, + /// The deps of the shallow frontiers + shallow_root_frontiers_deps: Frontiers, + /// The vv of shallow_frontiers_deps + shallow_since_vv: ImVersionVector, + /// Ops included in the version vector but not parsed yet + /// + /// # Invariants + /// + /// - `vv` >= `unparsed_vv` + unparsed_vv: Mutex, + /// It's a set of points which are deps of some parsed ops. + /// But the ops in this set are not parsed yet. When they are parsed, + /// we need to make sure it breaks at the given point. + unhandled_dep_points: Mutex>, + pending_txn_node: Option, + import_rollback_has_journal: AtomicBool, + import_rollback: Mutex>, +} + +#[derive(Debug)] +pub(crate) struct AppDagRollback { + frontiers: Frontiers, + vv: VersionVector, + unparsed_vv: VersionVector, + shallow_since_frontiers: Frontiers, + shallow_root_frontiers_deps: Frontiers, + shallow_since_vv: ImVersionVector, + pending_txn_node: Option, + map_entries_before_mutation: BTreeMap>, + unhandled_dep_point_log: Vec, +} + +#[derive(Debug)] +enum UnhandledDepPointLog { + Added(ID), + Removed(ID), +} + +#[derive(Debug, Clone)] +pub struct AppDagNode { + inner: Arc, +} + +impl Deref for AppDagNode { + type Target = AppDagNodeInner; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl AppDagNode { + pub fn new(inner: AppDagNodeInner) -> Self { + Self { + inner: Arc::new(inner), + } + } +} + +#[derive(Debug, Clone)] +pub struct AppDagNodeInner { + pub(crate) peer: PeerID, + pub(crate) cnt: Counter, + pub(crate) lamport: Lamport, + pub(crate) deps: Frontiers, + pub(crate) vv: OnceCell, + /// A flag indicating whether any other nodes from a different peer depend on this node. + /// The calculation of frontiers is based on the property that a node does not depend + /// on the middle of other nodes. + pub(crate) has_succ: bool, + pub(crate) len: usize, +} + +impl From for AppDagNode { + fn from(inner: AppDagNodeInner) -> Self { + AppDagNode { + inner: Arc::new(inner), + } + } +} + +impl AppDag { + pub(super) fn new(change_store: ChangeStore) -> Self { + Self { + change_store, + map: Mutex::new(BTreeMap::new()), + frontiers: Frontiers::default(), + vv: VersionVector::default(), + unparsed_vv: Mutex::new(VersionVector::default()), + unhandled_dep_points: Mutex::new(BTreeSet::new()), + shallow_since_frontiers: Default::default(), + shallow_root_frontiers_deps: Default::default(), + shallow_since_vv: Default::default(), + pending_txn_node: None, + import_rollback_has_journal: AtomicBool::new(false), + import_rollback: Mutex::new(None), + } + } + + pub fn frontiers(&self) -> &Frontiers { + &self.frontiers + } + + pub fn vv(&self) -> &VersionVector { + &self.vv + } + + pub fn shallow_since_vv(&self) -> &ImVersionVector { + &self.shallow_since_vv + } + + pub fn shallow_since_frontiers(&self) -> &Frontiers { + &self.shallow_since_frontiers + } + + pub(crate) fn begin_import_rollback(&mut self) { + let old_vv_is_empty = self.vv.is_empty(); + let mut rollback = self.import_rollback.lock(); + debug_assert!(rollback.is_none()); + *rollback = Some(AppDagRollback { + frontiers: self.frontiers.clone(), + vv: self.vv.clone(), + unparsed_vv: self.unparsed_vv.lock().clone(), + shallow_since_frontiers: self.shallow_since_frontiers.clone(), + shallow_root_frontiers_deps: self.shallow_root_frontiers_deps.clone(), + shallow_since_vv: self.shallow_since_vv.clone(), + pending_txn_node: self.pending_txn_node.clone(), + map_entries_before_mutation: BTreeMap::new(), + unhandled_dep_point_log: Vec::new(), + }); + self.import_rollback_has_journal + .store(!old_vv_is_empty, AtomicOrdering::Relaxed); + } + + pub(crate) fn commit_import_rollback(&mut self) { + self.import_rollback_has_journal + .store(false, AtomicOrdering::Relaxed); + *self.import_rollback.lock() = None; + } + + pub(crate) fn rollback_import(&mut self) { + self.import_rollback_has_journal + .store(false, AtomicOrdering::Relaxed); + let Some(checkpoint) = self.import_rollback.lock().take() else { + return; + }; + + let imported_spans = self.vv.sub_iter(&checkpoint.vv).collect::>(); + let mut map = self.map.lock(); + for span in imported_spans { + let start = ID::new(span.peer, span.counter.start); + let end = ID::new(span.peer, span.counter.end); + let keys = map.range(start..end).map(|(id, _)| *id).collect::>(); + for key in keys { + map.remove(&key); + } + } + + for (id, node) in checkpoint.map_entries_before_mutation { + if let Some(node) = node { + map.insert(id, node); + } else { + map.remove(&id); + } + } + drop(map); + + self.frontiers = checkpoint.frontiers; + self.vv = checkpoint.vv.clone(); + self.shallow_since_frontiers = checkpoint.shallow_since_frontiers; + self.shallow_root_frontiers_deps = checkpoint.shallow_root_frontiers_deps; + self.shallow_since_vv = checkpoint.shallow_since_vv; + *self.unparsed_vv.lock() = checkpoint.unparsed_vv; + + let mut unhandled_dep_points = self.unhandled_dep_points.lock(); + if checkpoint.vv.is_empty() { + unhandled_dep_points.clear(); + } + for item in checkpoint.unhandled_dep_point_log.into_iter().rev() { + match item { + UnhandledDepPointLog::Added(id) => { + unhandled_dep_points.remove(&id); + } + UnhandledDepPointLog::Removed(id) => { + unhandled_dep_points.insert(id); + } + } + } + drop(unhandled_dep_points); + + self.pending_txn_node = checkpoint.pending_txn_node; + } + + fn record_map_entry_before_mutation(&self, map: &BTreeMap, id: ID) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + self.record_map_entry_before_replacement(id, map.get(&id).cloned()); + } + + fn record_map_entry_before_replacement(&self, id: ID, old: Option) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + let mut rollback = self.import_rollback.lock(); + let Some(rollback) = rollback.as_mut() else { + return; + }; + + let old_end = rollback.vv.get(&id.peer).copied().unwrap_or(0); + if id.counter >= old_end { + return; + } + + rollback + .map_entries_before_mutation + .entry(id) + .or_insert(old); + } + + fn record_unhandled_dep_point_added(&self, id: ID) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + if let Some(rollback) = self.import_rollback.lock().as_mut() { + rollback + .unhandled_dep_point_log + .push(UnhandledDepPointLog::Added(id)); + } + } + + fn record_unhandled_dep_point_removed(&self, id: ID) { + if !self + .import_rollback_has_journal + .load(AtomicOrdering::Relaxed) + { + return; + } + + if let Some(rollback) = self.import_rollback.lock().as_mut() { + rollback + .unhandled_dep_point_log + .push(UnhandledDepPointLog::Removed(id)); + } + } + + pub fn is_empty(&self) -> bool { + self.vv.is_empty() + } + + #[tracing::instrument(skip_all, name = "handle_new_change")] + pub(super) fn handle_new_change( + &mut self, + change: &Change, + from_local: bool, + rollback_old_vv: Option<&VersionVector>, + ) { + let len = change.content_len(); + self.update_version_on_new_change(change, from_local); + #[cfg(debug_assertions)] + { + let unhandled_dep_points = self.unhandled_dep_points.lock(); + let c = unhandled_dep_points + .range(change.id_start()..change.id_end()) + .count(); + assert!(c == 0); + } + + let mut inserted = false; + if change.deps_on_self() { + let record_old_node_before_merge = rollback_old_vv + .and_then(|vv| vv.get(&change.id.peer).copied()) + .filter(|old_end| *old_end == change.id.counter); + // We may not need to push new element to dag because it only depends on itself + inserted = + self.with_last_mut_of_peer(change.id.peer, record_old_node_before_merge, |last| { + let (_, last) = last.unwrap(); + if last.has_succ { + // Don't merge the node if there are other nodes depending on it + return false; + } + + assert_eq!(last.peer, change.id.peer, "peer id is not the same"); + assert_eq!( + last.cnt + last.len as Counter, + change.id.counter, + "counter is not continuous" + ); + assert_eq!( + last.lamport + last.len as Lamport, + change.lamport, + "lamport is not continuous" + ); + let last = Arc::make_mut(&mut last.inner); + last.len = (change.id.counter - last.cnt) as usize + len; + last.has_succ = false; + true + }); + } + + if !inserted { + let node: AppDagNode = AppDagNodeInner { + vv: OnceCell::new(), + peer: change.id.peer, + cnt: change.id.counter, + lamport: change.lamport, + deps: change.deps.clone(), + has_succ: false, + len, + } + .into(); + + let mut map = self.map.lock(); + map.insert(node.id_start(), node); + self.handle_deps_break_points(change.deps.iter(), change.id.peer, Some(&mut map)); + } + } + + fn try_with_node_mut( + &self, + map: &mut BTreeMap, + id: ID, + f: impl FnOnce(Option<(ID, &mut AppDagNode)>) -> R, + ) -> R { + let x = map.range_mut(..=id).next_back(); + if let Some((node_id, node)) = x { + if node.contains_id(id) { + f(Some((*node_id, node))) + } else { + f(None) + } + } else { + f(None) + } + } + + /// If the lamport of change can be calculated, return Ok, otherwise, Err + pub(crate) fn calc_unknown_lamport_change(&self, change: &mut Change) -> Result<(), ()> { + for dep in change.deps.iter() { + match self.get_lamport(&dep) { + Some(lamport) => { + change.lamport = change.lamport.max(lamport + 1); + } + None => return Err(()), + } + } + Ok(()) + } + + pub(crate) fn find_deps_of_id(&self, id: ID) -> Frontiers { + let Some(node) = self.get(id) else { + return Frontiers::default(); + }; + + let offset = id.counter - node.cnt; + if offset == 0 { + node.deps.clone() + } else { + ID::new(id.peer, node.cnt + offset - 1).into() + } + } + + pub(crate) fn with_last_mut_of_peer( + &mut self, + peer: PeerID, + record_if_before: Option, + f: impl FnOnce(Option<(ID, &mut AppDagNode)>) -> R, + ) -> R { + self.lazy_load_last_of_peer(peer); + let mut binding = self.map.lock(); + let last = binding + .range_mut(..=ID::new(peer, Counter::MAX)) + .next_back() + .map(|(id, v)| { + if record_if_before.is_some_and(|old_end| id.counter < old_end) { + self.record_map_entry_before_replacement(*id, Some(v.clone())); + } + (*id, v) + }); + f(last) + } + + fn update_version_on_new_change(&mut self, change: &Change, from_local: bool) { + if from_local { + assert!(self.pending_txn_node.take().is_some()); + assert_eq!( + self.vv.get(&change.id.peer).copied().unwrap_or(0), + change.ctr_end() + ); + } else { + let id_last = change.id_last(); + self.frontiers + .update_frontiers_on_new_change(id_last, &change.deps); + assert!(self.pending_txn_node.is_none()); + assert_eq!( + self.vv.get(&change.id.peer).copied().unwrap_or(0), + change.id.counter + ); + self.vv.extend_to_include_last_id(id_last); + } + } + + pub(super) fn lazy_load_last_of_peer(&mut self, peer: u64) { + let unparsed_vv = self.unparsed_vv.lock(); + if !unparsed_vv.contains_key(&peer) || self.vv[&peer] >= unparsed_vv[&peer] { + return; + } + + let Some(nodes) = self.change_store.get_last_dag_nodes_for_peer(peer) else { + panic!("unparsed vv don't match with change store. Peer:{peer} is not in change store") + }; + + self.lazy_load_nodes_internal(nodes, peer, None); + } + + fn lazy_load_nodes_internal( + &self, + nodes: Vec, + peer: u64, + map_input: Option<&mut BTreeMap>, + ) { + assert!(!nodes.is_empty()); + let mut map_guard = None; + let map = map_input.unwrap_or_else(|| { + map_guard = Some(self.map.lock()); + map_guard.as_mut().unwrap() + }); + let new_dag_start_counter_for_the_peer = nodes[0].cnt; + let nodes_cnt_end = nodes.last().unwrap().ctr_end(); + let mut unparsed_vv = self.unparsed_vv.lock(); + let end_counter = unparsed_vv[&peer]; + assert!(end_counter <= nodes_cnt_end); + let mut deps_on_others = Vec::new(); + let mut break_point_set = self.unhandled_dep_points.lock(); + for mut node in nodes { + if node.cnt >= end_counter { + // skip already parsed nodes + break; + } + + if node.cnt + node.len as Counter > end_counter { + node = node.slice(0, (end_counter - node.cnt) as usize); + // This is unlikely to happen + } + + for dep in node.deps.iter() { + if dep.peer != peer { + deps_on_others.push(dep); + } + } + + // PERF: we can try to merge the node with the previous node + let break_point_ends: Vec<_> = break_point_set + .range(node.id_start()..node.id_end()) + .map(|id| (id.counter - node.cnt) as usize + 1) + .collect(); + if break_point_ends.is_empty() { + self.record_map_entry_before_mutation(map, node.id_start()); + map.insert(node.id_start(), node); + } else { + let mut slice_start = 0; + for slice_end in break_point_ends.iter().copied() { + let mut slice_node = node.slice(slice_start, slice_end); + let inner = Arc::make_mut(&mut slice_node.inner); + inner.has_succ = true; + self.record_map_entry_before_mutation(map, slice_node.id_start()); + map.insert(slice_node.id_start(), slice_node); + slice_start = slice_end; + } + + let last_break_point = break_point_ends.last().copied().unwrap(); + if last_break_point != node.len { + let slice_node = node.slice(last_break_point, node.len); + self.record_map_entry_before_mutation(map, slice_node.id_start()); + map.insert(slice_node.id_start(), slice_node); + } + + for break_point in break_point_ends.into_iter() { + let id = node.id_start().inc(break_point as Counter - 1); + if break_point_set.remove(&id) { + self.record_unhandled_dep_point_removed(id); + } + } + } + } + + if new_dag_start_counter_for_the_peer == 0 { + unparsed_vv.remove(&peer); + } else { + unparsed_vv.insert(peer, new_dag_start_counter_for_the_peer); + } + drop(unparsed_vv); + drop(break_point_set); + self.handle_deps_break_points(deps_on_others.iter().copied(), peer, Some(map)); + } + + fn handle_deps_break_points( + &self, + ids: impl IntoIterator, + skip_peer: PeerID, + map: Option<&mut BTreeMap>, + ) { + let mut map_guard = None; + let map = map.unwrap_or_else(|| { + map_guard = Some(self.map.lock()); + map_guard.as_mut().unwrap() + }); + for id in ids { + if id.peer == skip_peer { + continue; + } + + let mut handled = false; + let ans = self.try_with_node_mut(map, id, |target| { + // We don't need to break the dag node if it's not loaded yet + let (target_id, target) = target?; + if target.ctr_last() == id.counter { + self.record_map_entry_before_replacement(target_id, Some(target.clone())); + let target = Arc::make_mut(&mut target.inner); + handled = true; + target.has_succ = true; + None + } else { + // We need to split the target node into two part + // so that we can ensure the new change depends on the + // last id of a dag node. + + let new_node = + target.slice(id.counter as usize - target.cnt as usize + 1, target.len); + self.record_map_entry_before_replacement(target_id, Some(target.clone())); + let target = Arc::make_mut(&mut target.inner); + target.len -= new_node.len; + Some(new_node) + } + }); + + if let Some(new_node) = ans { + self.record_map_entry_before_mutation(map, new_node.id_start()); + map.insert(new_node.id_start(), new_node); + } else if !handled { + let mut unhandled_dep_points = self.unhandled_dep_points.lock(); + if unhandled_dep_points.insert(id) { + self.record_unhandled_dep_point_added(id); + } + } + } + } + + fn ensure_lazy_load_node(&self, id: ID) { + if self.shallow_since_vv.includes_id(id) { + return; + } + + loop { + // We need to load all the dag nodes that has the same peer and greater counter than the given `id` + // Because we only record the end counter of the unparsed version on `unparsed_vv` + let unparsed_end = { + let unparsed_vv = self.unparsed_vv.lock(); + unparsed_vv.get(&id.peer).copied().unwrap_or(0) + }; + if unparsed_end <= id.counter { + return; + } + + let last_unparsed_id = ID::new(id.peer, unparsed_end - 1); + let Some(nodes) = self + .change_store + .get_dag_nodes_that_contains(last_unparsed_id) + else { + panic!("unparsed vv don't match with change store. Id:{id} is not in change store") + }; + + self.lazy_load_nodes_internal(nodes, id.peer, None); + } + } + + pub fn total_parsed_dag_node(&self) -> usize { + self.map.lock().len() + } + + pub(crate) fn set_version_by_fast_snapshot_import(&mut self, v: BatchDecodeInfo) { + assert!(self.vv.is_empty()); + *self.unparsed_vv.lock() = v.vv.clone(); + self.vv = v.vv; + self.frontiers = v.frontiers; + if let Some((vv, f)) = v.start_version { + if !f.is_empty() { + assert!(f.len() == 1); + let id = f.as_single().unwrap(); + let node = self.get(id).unwrap(); + assert!(node.cnt == id.counter); + self.shallow_root_frontiers_deps = node.deps.clone(); + } + self.shallow_since_frontiers = f; + self.shallow_since_vv = ImVersionVector::from_vv(&vv); + } + } + + /// This method is slow and should only be used for debugging and testing. + /// + /// It will check the following properties: + /// + /// 1. Counter is continuous + /// 2. A node always depends of the last ids of other nodes + /// 3. Lamport is correctly calculated + /// 4. VV for each node is correctly calculated + /// 5. Frontiers are correctly calculated + #[instrument(skip(self))] + pub fn check_dag_correctness(&self) { + { + // parse all nodes + let unparsed_vv = self.unparsed_vv.lock().clone(); + for (peer, cnt) in unparsed_vv.iter() { + if *cnt == 0 { + continue; + } + + let mut end_cnt = *cnt; + let init_counter = self.shallow_since_vv.get(peer).copied().unwrap_or(0); + while end_cnt > init_counter { + let cnt = end_cnt - 1; + self.ensure_lazy_load_node(ID::new(*peer, cnt)); + end_cnt = self.unparsed_vv.lock().get(peer).copied().unwrap_or(0); + } + } + + self.unparsed_vv.lock().clear(); + } + { + // check property 1: Counter is continuous + let map = self.map.lock(); + let mut last_end_id = ID::new(0, 0); + for (&id, node) in map.iter() { + let init_counter = self.shallow_since_vv.get(&id.peer).copied().unwrap_or(0); + if id.peer == last_end_id.peer { + assert!(id.counter == last_end_id.counter); + } else { + assert_eq!(id.counter, init_counter); + } + + last_end_id = id.inc(node.len as Counter); + } + } + { + // check property 2: A node always depends of the last ids of other nodes + let map = self.map.lock(); + check_always_dep_on_last_id(&map); + } + { + // check property 3: Lamport is correctly calculated + let map = self.map.lock(); + 'outer: for (_, node) in map.iter() { + let mut this_lamport = 0; + for dep in node.deps.iter() { + if self.shallow_since_vv.includes_id(dep) { + continue 'outer; + } + + let (_, dep_node) = map.range(..=dep).next_back().unwrap(); + this_lamport = this_lamport.max(dep_node.lamport_end()); + } + + assert_eq!(this_lamport, node.lamport); + } + } + { + // check property 4: VV for each node is correctly calculated + let map = self.map.lock().clone(); + 'outer: for (_, node) in map.iter() { + let actual_vv = self.ensure_vv_for(node); + let mut expected_vv = ImVersionVector::default(); + for dep in node.deps.iter() { + if self.shallow_since_vv.includes_id(dep) { + continue 'outer; + } + + let (_, dep_node) = map.range(..=dep).next_back().unwrap(); + self.ensure_vv_for(dep_node); + expected_vv.extend_to_include_vv(dep_node.vv.get().unwrap().iter()); + expected_vv.extend_to_include_last_id(dep); + } + + assert_eq!(actual_vv, expected_vv); + } + } + { + // check property 5: Frontiers are correctly calculated + let mut maybe_frontiers = FxHashSet::default(); + let map = self.map.lock(); + for (_, node) in map.iter() { + maybe_frontiers.insert(node.id_last()); + } + + for (_, node) in map.iter() { + for dep in node.deps.iter() { + maybe_frontiers.remove(&dep); + } + } + + let frontiers = self.frontiers.iter().collect::>(); + assert_eq!(maybe_frontiers, frontiers); + } + } + + pub(crate) fn can_export_shallow_snapshot_on(&self, deps: &Frontiers) -> bool { + for id in deps.iter() { + if !self.vv.includes_id(id) { + return false; + } + } + + if self.is_before_shallow_root(deps) { + return false; + } + + true + } + + pub(crate) fn is_before_shallow_root(&self, deps: &Frontiers) -> bool { + // trace!("Is on shallow history? deps={:?}", deps); + // trace!("self.shallow_since_vv {:?}", &self.shallow_since_vv); + // trace!("self.shallow_frontiers {:?}", &self.shallow_since_frontiers); + + if self.shallow_since_vv.is_empty() { + return false; + } + + if deps.is_empty() { + return true; + } + + if deps.iter().any(|x| self.shallow_since_vv.includes_id(x)) { + return true; + } + + if deps + .iter() + .any(|x| self.shallow_since_frontiers.contains(&x)) + { + return deps != &self.shallow_since_frontiers; + } + + false + } + + /// Travel the ancestors of the given id, and call the callback for each node + /// + /// It will travel the ancestors in the reverse order (from the greatest lamport to the smallest) + pub(crate) fn travel_ancestors( + &self, + id: ID, + f: &mut dyn FnMut(&AppDagNode) -> ControlFlow<()>, + ) { + struct PendingNode(AppDagNode); + impl PartialEq for PendingNode { + fn eq(&self, other: &Self) -> bool { + self.0.lamport_last() == other.0.lamport_last() && self.0.peer == other.0.peer + } + } + impl Eq for PendingNode {} + impl PartialOrd for PendingNode { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + impl Ord for PendingNode { + fn cmp(&self, other: &Self) -> Ordering { + self.0 + .lamport_last() + .cmp(&other.0.lamport_last()) + .then_with(|| self.0.peer.cmp(&other.0.peer)) + } + } + + let mut visited = FxHashSet::default(); + let mut pending: BinaryHeap = BinaryHeap::new(); + pending.push(PendingNode(self.get(id).unwrap())); + while let Some(PendingNode(node)) = pending.pop() { + if f(&node).is_break() { + break; + } + + for dep in node.deps.iter() { + let Some(dep_node) = self.get(dep) else { + continue; + }; + if visited.contains(&dep_node.id_start()) { + continue; + } + + visited.insert(dep_node.id_start()); + pending.push(PendingNode(dep_node)); + } + } + } + + pub(crate) fn update_version_on_new_local_op( + &mut self, + deps: &Frontiers, + start_id: ID, + start_lamport: Lamport, + len: usize, + ) { + let last_id = start_id.inc(len as Counter - 1); + // PERF: we can cache this last_id - this is a hot path + self.vv.set_last(last_id); + self.frontiers.update_frontiers_on_new_change(last_id, deps); + match &mut self.pending_txn_node { + Some(node) => { + assert!( + node.peer == start_id.peer + && node.cnt + node.len as Counter == start_id.counter + && deps.len() == 1 + && deps.as_single().unwrap().peer == start_id.peer + ); + let inner = Arc::make_mut(&mut node.inner); + inner.len += len; + } + None => { + let node = AppDagNode { + inner: Arc::new(AppDagNodeInner { + peer: start_id.peer, + cnt: start_id.counter, + lamport: start_lamport, + deps: deps.clone(), + vv: OnceCell::new(), + has_succ: false, + len, + }), + }; + self.pending_txn_node = Some(node); + } + } + } + + pub(crate) fn latest_vv_contains_peer(&self, peer: PeerID) -> bool { + self.vv.contains_key(&peer) && *self.vv.get(&peer).unwrap() > 0 + } +} + +fn check_always_dep_on_last_id(map: &BTreeMap) { + for (_, node) in map.iter() { + for dep in node.deps.iter() { + let Some((&dep_id, dep_node)) = map.range(..=dep).next_back() else { + // It's shallow + continue; + }; + assert_eq!(dep_node.id_start(), dep_id); + if dep_node.contains_id(dep) { + assert_eq!(dep_node.id_last(), dep); + } + } + } +} + +impl HasIndex for AppDagNode { + type Int = Counter; + fn get_start_index(&self) -> Self::Int { + self.cnt + } + + fn get_end_index(&self) -> Self::Int { + self.cnt + self.len as Counter + } +} + +impl Sliceable for AppDagNode { + fn slice(&self, from: usize, to: usize) -> Self { + AppDagNodeInner { + peer: self.peer, + cnt: self.cnt + from as Counter, + lamport: self.lamport + from as Lamport, + deps: if from > 0 { + Frontiers::from_id(self.id_start().inc(from as Counter - 1)) + } else { + self.deps.clone() + }, + vv: if let Some(vv) = self.vv.get() { + let mut new = vv.clone(); + new.insert(self.peer, self.cnt + from as Counter); + OnceCell::with_value(new) + } else { + OnceCell::new() + }, + has_succ: if to == self.len { self.has_succ } else { true }, + len: to - from, + } + .into() + } +} + +impl HasId for AppDagNode { + fn id_start(&self) -> ID { + ID { + peer: self.peer, + counter: self.cnt, + } + } +} + +impl HasCounter for AppDagNode { + fn ctr_start(&self) -> Counter { + self.cnt + } +} + +impl HasLength for AppDagNode { + fn atom_len(&self) -> usize { + self.len + } + + fn content_len(&self) -> usize { + self.len + } +} + +impl Mergable for AppDagNode { + fn is_mergable(&self, other: &Self, _conf: &()) -> bool + where + Self: Sized, + { + !self.has_succ + && self.peer == other.peer + && self.cnt + self.len as Counter == other.cnt + && other.deps.len() == 1 + && self.lamport + self.len as Lamport == other.lamport + && other.deps.as_single().unwrap().peer == self.peer + } + + fn merge(&mut self, other: &Self, _conf: &()) + where + Self: Sized, + { + assert_eq!( + other.deps.as_single().unwrap().counter, + self.cnt + self.len as Counter - 1 + ); + let this = Arc::make_mut(&mut self.inner); + this.len += other.len; + this.has_succ = other.has_succ; + } +} + +impl HasLamport for AppDagNode { + fn lamport(&self) -> Lamport { + self.lamport + } +} + +impl DagNode for AppDagNode { + fn deps(&self) -> &Frontiers { + &self.deps + } +} + +impl Dag for AppDag { + type Node = AppDagNode; + + fn frontier(&self) -> &Frontiers { + &self.frontiers + } + + fn get(&self, id: ID) -> Option { + self.ensure_lazy_load_node(id); + let binding = self.map.lock(); + if let Some(x) = binding.range(..=id).next_back() { + if x.1.contains_id(id) { + // PERF: do we need to optimize clone like this? + // by adding another layer of Arc? + return Some(x.1.clone()); + } + } + + if let Some(node) = &self.pending_txn_node { + if node.peer == id.peer && node.cnt <= id.counter { + assert!(node.cnt + node.len as Counter > id.counter); + return Some(node.clone()); + } + } + + None + } + + fn vv(&self) -> &VersionVector { + &self.vv + } + + fn contains(&self, id: ID) -> bool { + self.vv.includes_id(id) + } +} + +impl AppDag { + // PERF: this may be painfully slow + /// get the version vector for a certain op. + /// It's the version when the op is applied + pub fn get_vv(&self, id: ID) -> Option { + self.get(id).map(|x| { + let mut vv = self.ensure_vv_for(&x); + vv.insert(id.peer, id.counter + 1); + vv + }) + } + + pub(crate) fn ensure_vv_for(&self, target_node: &AppDagNode) -> ImVersionVector { + if target_node.vv.get().is_none() { + // Iterative DFS. When the DAG contains a diamond, a dep can end + // up on the stack multiple times (once for each ancestor that + // re-queued the current node before its deps were ready). We + // skip nodes whose vv has already been computed — the original + // code eagerly called `OnceCell::set(..).unwrap()` on every pop + // and panicked on the second visit (see loro-dev/loro#929). + let mut stack: SmallVec<[AppDagNode; 4]> = smallvec::smallvec![target_node.clone()]; + while let Some(top_node) = stack.pop() { + if top_node.vv.get().is_some() { + continue; + } + + let mut ans_vv = ImVersionVector::default(); + if top_node.deps == self.shallow_root_frontiers_deps { + for (&p, &c) in self.shallow_since_vv.iter() { + ans_vv.insert(p, c); + } + } else { + let mut all_deps_processed = true; + for id in top_node.deps.iter() { + let node = self.get(id).expect("deps should be in the dag"); + if node.vv.get().is_none() { + if all_deps_processed { + stack.push(top_node.clone()); + } + all_deps_processed = false; + stack.push(node); + continue; + }; + } + + if !all_deps_processed { + continue; + } + + for id in top_node.deps.iter() { + let node = self.get(id).expect("deps should be in the dag"); + let dep_vv = node.vv.get().unwrap(); + if ans_vv.is_empty() { + ans_vv = dep_vv.clone(); + } else { + ans_vv.extend_to_include_vv(dep_vv.iter()); + } + + ans_vv.insert(node.peer, node.ctr_end()); + } + } + + // Tolerate a racing set from the diamond case above: if + // another path already initialized this cell, trust that + // value (it was computed from the same DAG) and move on. + let _ = top_node.vv.set(ans_vv); + } + } + + target_node.vv.get().unwrap().clone() + } + + /// Compare the causal order of two versions. + /// If None, two versions are concurrent to each other + pub fn cmp_version(&self, a: ID, b: ID) -> Option { + if a.peer == b.peer { + return Some(a.counter.cmp(&b.counter)); + } + + let a = self.get_vv(a).unwrap(); + let b = self.get_vv(b).unwrap(); + a.partial_cmp(&b) + } + + pub fn get_lamport(&self, id: &ID) -> Option { + self.get(*id).and_then(|node| { + assert!(id.counter >= node.cnt); + if node.cnt + node.len as Counter > id.counter { + Some(node.lamport + (id.counter - node.cnt) as Lamport) + } else { + None + } + }) + } + + pub fn get_change_lamport_from_deps(&self, deps: &Frontiers) -> Option { + let mut lamport = 0; + for id in deps.iter() { + let x = self.get_lamport(&id)?; + lamport = lamport.max(x + 1); + } + + Some(lamport) + } + + /// Convert a frontiers to a version vector + /// + /// If the frontiers version is not found in the dag, return None + pub fn frontiers_to_vv(&self, frontiers: &Frontiers) -> Option { + if frontiers == &self.shallow_root_frontiers_deps { + let vv = VersionVector::from_im_vv(&self.shallow_since_vv); + return Some(vv); + } + + let mut vv: VersionVector = Default::default(); + for id in frontiers.iter() { + let x = self.get(id)?; + let target_vv = self.ensure_vv_for(&x); + vv.extend_to_include_vv(target_vv.iter()); + vv.extend_to_include_last_id(id); + } + + Some(vv) + } + + #[allow(unused)] + pub(crate) fn frontiers_to_im_vv(&self, frontiers: &Frontiers) -> ImVersionVector { + if frontiers.is_empty() { + return Default::default(); + } + + let mut iter = frontiers.iter(); + let mut vv = { + let id = iter.next().unwrap(); + let Some(x) = self.get(id) else { + unreachable!() + }; + let mut vv = self.ensure_vv_for(&x); + vv.extend_to_include_last_id(id); + vv + }; + + for id in iter { + let Some(x) = self.get(id) else { + unreachable!() + }; + let x = self.ensure_vv_for(&x); + vv.extend_to_include_vv(x.iter()); + vv.extend_to_include_last_id(id); + } + + vv + } + + pub fn im_vv_to_frontiers(&self, vv: &ImVersionVector) -> Frontiers { + if vv.is_empty() { + return Default::default(); + } + + let this = vv; + let last_ids: Frontiers = this + .iter() + .filter_map(|(client_id, cnt)| { + if *cnt <= 0 { + return None; + } + + if self + .shallow_since_vv + .includes_id(ID::new(*client_id, *cnt - 1)) + { + return None; + } + + Some(ID::new(*client_id, cnt - 1)) + }) + .collect(); + + if last_ids.is_empty() { + return self.shallow_since_frontiers.clone(); + } + + shrink_frontiers(&last_ids, self).unwrap() + } + + pub fn vv_to_frontiers(&self, vv: &VersionVector) -> Frontiers { + if vv.is_empty() { + return Default::default(); + } + + let this = vv; + let last_ids: Frontiers = this + .iter() + .filter_map(|(client_id, cnt)| { + if *cnt <= 0 { + return None; + } + + if self + .shallow_since_vv + .includes_id(ID::new(*client_id, *cnt - 1)) + { + return None; + } + + Some(ID::new(*client_id, cnt - 1)) + }) + .collect(); + + if last_ids.is_empty() { + return self.shallow_since_frontiers.clone(); + } + + shrink_frontiers(&last_ids, self).unwrap() + } + + pub(crate) fn frontiers_to_next_lamport(&self, frontiers: &Frontiers) -> Lamport { + if frontiers.is_empty() { + return 0; + } + + let mut iter = frontiers.iter(); + let mut lamport = { + let id = iter.next().unwrap(); + let Some(x) = self.get(id) else { + unreachable!() + }; + assert!(id.counter >= x.cnt); + (id.counter - x.cnt) as Lamport + x.lamport + 1 + }; + + for id in iter { + let Some(x) = self.get(id) else { + unreachable!() + }; + assert!(id.counter >= x.cnt); + lamport = lamport.max((id.counter - x.cnt) as Lamport + x.lamport + 1); + } + + lamport + } + + pub fn get_frontiers(&self) -> &Frontiers { + &self.frontiers + } + + /// - Ordering::Less means self is less than target or parallel + /// - Ordering::Equal means versions equal + /// - Ordering::Greater means self's version is greater than target + pub fn cmp_with_frontiers(&self, other: &Frontiers) -> Ordering { + if &self.frontiers == other { + Ordering::Equal + } else if other.iter().all(|id| self.vv.includes_id(id)) { + Ordering::Greater + } else { + Ordering::Less + } + } + + // PERF + /// Compare two [Frontiers] causally. + /// + /// If one of the [Frontiers] are not included, it will return [FrontiersNotIncluded]. + pub fn cmp_frontiers( + &self, + a: &Frontiers, + b: &Frontiers, + ) -> Result, FrontiersNotIncluded> { + let a = self.frontiers_to_vv(a).ok_or(FrontiersNotIncluded)?; + let b = self.frontiers_to_vv(b).ok_or(FrontiersNotIncluded)?; + Ok(a.partial_cmp(&b)) + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct FrontiersNotIncluded; +impl Display for FrontiersNotIncluded { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("The given Frontiers are not included by the doc") + } +} + +#[cfg(test)] +mod ensure_vv_for_tests { + use super::*; + use crate::arena::SharedArena; + use std::sync::atomic::AtomicI64; + + fn make_dag_node(peer: PeerID, cnt: Counter, len: usize, deps: Frontiers) -> AppDagNode { + AppDagNodeInner { + vv: OnceCell::new(), + peer, + cnt, + lamport: cnt as Lamport, + deps, + has_succ: false, + len, + } + .into() + } + + /// Regression for loro-dev/loro#929: when computing the vv for a node + /// whose DAG fan-in contains a shared ancestor reached through multiple + /// paths, the iterative DFS used to push the shared ancestor onto the + /// stack twice. The second visit would then call `OnceCell::set(..) + /// .unwrap()` on an already-initialized cell and panic with + /// "called `Result::unwrap()` on an `Err` value: ImVersionVector(..)". + /// + /// Topology: + /// x (peer 1, counter 0) + /// | + /// y (peer 2, counter 0, deps = [x]) + /// | + /// z (peer 3, counter 0, deps = [x, y]) + /// + /// When we call `ensure_vv_for(z)`, z pushes x and y. Processing y then + /// re-pushes x (still uninitialized), so the stack ends up with two + /// copies of x; the second pop must be a no-op after the fix. + #[test] + fn diamond_dep_ensure_vv_for_does_not_double_set() { + let change_store = ChangeStore::new_mem(&SharedArena::new(), Arc::new(AtomicI64::new(0))); + let dag = AppDag::new(change_store); + + let x = make_dag_node(1, 0, 1, Frontiers::default()); + let y = make_dag_node(2, 0, 1, Frontiers::from_id(ID::new(1, 0))); + let mut z_deps = Frontiers::default(); + z_deps.push(ID::new(1, 0)); + z_deps.push(ID::new(2, 0)); + let z = make_dag_node(3, 0, 1, z_deps); + + { + let mut map = dag.map.lock(); + map.insert(x.id_start(), x); + map.insert(y.id_start(), y); + map.insert(z.id_start(), z.clone()); + } + + // Historically this panicked at the inner `vv.set(...).unwrap()`. + // `ensure_vv_for` returns the vv *at* the node: it covers every + // peer in the causal past, but not the node's own peer counter. + let vv = dag.ensure_vv_for(&z); + assert_eq!(vv.get(&1).copied(), Some(1)); + assert_eq!(vv.get(&2).copied(), Some(1)); + assert!(vv.get(&3).is_none()); + } +} diff --git a/vendor/loro-internal/src/oplog/pending_changes.rs b/vendor/loro-internal/src/oplog/pending_changes.rs new file mode 100644 index 00000000..8d84e2b3 --- /dev/null +++ b/vendor/loro-internal/src/oplog/pending_changes.rs @@ -0,0 +1,428 @@ +use std::{collections::BTreeMap, ops::Deref}; + +use crate::{ + change::Change, + version::{ImVersionVector, VersionRange}, + OpLog, VersionVector, +}; +use loro_common::{ + ContainerType, Counter, CounterSpan, HasCounterSpan, HasIdSpan, LoroResult, PeerID, ID, +}; +use rustc_hash::FxHashMap; + +#[derive(Debug, Clone)] +pub enum PendingChange { + // The lamport of the change decoded by `enhanced` is unknown. + // we need calculate it when the change can be applied + Unknown(Change), + // TODO: Refactor, remove this? + #[allow(unused)] + Known(Change), +} + +impl Deref for PendingChange { + type Target = Change; + fn deref(&self) -> &Self::Target { + match self { + Self::Unknown(a) => a, + Self::Known(a) => a, + } + } +} + +#[derive(Debug, Default)] +pub(crate) struct PendingChanges { + changes: FxHashMap>>, +} + +impl PendingChanges { + pub(crate) fn has_state_apply_rollback_ops(&self) -> bool { + self.changes.values().any(|tree| { + tree.values().any(|changes| { + changes.iter().any(|change| { + change.ops.iter().any(|op| { + matches!( + op.container.get_type(), + ContainerType::List | ContainerType::Tree + ) + }) + }) + }) + }) + } + + pub(crate) fn version_range(&self) -> VersionRange { + let mut range = VersionRange::default(); + for tree in self.changes.values() { + for pending_changes in tree.values() { + for pending_change in pending_changes { + range.extends_to_include_id_span(pending_change.id_span()); + } + } + } + + range + } +} + +#[cfg(test)] +#[allow(dead_code)] +impl PendingChanges { + pub(crate) fn len(&self) -> usize { + self.changes + .values() + .map(|tree| tree.values().map(Vec::len).sum::()) + .sum() + } +} + +#[derive(Debug, Default)] +pub(crate) struct PendingChangesRollback { + added: Vec<(PeerID, Counter)>, + removed: Vec<(PeerID, Counter, Vec)>, +} + +impl PendingChangesRollback { + fn record_added(&mut self, id: ID) { + self.added.push((id.peer, id.counter)); + } + + fn record_removed(&mut self, peer: PeerID, counter: Counter, changes: Vec) { + self.removed.push((peer, counter, changes)); + } + + pub(crate) fn rollback(self, pending_changes: &mut PendingChanges) { + for (peer, counter) in self.added.into_iter().rev() { + let Some(tree) = pending_changes.changes.get_mut(&peer) else { + continue; + }; + let Some(changes) = tree.get_mut(&counter) else { + continue; + }; + changes.pop(); + if changes.is_empty() { + tree.remove(&counter); + } + if tree.is_empty() { + pending_changes.changes.remove(&peer); + } + } + + for (peer, counter, changes) in self.removed.into_iter().rev() { + pending_changes + .changes + .entry(peer) + .or_default() + .insert(counter, changes); + } + } +} + +impl OpLog { + fn push_pending_change(&mut self, missing_dep: ID, change: PendingChange) { + if let Some(rollback) = self.import_rollback.as_mut() { + rollback.pending.record_added(missing_dep); + } + + self.pending_changes + .changes + .entry(missing_dep.peer) + .or_default() + .entry(missing_dep.counter) + .or_default() + .push(change); + } + + pub(super) fn extend_pending_changes_with_unknown_lamport( + &mut self, + remote_changes: Vec, + ) -> LoroResult<()> { + for change in remote_changes { + let local_change = PendingChange::Unknown(change); + match remote_change_apply_state(self.vv(), self.shallow_since_vv(), &local_change) { + ChangeState::AwaitingMissingDependency(miss_dep) => { + self.push_pending_change(miss_dep, local_change) + } + ChangeState::Applied => unreachable!("already applied"), + ChangeState::CanApplyDirectly => unreachable!("can apply directly"), + } + } + + Ok(()) + } +} + +impl OpLog { + /// Try to apply pending changes. + /// + /// `new_ids` are the ID of the op that is just applied. + pub(crate) fn try_apply_pending( + &mut self, + mut new_ids: Vec, + mut would_affect: Option<&mut VersionRange>, + ) { + while let Some(id) = new_ids.pop() { + let Some(tree) = self.pending_changes.changes.get_mut(&id.peer) else { + continue; + }; + + let mut to_remove = Vec::new(); + for (cnt, _) in tree.range_mut(0..=id.counter) { + to_remove.push(*cnt); + } + + let mut pending_set = Vec::with_capacity(to_remove.len()); + for cnt in to_remove { + let pending_changes = tree.remove(&cnt).unwrap(); + if let Some(rollback) = self.import_rollback.as_mut() { + rollback + .pending + .record_removed(id.peer, cnt, pending_changes.clone()); + } + pending_set.push(pending_changes); + } + + if tree.is_empty() { + self.pending_changes.changes.remove(&id.peer); + } + + for pending_changes in pending_set { + for pending_change in pending_changes { + match remote_change_apply_state( + self.dag.vv(), + self.dag.shallow_since_vv(), + &pending_change, + ) { + ChangeState::CanApplyDirectly => { + new_ids.push(pending_change.id_last()); + self.apply_change_from_remote( + pending_change, + would_affect.as_deref_mut(), + ); + } + ChangeState::Applied => {} + ChangeState::AwaitingMissingDependency(miss_dep) => { + self.push_pending_change(miss_dep, pending_change) + } + } + } + } + } + } + + pub(super) fn apply_change_from_remote( + &mut self, + change: PendingChange, + would_affect: Option<&mut VersionRange>, + ) { + let change = match change { + PendingChange::Known(mut c) => { + self.dag.calc_unknown_lamport_change(&mut c).unwrap(); + c + } + PendingChange::Unknown(mut c) => { + self.dag.calc_unknown_lamport_change(&mut c).unwrap(); + c + } + }; + + let Some(change) = self.trim_the_known_part_of_change(change) else { + return; + }; + + if let Some(w) = would_affect { + w.extends_to_include_id_span(change.id_span()); + } + self.insert_new_change(change, false); + } +} + +enum ChangeState { + Applied, + CanApplyDirectly, + // The id of first missing dep + AwaitingMissingDependency(ID), +} + +fn remote_change_apply_state( + vv: &VersionVector, + _shallow_vv: &ImVersionVector, + change: &Change, +) -> ChangeState { + let peer = change.id.peer; + let CounterSpan { start, end } = change.ctr_span(); + let vv_latest_ctr = vv.get(&peer).copied().unwrap_or(0); + if vv_latest_ctr >= end { + return ChangeState::Applied; + } + + if vv_latest_ctr < start { + return ChangeState::AwaitingMissingDependency(change.id.inc(-1)); + } + + for dep in change.deps.iter() { + let dep_vv_latest_ctr = vv.get(&dep.peer).copied().unwrap_or(0); + if dep_vv_latest_ctr - 1 < dep.counter { + return ChangeState::AwaitingMissingDependency(dep); + } + } + + ChangeState::CanApplyDirectly +} + +#[cfg(test)] +mod test { + use crate::{cursor::PosType, loro::ExportMode, LoroDoc, ToJson, VersionVector}; + + #[test] + fn import_pending() { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let text_a = a.get_text("text"); + text_a.insert(0, "a", PosType::Unicode).unwrap(); + let update1 = a + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version1 = a.oplog_vv(); + text_a.insert(0, "b", PosType::Unicode).unwrap(); + let update2 = a.export(ExportMode::updates(&version1)).unwrap(); + let version2 = a.oplog_vv(); + text_a.insert(0, "c", PosType::Unicode).unwrap(); + let update3 = a.export(ExportMode::updates(&version2)).unwrap(); + let version3 = a.oplog_vv(); + text_a.insert(0, "d", PosType::Unicode).unwrap(); + let update4 = a.export(ExportMode::updates(&version3)).unwrap(); + // let version4 = a.oplog_vv(); + text_a.insert(0, "e", PosType::Unicode).unwrap(); + let update3_5 = a.export(ExportMode::updates(&version2)).unwrap(); + b.import(&update3_5).unwrap(); + b.import(&update4).unwrap(); + b.import(&update2).unwrap(); + b.import(&update3).unwrap(); + b.import(&update1).unwrap(); + assert_eq!(a.get_deep_value(), b.get_deep_value()); + } + + #[test] + fn pending_import_snapshot() { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let text_a = a.get_text("text"); + text_a.insert(0, "a", PosType::Unicode).unwrap(); + let update1 = a.export(ExportMode::Snapshot).unwrap(); + let version1 = a.oplog_vv(); + text_a.insert(0, "b", PosType::Unicode).unwrap(); + let update2 = a.export(ExportMode::updates(&version1)).unwrap(); + let _version2 = a.oplog_vv(); + b.import(&update2).unwrap(); + // snapshot will be converted to updates + b.import(&update1).unwrap(); + assert_eq!(a.get_deep_value(), b.get_deep_value()); + } + + #[test] + fn need_deps_pending_import() { + // a: a1 <--- a2 + // \ / + // b: b1 + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let c = LoroDoc::new_auto_commit(); + c.set_peer_id(3).unwrap(); + let d = LoroDoc::new_auto_commit(); + d.set_peer_id(4).unwrap(); + let text_a = a.get_text("text"); + let text_b = b.get_text("text"); + text_a.insert(0, "a", PosType::Unicode).unwrap(); + let version_a1 = a.oplog_vv(); + let update_a1 = a + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + b.import(&update_a1).unwrap(); + text_b.insert(1, "b", PosType::Unicode).unwrap(); + let update_b1 = b.export(ExportMode::updates(&version_a1)).unwrap(); + a.import(&update_b1).unwrap(); + let version_a1b1 = a.oplog_vv(); + text_a.insert(2, "c", PosType::Unicode).unwrap(); + let update_a2 = a.export(ExportMode::updates(&version_a1b1)).unwrap(); + c.import(&update_a2).unwrap(); + assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"\"}"); + c.import(&update_a1).unwrap(); + assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"a\"}"); + c.import(&update_b1).unwrap(); + assert_eq!(a.get_deep_value(), c.get_deep_value()); + + d.import(&update_a2).unwrap(); + assert_eq!(d.get_deep_value().to_json(), "{\"text\":\"\"}"); + d.import(&update_b1).unwrap(); + assert_eq!(d.get_deep_value().to_json(), "{\"text\":\"\"}"); + d.import(&update_a1).unwrap(); + assert_eq!(a.get_deep_value(), d.get_deep_value()); + } + + #[test] + fn should_activate_pending_change_when() { + // 0@a <- 0@b + // 0@a <- 1@a, where 0@a and 1@a will be merged + // In this case, c apply b's change first, then apply all the changes from a. + // C is expected to have the same content as a, after a imported b's change + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(1).unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + let c = LoroDoc::new_auto_commit(); + c.set_peer_id(3).unwrap(); + let text_a = a.get_text("text"); + let text_b = b.get_text("text"); + text_a.insert(0, "1", PosType::Unicode).unwrap(); + b.import(&a.export(ExportMode::Snapshot).unwrap()).unwrap(); + text_b.insert(0, "1", PosType::Unicode).unwrap(); + let b_change = b.export(ExportMode::updates(&a.oplog_vv())).unwrap(); + text_a.insert(0, "1", PosType::Unicode).unwrap(); + c.import(&b_change).unwrap(); + c.import(&a.export(ExportMode::Snapshot).unwrap()).unwrap(); + a.import(&b_change).unwrap(); + assert_eq!(c.get_deep_value(), a.get_deep_value()); + } + + // Change cannot be merged now + // #[test] + // fn pending_changes_may_deps_merged_change() { + // // a: (a1 <-- a2 <-- a3) <-- a4 a1~a3 is a merged change + // // \ / + // // b: b1 + // let a = LoroDoc::new(); + // a.set_peer_id(1); + // let b = LoroDoc::new(); + // b.set_peer_id(2); + // let c = LoroDoc::new(); + // c.set_peer_id(3); + // let text_a = a.get_text("text"); + // let text_b = b.get_text("text"); + // a.with_txn(|txn| text_a.insert(txn, 0, "a")).unwrap(); + // a.with_txn(|txn| text_a.insert(txn, 1, "b")).unwrap(); + // let version_a12 = a.oplog_vv(); + // let updates_a12 = a.export(ExportMode::Snapshot); + // a.with_txn(|txn| text_a.insert(txn, 2, "c")).unwrap(); + // let updates_a123 = a.export(ExportMode::Snapshot); + // b.import(&updates_a12).unwrap(); + // b.with_txn(|txn| text_b.insert(txn, 2, "d")).unwrap(); + // let update_b1 = b.export(ExportMode::updates(&version_a12)).unwrap(); + // a.import(&update_b1).unwrap(); + // let version_a123_b1 = a.oplog_vv(); + // a.with_txn(|txn| text_a.insert(txn, 4, "e")).unwrap(); + // let update_a4 = a.export(ExportMode::updates(&version_a123_b1)).unwrap(); + // c.import(&update_b1).unwrap(); + // assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"\"}"); + // c.import(&update_a4).unwrap(); + // assert_eq!(c.get_deep_value().to_json(), "{\"text\":\"\"}"); + // c.import(&updates_a123).unwrap(); + // assert_eq!(c.get_deep_value(), a.get_deep_value()); + // } +} diff --git a/vendor/loro-internal/src/parent.rs b/vendor/loro-internal/src/parent.rs new file mode 100644 index 00000000..371950f3 --- /dev/null +++ b/vendor/loro-internal/src/parent.rs @@ -0,0 +1,88 @@ +//! This file contains the methods to maintain the parent-child relationship in Loro. +//! The child-parent relationship is established when the child container is created. +//! It won't be changed. So the depth of a container is fixed. +//! +//! + +use loro_common::LoroValue; + +use crate::{ + arena::SharedArena, + change::Change, + container::{list::list_op::ListOp, map::MapSet}, + op::{ListSlice, RawOp, RawOpContent}, + DocState, OpLog, +}; + +impl OpLog { + /// Establish the link between the child container and the parent container. + /// This relationship will never be broken. + /// It's used at the entry of creating new ops to ensure all the valid containers + /// in Loro are properly linked. + pub(super) fn register_container_and_parent_link(&self, change: &Change) { + let arena = &self.arena; + register_container_and_parent_link(arena, change); + } +} + +pub(super) fn register_container_and_parent_link(arena: &SharedArena, change: &Change) { + for op in change.ops.iter() { + op.content.visit_created_children(arena, &mut |c| { + let idx = arena.register_container(c); + arena.set_parent(idx, Some(op.container)); + }); + } +} + +impl DocState { + /// This is used in txn to short cut the process of applying an op to the state. + /// So the op here has not been registered on the oplog yet. + pub(super) fn set_container_parent_by_raw_op(&self, raw_op: &RawOp) { + let container = raw_op.container; + match &raw_op.content { + RawOpContent::List(op) => { + if let ListOp::Insert { + slice: ListSlice::RawData(list), + .. + } = op + { + let list = match list { + std::borrow::Cow::Borrowed(list) => list.iter(), + std::borrow::Cow::Owned(list) => list.iter(), + }; + for value in list { + if let LoroValue::Container(c) = value { + let idx = self.arena.register_container(c); + self.arena.set_parent(idx, Some(container)); + } + } + } + if let ListOp::Set { + elem_id: _, + value: LoroValue::Container(c), + } = op + { + let idx = self.arena.register_container(c); + self.arena.set_parent(idx, Some(container)); + } + } + RawOpContent::Map(MapSet { key: _, value }) => { + if let Some(LoroValue::Container(c)) = value { + let idx = self.arena.register_container(c); + self.arena.set_parent(idx, Some(container)); + } + } + RawOpContent::Tree(tree) => { + let target = tree.target(); + // create associated metadata container + // TODO: maybe we could create map container only when setting metadata + let container_id = target.associated_meta_container(); + let child_idx = self.arena.register_container(&container_id); + self.arena.set_parent(child_idx, Some(container)); + } + #[cfg(feature = "counter")] + RawOpContent::Counter(_) => {} + RawOpContent::Unknown { .. } => {} + } + } +} diff --git a/vendor/loro-internal/src/pre_commit.rs b/vendor/loro-internal/src/pre_commit.rs new file mode 100644 index 00000000..48c32ecd --- /dev/null +++ b/vendor/loro-internal/src/pre_commit.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +use crate::sync::Mutex; +use crate::{ + change::{Change, Timestamp}, + oplog::get_timestamp_now_txn, + ChangeMeta, +}; +use loro_common::PeerID; + +/// The callback of the first commit from a peer. +pub type FirstCommitFromPeerCallback = + Box bool + Send + Sync + 'static>; +pub type PreCommitCallback = Box bool + Send + Sync + 'static>; + +/// The payload of the pre commit callback. +#[derive(Debug, Clone)] +pub struct PreCommitCallbackPayload { + /// The metadata of the change which will be committed. + pub change_meta: ChangeMeta, + /// The origin of the commit. + pub origin: String, + /// The modifier of the change. You can modify the change in the callback. + pub modifier: ChangeModifier, +} + +/// The payload of the first commit from a peer callback. +#[derive(Debug, Clone)] +pub struct FirstCommitFromPeerPayload { + /// The peer id of the first commit. + pub peer: PeerID, +} + +#[derive(Debug, Clone, Default)] +pub struct ChangeModifier(Arc>); + +#[derive(Debug, Default)] +struct ChangeModifierInner { + new_msg: Option>, + new_timestamp: Option, +} + +impl ChangeModifier { + pub fn set_message(&self, msg: &str) -> &Self { + self.0.lock().new_msg = Some(Arc::from(msg)); + self + } + + pub fn set_timestamp(&self, timestamp: Timestamp) -> &Self { + self.0.lock().new_timestamp = Some(timestamp); + self + } + + pub fn set_timestamp_now(&self) -> &Self { + self.0.lock().new_timestamp = Some(get_timestamp_now_txn()); + self + } + + pub(crate) fn modify_change(&self, change: &mut Change) { + let m = self.0.lock(); + if let Some(msg) = &m.new_msg { + change.commit_msg = Some(msg.clone()); + } + + if let Some(timestamp) = m.new_timestamp { + change.timestamp = timestamp; + } + } +} diff --git a/vendor/loro-internal/src/span.rs b/vendor/loro-internal/src/span.rs new file mode 100644 index 00000000..b1112193 --- /dev/null +++ b/vendor/loro-internal/src/span.rs @@ -0,0 +1,3 @@ +pub use loro_common::{ + CounterSpan, HasCounter, HasCounterSpan, HasId, HasIdSpan, HasLamport, HasLamportSpan, IdSpan, +}; diff --git a/vendor/loro-internal/src/state.rs b/vendor/loro-internal/src/state.rs new file mode 100644 index 00000000..7cd015cd --- /dev/null +++ b/vendor/loro-internal/src/state.rs @@ -0,0 +1,2359 @@ +use crate::sync::{AtomicU64, Mutex, RwLock}; +#[cfg(test)] +use std::cell::Cell; +use std::sync::{Arc, Weak}; +use std::{borrow::Cow, io::Write, sync::atomic::Ordering}; + +use container_store::ContainerStore; +use dead_containers_cache::DeadContainersCache; +use enum_as_inner::EnumAsInner; +use enum_dispatch::enum_dispatch; +use itertools::Itertools; +use loro_common::{ContainerID, Lamport, LoroError, LoroResult, TreeID, ID}; +use loro_delta::DeltaItem; +use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::{info_span, instrument, warn}; + +use crate::{ + configure::{Configure, DefaultRandom, SecureRandomGenerator}, + container::{idx::ContainerIdx, richtext::config::StyleConfigMap}, + cursor::Cursor, + delta::TreeExternalDiff, + diff_calc::{DiffCalculator, DiffMode}, + event::{Diff, EventTriggerKind, Index, InternalContainerDiff, InternalDiff}, + fx_map, + handler::ValueOrHandler, + id::PeerID, + lock::{LoroLockGroup, LoroMutex}, + op::{Op, RawOp}, + version::Frontiers, + ContainerDiff, ContainerType, DocDiff, InternalString, LoroDocInner, LoroValue, OpLog, +}; + +pub(crate) mod analyzer; +pub(crate) mod container_store; +#[cfg(feature = "counter")] +mod counter_state; +mod dead_containers_cache; +mod list_state; +mod map_state; +mod mergeable; +mod movable_list_state; +mod richtext_state; +mod tree_state; +mod unknown_state; + +pub(crate) use self::movable_list_state::{IndexType, MovableListState}; +pub(crate) use container_store::GcStore; +pub(crate) use list_state::ListState; +pub(crate) use map_state::MapState; +pub(crate) use richtext_state::RichtextState; +pub(crate) use tree_state::FiIfNotConfigured; +pub(crate) use tree_state::{get_meta_value, FractionalIndexGenResult, NodePosition, TreeState}; +pub use tree_state::{TreeNode, TreeNodeWithChildren, TreeParentId}; + +use self::{container_store::ContainerWrapper, unknown_state::UnknownState}; + +#[cfg(feature = "counter")] +use self::counter_state::CounterState; + +use super::{arena::SharedArena, event::InternalDocDiff}; + +#[cfg(test)] +thread_local! { + static FAIL_NEXT_IMPORT_STATE_APPLY: Cell = Cell::new(false); +} + +#[cfg(test)] +pub(crate) fn fail_next_import_state_apply_for_test() { + FAIL_NEXT_IMPORT_STATE_APPLY.with(|fail| fail.set(true)); +} + +fn visible_container_value_is_empty(kind: ContainerType, value: &LoroValue) -> bool { + match kind { + ContainerType::Text => value.as_string().is_some_and(|value| value.is_empty()), + ContainerType::Map | ContainerType::List | ContainerType::MovableList => { + value.is_empty_collection() + } + ContainerType::Tree => value.as_list().is_some_and(|value| value.is_empty()), + #[cfg(feature = "counter")] + ContainerType::Counter => false, + ContainerType::Unknown(_) => false, + } +} + +fn deleted_root_container_value_is_cleared(kind: ContainerType, value: &LoroValue) -> bool { + match kind { + #[cfg(feature = "counter")] + ContainerType::Counter => value.as_double().is_some_and(|value| *value == 0.0), + _ => visible_container_value_is_empty(kind, value), + } +} + +fn state_decode_error(message: impl Into>) -> LoroError { + LoroError::DecodeError(message.into()) +} + +fn decode_peer_table(bytes: &mut &[u8], context: &str) -> LoroResult> { + let peer_num = leb128::read::unsigned(bytes) + .map_err(|_| state_decode_error(format!("{context}: invalid peer table length")))?; + let peer_num = usize::try_from(peer_num) + .map_err(|_| state_decode_error(format!("{context}: peer table length overflow")))?; + let peer_bytes_len = peer_num + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| state_decode_error(format!("{context}: peer table byte length overflow")))?; + if bytes.len() < peer_bytes_len { + return Err(state_decode_error(format!( + "{context}: truncated peer table" + ))); + } + + let peer_bytes = &bytes[..peer_bytes_len]; + let peers = peer_bytes + .chunks_exact(std::mem::size_of::()) + .map(|chunk| { + let mut buf = [0u8; std::mem::size_of::()]; + buf.copy_from_slice(chunk); + PeerID::from_le_bytes(buf) + }) + .collect(); + *bytes = &bytes[peer_bytes_len..]; + Ok(peers) +} + +fn decode_peer_from_table(peers: &[PeerID], peer_idx: usize, context: &str) -> LoroResult { + peers + .get(peer_idx) + .copied() + .ok_or_else(|| state_decode_error(format!("{context}: peer index out of range"))) +} + +fn read_state_leb_u64(bytes: &mut &[u8], context: &str) -> LoroResult { + leb128::read::unsigned(bytes) + .map_err(|_| state_decode_error(format!("{context}: invalid integer"))) +} + +fn decode_counter(counter: i32, context: &str) -> LoroResult { + if counter < 0 { + return Err(state_decode_error(format!("{context}: negative counter"))); + } + + Ok(counter) +} + +fn decode_lamport_from_delta( + counter: i32, + lamport_sub_counter: i32, + context: &str, +) -> LoroResult { + decode_counter(counter, context)?; + let lamport = counter + .checked_add(lamport_sub_counter) + .ok_or_else(|| state_decode_error(format!("{context}: lamport overflow")))?; + u32::try_from(lamport).map_err(|_| state_decode_error(format!("{context}: negative lamport"))) +} + +pub struct DocState { + pub(super) peer: Arc, + + pub(super) frontiers: Frontiers, + // pub(super) states: FxHashMap, + pub(super) store: ContainerStore, + pub(super) arena: SharedArena, + pub(crate) config: Configure, + // resolve event stuff + doc: Weak, + // txn related stuff + in_txn: bool, + changed_idx_in_txn: FxHashSet, + + // diff related stuff + event_recorder: EventRecorder, + + dead_containers_cache: DeadContainersCache, +} + +impl std::fmt::Debug for DocState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DocState") + .field("peer", &self.peer) + .finish() + } +} + +#[derive(Clone, Copy)] +pub(crate) struct ContainerCreationContext<'a> { + pub configure: &'a Configure, + pub peer: PeerID, +} + +pub(crate) struct DiffApplyContext<'a> { + pub mode: DiffMode, + pub doc: &'a Weak, +} + +pub(crate) trait FastStateSnapshot { + fn encode_snapshot_fast(&mut self, w: W); + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])>; + fn decode_snapshot_fast( + idx: ContainerIdx, + v: (LoroValue, &[u8]), + ctx: ContainerCreationContext, + ) -> LoroResult + where + Self: Sized; +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct ApplyLocalOpReturn { + pub deleted_containers: Vec, +} + +#[enum_dispatch] +pub(crate) trait ContainerState { + fn container_idx(&self) -> ContainerIdx; + + fn is_state_empty(&self) -> bool; + + #[must_use] + fn apply_diff_and_convert(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> Diff; + + /// Apply a state diff. Returns `Err` when the diff is malformed in a + /// way that would corrupt the container (e.g. a `Retain`/`Insert` that + /// overruns the current state). The error is propagated up to + /// `doc.import` so the caller can observe the failure without the + /// doc-state panicking inside a lock. + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()>; + + /// Validate a state diff before it is applied. Implementations should keep + /// this check proportional to the diff size and must not mutate container + /// state. + fn validate_diff(&self, _diff: &InternalDiff) -> LoroResult<()> { + Ok(()) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, op: &Op) -> LoroResult; + /// Convert a state to a diff, such that an empty state will be transformed into the same as this state when it's applied. + fn to_diff(&mut self, doc: &Weak) -> Diff; + + fn get_value(&mut self) -> LoroValue; + + /// Get the index of the child container + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option; + + #[allow(unused)] + fn contains_child(&self, id: &ContainerID) -> bool; + + #[allow(unused)] + fn get_child_containers(&self) -> Vec; + + fn fork(&self, config: &Configure) -> Self; +} + +impl FastStateSnapshot for Box { + fn encode_snapshot_fast(&mut self, w: W) { + self.as_mut().encode_snapshot_fast(w) + } + + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> { + T::decode_value(bytes) + } + + fn decode_snapshot_fast( + idx: ContainerIdx, + v: (LoroValue, &[u8]), + ctx: ContainerCreationContext, + ) -> LoroResult + where + Self: Sized, + { + T::decode_snapshot_fast(idx, v, ctx).map(|x| Box::new(x)) + } +} + +impl ContainerState for Box { + fn container_idx(&self) -> ContainerIdx { + self.as_ref().container_idx() + } + + fn is_state_empty(&self) -> bool { + self.as_ref().is_state_empty() + } + + fn apply_diff_and_convert(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> Diff { + self.as_mut().apply_diff_and_convert(diff, ctx) + } + + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + self.as_mut().apply_diff(diff, ctx) + } + + fn validate_diff(&self, diff: &InternalDiff) -> LoroResult<()> { + self.as_ref().validate_diff(diff) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, op: &Op) -> LoroResult { + self.as_mut().apply_local_op(raw_op, op) + } + + #[doc = r" Convert a state to a diff, such that an empty state will be transformed into the same as this state when it's applied."] + fn to_diff(&mut self, doc: &Weak) -> Diff { + self.as_mut().to_diff(doc) + } + + fn get_value(&mut self) -> LoroValue { + self.as_mut().get_value() + } + + #[doc = r" Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + self.as_ref().get_child_index(id) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.as_ref().contains_child(id) + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + self.as_ref().get_child_containers() + } + + fn fork(&self, config: &Configure) -> Self { + Box::new(self.as_ref().fork(config)) + } +} + +#[allow(clippy::enum_variant_names)] +#[enum_dispatch(ContainerState)] +#[derive(EnumAsInner, Debug)] +pub enum State { + ListState(Box), + MovableListState(Box), + MapState(Box), + RichtextState(Box), + TreeState(Box), + #[cfg(feature = "counter")] + CounterState(Box), + UnknownState(UnknownState), +} + +impl From for State { + fn from(s: ListState) -> Self { + Self::ListState(Box::new(s)) + } +} + +impl From for State { + fn from(s: RichtextState) -> Self { + Self::RichtextState(Box::new(s)) + } +} + +impl From for State { + fn from(s: MovableListState) -> Self { + Self::MovableListState(Box::new(s)) + } +} + +impl From for State { + fn from(s: MapState) -> Self { + Self::MapState(Box::new(s)) + } +} + +impl From for State { + fn from(s: TreeState) -> Self { + Self::TreeState(Box::new(s)) + } +} + +#[cfg(feature = "counter")] +impl From for State { + fn from(s: CounterState) -> Self { + Self::CounterState(Box::new(s)) + } +} + +impl State { + pub fn new_list(idx: ContainerIdx) -> Self { + Self::ListState(Box::new(ListState::new(idx))) + } + + pub fn new_map(idx: ContainerIdx) -> Self { + Self::MapState(Box::new(MapState::new(idx))) + } + + pub fn new_richtext(idx: ContainerIdx, config: Arc>) -> Self { + Self::RichtextState(Box::new(RichtextState::new(idx, config))) + } + + pub fn new_tree(idx: ContainerIdx, peer: PeerID) -> Self { + Self::TreeState(Box::new(TreeState::new(idx, peer))) + } + + pub fn new_unknown(idx: ContainerIdx) -> Self { + Self::UnknownState(UnknownState::new(idx)) + } + + pub fn encode_snapshot_fast(&mut self, mut w: W) { + match self { + State::ListState(s) => s.encode_snapshot_fast(&mut w), + State::MovableListState(s) => s.encode_snapshot_fast(&mut w), + State::MapState(s) => s.encode_snapshot_fast(&mut w), + State::RichtextState(s) => s.encode_snapshot_fast(&mut w), + State::TreeState(s) => s.encode_snapshot_fast(&mut w), + #[cfg(feature = "counter")] + State::CounterState(s) => s.encode_snapshot_fast(&mut w), + State::UnknownState(s) => s.encode_snapshot_fast(&mut w), + } + } + + pub fn fork(&self, config: &Configure) -> Self { + match self { + State::ListState(list_state) => State::ListState(list_state.fork(config)), + State::MovableListState(movable_list_state) => { + State::MovableListState(movable_list_state.fork(config)) + } + State::MapState(map_state) => State::MapState(map_state.fork(config)), + State::RichtextState(richtext_state) => { + State::RichtextState(richtext_state.fork(config)) + } + State::TreeState(tree_state) => State::TreeState(tree_state.fork(config)), + #[cfg(feature = "counter")] + State::CounterState(counter_state) => State::CounterState(counter_state.fork(config)), + State::UnknownState(unknown_state) => State::UnknownState(unknown_state.fork(config)), + } + } +} + +impl DocState { + #[inline] + pub fn new_arc( + doc: Weak, + arena: SharedArena, + config: Configure, + lock_group: &LoroLockGroup, + ) -> Arc> { + let peer = DefaultRandom.next_u64(); + // TODO: maybe we should switch to certain version in oplog? + + let peer = Arc::new(AtomicU64::new(peer)); + Arc::new(lock_group.new_lock( + Self { + store: ContainerStore::new(arena.clone(), config.clone(), peer.clone()), + peer, + arena, + frontiers: Frontiers::default(), + doc, + config, + in_txn: false, + changed_idx_in_txn: FxHashSet::default(), + event_recorder: Default::default(), + dead_containers_cache: Default::default(), + }, + crate::lock::LockKind::DocState, + )) + } + + pub fn fork_with_new_peer_id( + &mut self, + doc: Weak, + arena: SharedArena, + config: Configure, + ) -> Arc> { + let peer = Arc::new(AtomicU64::new(DefaultRandom.next_u64())); + let store = self.store.fork(arena.clone(), peer.clone(), config.clone()); + Arc::new(Mutex::new(Self { + peer, + frontiers: self.frontiers.clone(), + store, + arena, + config, + doc, + in_txn: false, + changed_idx_in_txn: FxHashSet::default(), + event_recorder: Default::default(), + dead_containers_cache: Default::default(), + })) + } + + pub fn start_recording(&mut self) { + if self.is_recording() { + return; + } + + self.event_recorder.recording_diff = true; + self.event_recorder.diff_start_version = Some(self.frontiers.clone()); + } + + #[inline(always)] + pub fn stop_and_clear_recording(&mut self) { + self.event_recorder = Default::default(); + } + + #[inline(always)] + pub fn is_recording(&self) -> bool { + self.event_recorder.recording_diff + } + + pub fn refresh_peer_id(&mut self) { + self.peer.store( + DefaultRandom.next_u64(), + std::sync::atomic::Ordering::Relaxed, + ); + } + + /// Take all the diffs that are recorded and convert them to events. + pub fn take_events(&mut self) -> Vec { + if !self.is_recording() { + return vec![]; + } + + self.convert_current_batch_diff_into_event(); + std::mem::take(&mut self.event_recorder.events) + } + + /// Record the next diff. + /// Caller should call [pre_txn] before calling this. + /// + /// # Panic + /// + /// Panic when the diff cannot be merged with the previous diff. + /// Caller should call [pre_txn] before calling this to avoid panic. + fn record_diff(&mut self, diff: InternalDocDiff) { + if !self.event_recorder.recording_diff || diff.diff.is_empty() { + return; + } + + let Some(last_diff) = self.event_recorder.diffs.last_mut() else { + self.event_recorder.diffs.push(diff.into_owned()); + return; + }; + + if last_diff.can_merge(&diff) { + self.event_recorder.diffs.push(diff.into_owned()); + return; + } + + panic!("should call pre_txn before record_diff") + } + + /// This should be called when DocState is going to apply a transaction / a diff. + fn pre_txn(&mut self, next_origin: InternalString, next_trigger: EventTriggerKind) { + if !self.is_recording() { + return; + } + + let Some(last_diff) = self.event_recorder.diffs.last() else { + return; + }; + + if last_diff.origin == next_origin && last_diff.by == next_trigger { + return; + } + + // current diff batch cannot merge with the incoming diff, + // need to convert all the current diffs into event + self.convert_current_batch_diff_into_event() + } + + fn convert_current_batch_diff_into_event(&mut self) { + let recorder = &mut self.event_recorder; + if recorder.diffs.is_empty() { + return; + } + + let diffs = std::mem::take(&mut recorder.diffs); + let start = recorder.diff_start_version.take().unwrap(); + recorder.diff_start_version = Some((*diffs.last().unwrap().new_version).to_owned()); + let event = self.diffs_to_event(diffs, start); + self.event_recorder.events.push(event); + } + + /// Change the peer id of this doc state. + /// It changes the peer id for the future txn on this AppState + #[inline] + pub fn set_peer_id(&mut self, peer: PeerID) { + self.peer.store(peer, std::sync::atomic::Ordering::Relaxed); + } + + pub fn peer_id(&self) -> PeerID { + self.peer.load(std::sync::atomic::Ordering::Relaxed) + } + + /// It's expected that diff only contains [`InternalDiff`] + /// + /// Returns `Err` if any container state rejects its diff as malformed + /// (see `ContainerState::validate_diff`). Validation runs before mutating + /// any container state, so a returned error leaves the doc-state view at + /// the previous version. + #[instrument(skip_all)] + pub(crate) fn apply_diff( + &mut self, + mut diff: InternalDocDiff<'static>, + diff_mode: DiffMode, + ) -> LoroResult<()> { + if self.in_txn { + return Err(LoroError::TransactionError( + "apply_diff should not be called in a transaction" + .to_string() + .into_boxed_str(), + )); + } + + let is_recording = self.is_recording(); + let Cow::Owned(mut diffs) = std::mem::take(&mut diff.diff) else { + unreachable!() + }; + self.validate_diff_batch(&diffs)?; + #[cfg(test)] + if diff.origin.as_str() == "__loro_fail_import_state_apply" { + return Err(LoroError::internal("state apply failpoint")); + } + #[cfg(test)] + if diff.by == EventTriggerKind::Import { + let should_fail = FAIL_NEXT_IMPORT_STATE_APPLY.with(|fail| { + let should_fail = fail.get(); + if should_fail { + fail.set(false); + } + should_fail + }); + if should_fail { + return Err(LoroError::internal("state apply failpoint")); + } + } + match diff_mode { + DiffMode::Checkout => { + self.dead_containers_cache.clear(); + } + _ => { + self.dead_containers_cache.clear_alive(); + } + } + self.pre_txn(diff.origin.clone(), diff.by); + + // # Revival + // + // A Container, if it is deleted from its parent Container, will still exist + // in the internal state of Loro; whereas on the user side, a tree structure + // is maintained following Events, and at this point, the corresponding state + // is considered deleted. + // + // Sometimes, this "pseudo-dead" Container may be revived (for example, through + // backtracking or parallel editing), and the user side should receive an Event + // that restores the consistency between the revived Container and the internal + // state of Loro. This Event is required to restore the pseudo-dead Container + // State to its current state on Loro, and we refer to this process as "revival". + // + // Revival occurs during the application of the internal diff, and this operation + // is necessary when it needs to be converted into an external Event. + // + // We can utilize the output of the Diff to determine which child nodes should be revived. + // + // For nodes that are to be revived, we can disregard the Events output by their + // round of apply_diff_and_convert, and instead, directly convert their state into + // an Event once their application is complete. + // + // Suppose A is revived and B is A's child, and B also needs to be revived; therefore, + // we should process each level alternately. + + // We need to ensure diff is processed in order + diffs.sort_by_cached_key(|diff| self.arena.get_depth(diff.idx)); + let mut to_revive_in_next_layer: FxHashSet = FxHashSet::default(); + let mut to_revive_in_this_layer: FxHashSet = FxHashSet::default(); + let mut last_depth = 0; + let len = diffs.len(); + for mut diff in std::mem::replace(&mut diffs, Vec::with_capacity(len)) { + let Some(depth) = self.arena.get_depth(diff.idx) else { + warn!("{:?} is not in arena. It could be a dangling container that was deleted before the shallow start version.", self.arena.idx_to_id(diff.idx)); + continue; + }; + let this_depth = depth.get(); + while this_depth > last_depth { + // Clear `to_revive` when we are going to process a new level + // so that we can process the revival of the next level + let to_create = std::mem::take(&mut to_revive_in_this_layer); + to_revive_in_this_layer = std::mem::take(&mut to_revive_in_next_layer); + for new in to_create { + let state = self.store.get_or_create_mut(new); + if state.is_state_empty() { + continue; + } + + let external_diff = state.to_diff(&self.doc); + trigger_on_new_container( + &external_diff, + |cid| { + to_revive_in_this_layer.insert(cid); + }, + &self.arena, + ); + + diffs.push(InternalContainerDiff { + idx: new, + bring_back: true, + diff: external_diff.into(), + diff_mode: DiffMode::Checkout, + }); + } + + last_depth += 1; + } + + let idx = diff.idx; + let internal_diff = std::mem::take(&mut diff.diff); + match &internal_diff { + crate::event::DiffVariant::None => { + if is_recording { + let state = self.store.get_or_create_mut(diff.idx); + let extern_diff = state.to_diff(&self.doc); + trigger_on_new_container( + &extern_diff, + |cid| { + to_revive_in_next_layer.insert(cid); + }, + &self.arena, + ); + diff.diff = extern_diff.into(); + } + } + crate::event::DiffVariant::Internal(_) => { + let cid = self.arena.idx_to_id(idx).unwrap(); + info_span!("apply diff on", container_id = ?cid).in_scope( + || -> LoroResult<()> { + if self.in_txn { + self.changed_idx_in_txn.insert(idx); + } + let state = self.store.get_or_create_mut(idx); + if is_recording { + // process bring_back before apply + let external_diff = + if diff.bring_back || to_revive_in_this_layer.contains(&idx) { + state.apply_diff( + internal_diff.into_internal().unwrap(), + DiffApplyContext { + mode: diff.diff_mode, + doc: &self.doc, + }, + )?; + state.to_diff(&self.doc) + } else { + state.apply_diff_and_convert( + internal_diff.into_internal().unwrap(), + DiffApplyContext { + mode: diff.diff_mode, + doc: &self.doc, + }, + ) + }; + trigger_on_new_container( + &external_diff, + |cid| { + to_revive_in_next_layer.insert(cid); + }, + &self.arena, + ); + diff.diff = external_diff.into(); + } else { + state.apply_diff( + internal_diff.into_internal().unwrap(), + DiffApplyContext { + mode: diff.diff_mode, + doc: &self.doc, + }, + )?; + } + Ok(()) + }, + )?; + } + crate::event::DiffVariant::External(_) => unreachable!(), + } + + to_revive_in_this_layer.remove(&idx); + if !diff.diff.is_empty() { + diffs.push(diff); + } + } + + // Revive the last several layers + while !to_revive_in_this_layer.is_empty() || !to_revive_in_next_layer.is_empty() { + let to_create = std::mem::take(&mut to_revive_in_this_layer); + for new in to_create { + let state = self.store.get_or_create_mut(new); + if state.is_state_empty() { + continue; + } + + let external_diff = state.to_diff(&self.doc); + trigger_on_new_container( + &external_diff, + |cid| { + to_revive_in_next_layer.insert(cid); + }, + &self.arena, + ); + + if !external_diff.is_empty() { + diffs.push(InternalContainerDiff { + idx: new, + bring_back: true, + diff: external_diff.into(), + diff_mode: DiffMode::Checkout, + }); + } + } + + to_revive_in_this_layer = std::mem::take(&mut to_revive_in_next_layer); + } + + diff.diff = diffs.into(); + self.frontiers = diff.new_version.clone().into_owned(); + + if self.is_recording() { + self.record_diff(diff) + } + Ok(()) + } + + fn validate_diff_batch(&mut self, diffs: &[InternalContainerDiff]) -> LoroResult<()> { + for diff in diffs { + let crate::event::DiffVariant::Internal(internal_diff) = &diff.diff else { + continue; + }; + if let Some(state) = self.store.get_container(diff.idx) { + state.validate_diff(internal_diff)?; + } else { + let state = create_state_(diff.idx, &self.config, self.peer_id()); + state.validate_diff(internal_diff)?; + } + } + + Ok(()) + } + + pub fn apply_local_op(&mut self, raw_op: &RawOp, op: &Op) -> LoroResult<()> { + // set parent first, `MapContainer` will only be created for TreeID that does not contain + self.set_container_parent_by_raw_op(raw_op); + let state = self.store.get_or_create_mut(op.container); + if self.in_txn { + self.changed_idx_in_txn.insert(op.container); + } + let ret = state.apply_local_op(raw_op, op)?; + if !ret.deleted_containers.is_empty() { + self.dead_containers_cache.clear_alive(); + } + + Ok(()) + } + + pub(crate) fn start_txn(&mut self, origin: InternalString, trigger: EventTriggerKind) { + self.pre_txn(origin, trigger); + self.in_txn = true; + } + + pub(crate) fn abort_txn(&mut self) { + self.in_txn = false; + } + + pub fn iter_and_decode_all(&mut self) -> impl Iterator { + self.store.iter_and_decode_all() + } + + pub(crate) fn iter_all_containers_mut( + &mut self, + ) -> impl Iterator { + self.store.iter_all_containers() + } + + pub fn does_container_exist(&mut self, id: &ContainerID) -> bool { + // A container may exist even if not yet registered in the arena. + // Check arena first, then fall back to KV presence in the store. + let is_mergeable = id.is_mergeable(); + if id.is_root() && !is_mergeable { + return true; + } + + if !is_mergeable { + if let Some(idx) = self.arena.id_to_idx(id) { + if self.arena.get_depth(idx).is_some() { + return true; + } + } + } + + self.store.contains_id(id) + } + + pub(crate) fn commit_txn(&mut self, new_frontiers: Frontiers, diff: Option) { + self.in_txn = false; + self.frontiers = new_frontiers; + if let Some(diff) = diff { + if self.is_recording() { + self.record_diff(diff); + } + } + } + + /// Ensure the container is created and will be encoded in the next `encode` call + #[inline] + pub(crate) fn ensure_container(&mut self, id: &ContainerID) { + self.store.ensure_container(id); + } + + /// Ensure all alive containers are created in DocState and will be encoded in the next `encode` call + pub(crate) fn ensure_all_alive_containers(&mut self) -> FxHashSet { + // TODO: PERF This can be optimized because we shouldn't need to call get_value for + // all the containers every time we export + let ans = self.get_all_alive_containers(); + for id in ans.iter() { + self.ensure_container(id); + } + + ans + } + + pub(crate) fn get_value_by_idx(&mut self, container_idx: ContainerIdx) -> LoroValue { + self.store + .get_value(container_idx) + .unwrap_or_else(|| container_idx.get_type().default_value()) + } + + pub(crate) fn get_map_value_by_key( + &mut self, + container_idx: ContainerIdx, + key: &str, + ) -> Option { + self.store.map_get(container_idx, key) + } + + pub(crate) fn get_map_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.map_len(container_idx) + } + + pub(crate) fn get_map_keys(&mut self, container_idx: ContainerIdx) -> Vec { + self.store.map_keys(container_idx) + } + + pub(crate) fn get_map_entries( + &mut self, + container_idx: ContainerIdx, + ) -> Vec<(InternalString, LoroValue)> { + self.store.map_entries(container_idx) + } + + pub(crate) fn get_list_value_at( + &mut self, + container_idx: ContainerIdx, + index: usize, + ) -> Option { + self.store.list_get(container_idx, index) + } + + pub(crate) fn get_list_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.list_len(container_idx) + } + + pub(crate) fn get_list_values(&mut self, container_idx: ContainerIdx) -> Vec { + self.store.list_values(container_idx) + } + + pub(crate) fn get_text_unicode_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.text_unicode_len(container_idx).unwrap_or(0) + } + + pub(crate) fn get_text_utf16_len(&mut self, container_idx: ContainerIdx) -> usize { + self.store.text_utf16_len(container_idx).unwrap_or(0) + } + + pub(crate) fn has_decoded_container_state(&mut self, container_idx: ContainerIdx) -> bool { + self.store.has_decoded_state(container_idx) + } + + /// Set the state of the container with the given container idx. + /// This is only used for decode. + /// + /// # Panic + /// + /// If the state is not empty. + pub(super) fn init_with_states_and_version( + &mut self, + frontiers: Frontiers, + oplog: &OpLog, + unknown_containers: Vec, + need_to_register_parent: bool, + origin: InternalString, + ) -> LoroResult<()> { + self.pre_txn(Default::default(), EventTriggerKind::Import); + if need_to_register_parent { + for state in self.store.iter_and_decode_all() { + let idx = state.container_idx(); + let s = state; + for child_id in s.get_child_containers() { + let child_idx = self.arena.register_container(&child_id); + self.arena.set_parent(child_idx, Some(idx)); + } + } + } + + if !unknown_containers.is_empty() { + let mut diff_calc = DiffCalculator::new(false); + let stack_vv; + let vv = if oplog.frontiers() == &frontiers { + oplog.vv() + } else { + stack_vv = oplog.dag().frontiers_to_vv(&frontiers); + stack_vv.as_ref().unwrap() + }; + + let (unknown_diffs, _diff_mode) = diff_calc.calc_diff_internal_tagged( + oplog, + &Default::default(), + &Default::default(), + vv, + &frontiers, + Some(&|idx| !idx.is_unknown() && unknown_containers.contains(&idx)), + "unknown-containers", + ); + self.apply_diff( + InternalDocDiff { + origin: origin.clone(), + by: EventTriggerKind::Import, + diff: unknown_diffs.into(), + new_version: Cow::Owned(frontiers.clone()), + }, + DiffMode::Checkout, + )?; + } + + if self.is_recording() { + let diff: Vec<_> = self + .store + .iter_all_containers() + .map(|(idx, state)| InternalContainerDiff { + idx, + bring_back: false, + diff: state + .get_state_mut( + idx, + ContainerCreationContext { + configure: &self.config, + peer: self.peer.load(Ordering::Relaxed), + }, + ) + .to_diff(&self.doc) + .into(), + diff_mode: DiffMode::Checkout, + }) + .collect(); + + self.record_diff(InternalDocDiff { + origin, + by: EventTriggerKind::Import, + diff: diff.into(), + new_version: Cow::Borrowed(&frontiers), + }); + } + + self.frontiers = frontiers; + Ok(()) + } + + #[inline(always)] + #[allow(unused)] + pub(crate) fn with_state(&mut self, idx: ContainerIdx, f: F) -> R + where + F: FnOnce(&State) -> R, + { + let depth = self.arena.get_depth(idx).unwrap().get() as usize; + let state = self.store.get_or_create_imm(idx); + f(state) + } + + #[inline(always)] + pub(crate) fn with_state_mut(&mut self, idx: ContainerIdx, f: F) -> R + where + F: FnOnce(&mut State) -> R, + { + let state = self.store.get_or_create_mut(idx); + f(state) + } + + pub(super) fn is_in_txn(&self) -> bool { + self.in_txn + } + + pub fn can_import_snapshot(&self) -> bool { + !self.in_txn && self.arena.can_import_snapshot() && self.store.can_import_snapshot() + } + + pub(crate) fn reset_to_empty_for_failed_snapshot_import(&mut self) { + self.frontiers = Frontiers::default(); + self.store = + ContainerStore::new(self.arena.clone(), self.config.clone(), self.peer.clone()); + self.in_txn = false; + self.changed_idx_in_txn.clear(); + self.event_recorder = Default::default(); + self.dead_containers_cache = Default::default(); + } + + pub fn get_value(&mut self) -> LoroValue { + let roots = self.preferred_root_containers(); + let ans: loro_common::LoroMapValue = roots + .into_iter() + .map(|idx| { + let id = self.arena.idx_to_id(idx).unwrap(); + let ContainerID::Root { + name, + container_type: _, + } = &id + else { + unreachable!() + }; + (name.to_string(), LoroValue::Container(id)) + }) + .collect(); + LoroValue::Map(ans) + } + + pub fn get_deep_value(&mut self) -> LoroValue { + let roots = self.preferred_root_containers(); + let mut ans = FxHashMap::with_capacity_and_hasher(roots.len(), Default::default()); + let binding = self.config.deleted_root_containers.clone(); + let deleted_root_container = binding.lock(); + let should_hide_empty_root_container = self + .config + .hide_empty_root_containers + .load(Ordering::Relaxed); + for root_idx in roots { + let id = self.arena.idx_to_id(root_idx).unwrap(); + match &id { + loro_common::ContainerID::Root { name, .. } => { + let v = self.get_container_deep_value(root_idx); + if should_hide_empty_root_container + && visible_container_value_is_empty(root_idx.get_type(), &v) + { + continue; + } + + if deleted_root_container.contains(&id) + && deleted_root_container_value_is_cleared(root_idx.get_type(), &v) + { + continue; + } + + ans.insert(name.to_string(), v); + } + loro_common::ContainerID::Normal { .. } => { + unreachable!() + } + } + } + + LoroValue::Map(ans.into()) + } + + pub fn get_deep_value_with_id(&mut self) -> LoroValue { + let roots = self.preferred_root_containers(); + let mut ans = FxHashMap::with_capacity_and_hasher(roots.len(), Default::default()); + for root_idx in roots { + let id = self.arena.idx_to_id(root_idx).unwrap(); + match id.clone() { + loro_common::ContainerID::Root { name, .. } => { + ans.insert( + name.to_string(), + self.get_container_deep_value_with_id(root_idx, Some(id)), + ); + } + loro_common::ContainerID::Normal { .. } => { + unreachable!() + } + } + } + + LoroValue::Map(ans.into()) + } + + pub(crate) fn preferred_root_containers(&mut self) -> Vec { + let flag = self.store.load_root_containers(); + // Mergeable cids live in a private namespace and are logically children of a regular + // Map — they must not appear here. `top_level_root_containers` already excludes them, + // so this loop is O(top_level_roots) regardless of the number of mergeable cids. + let roots = self.arena.top_level_root_containers(flag); + let mut selected = FxHashMap::default(); + let mut names = Vec::new(); + + for idx in roots { + let Some(id) = self.arena.idx_to_id(idx) else { + continue; + }; + if !self.store.contains_id(&id) { + continue; + } + let Some(name) = self.root_container_name(idx) else { + continue; + }; + let is_empty = self.root_container_is_empty(idx); + match selected.entry(name.clone()) { + std::collections::hash_map::Entry::Vacant(entry) => { + names.push(name); + entry.insert((idx, is_empty)); + } + std::collections::hash_map::Entry::Occupied(mut entry) => { + let (_, selected_is_empty) = entry.get(); + // Keep the previous last-root-wins behavior, except an empty + // root should not hide a non-empty root with the same name. + if *selected_is_empty || !is_empty { + entry.insert((idx, is_empty)); + } + } + } + } + + names + .into_iter() + .filter_map(|name| selected.remove(&name).map(|(idx, _)| idx)) + .collect() + } + + pub(crate) fn preferred_root_container_idx_by_key( + &mut self, + root_index: &InternalString, + ) -> Option { + let flag = self.store.load_root_containers(); + // Same reasoning as `preferred_root_containers`: only top-level roots are eligible. + let roots = self.arena.top_level_root_containers(flag); + let mut selected = None; + + for idx in roots { + let Some(id) = self.arena.idx_to_id(idx) else { + continue; + }; + if !self.store.contains_id(&id) { + continue; + } + let Some(name) = self.root_container_name(idx) else { + continue; + }; + if &name != root_index { + continue; + } + + let is_empty = self.root_container_is_empty(idx); + match selected { + None => selected = Some((idx, is_empty)), + Some((_, selected_is_empty)) => { + if selected_is_empty || !is_empty { + selected = Some((idx, is_empty)); + } + } + } + } + + selected.map(|(idx, _)| idx) + } + + fn root_container_name(&self, idx: ContainerIdx) -> Option { + match self.arena.idx_to_id(idx)? { + ContainerID::Root { name, .. } => Some(name), + ContainerID::Normal { .. } => None, + } + } + + fn root_container_is_empty(&mut self, idx: ContainerIdx) -> bool { + let value = self + .store + .get_value(idx) + .unwrap_or_else(|| idx.get_type().default_value()); + visible_container_value_is_empty(idx.get_type(), &value) + } + + pub fn get_all_container_value_flat(&mut self) -> LoroValue { + let mut map = FxHashMap::default(); + self.store.iter_and_decode_all().for_each(|c| { + let value = c.get_value(); + let cid = self.arena.idx_to_id(c.container_idx()).unwrap().to_string(); + map.insert(cid, value); + }); + + LoroValue::Map(map.into()) + } + + pub(crate) fn get_container_deep_value_with_id( + &mut self, + container: ContainerIdx, + id: Option, + ) -> LoroValue { + let id = id.unwrap_or_else(|| self.arena.idx_to_id(container).unwrap()); + let Some(value) = self.store.get_value(container) else { + return container.get_type().default_value(); + }; + let cid_str = LoroValue::String(format!("idx:{}, id:{}", container.to_index(), id).into()); + match value { + LoroValue::Container(_) => unreachable!(), + LoroValue::List(mut list) => { + if container.get_type() == ContainerType::Tree { + get_meta_value(list.make_mut(), self); + } else { + if list.iter().all(|x| !x.is_container()) { + return LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => LoroValue::List(list) + )) + .into(), + ); + } + + let list_mut = list.make_mut(); + for item in list_mut.iter_mut() { + if item.is_container() { + let container = item.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let value = self.get_container_deep_value_with_id( + container_idx, + Some(container.clone()), + ); + *item = value; + } + } + } + LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => LoroValue::List(list) + )) + .into(), + ) + } + LoroValue::Map(mut map) => { + // A map's mergeable children are encoded as compact binary markers in the map's + // own value table (loro-dev/loro#759), so they are already present in `map` here. + // Derive the active children straight from that value — re-fetching the + // `MapState` would force the snapshot-backed container to decode and break the + // lazy-value invariant for roots. + let mergeable_children = self.mergeable_children_from_value(&id, &map); + + let map_mut = map.make_mut(); + for (_key, value) in map_mut.iter_mut() { + if value.is_container() { + let container = value.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let new_value = self.get_container_deep_value_with_id( + container_idx, + Some(container.clone()), + ); + *value = new_value; + } + } + // Replace each marker with the nested deep value of its resolved child, keyed + // under the same logical key. + for (key, cid) in mergeable_children { + let child_idx = self.arena.register_container(&cid); + let new_value = self.get_container_deep_value_with_id(child_idx, Some(cid)); + map_mut.insert(key.to_string(), new_value); + } + + LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => LoroValue::Map(map) + )) + .into(), + ) + } + _ => LoroValue::Map( + (fx_map!( + "cid".into() => cid_str, + "value".into() => value + )) + .into(), + ), + } + } + + pub fn get_container_deep_value(&mut self, container: ContainerIdx) -> LoroValue { + let Some(value) = self.store.get_value(container) else { + return container.get_type().default_value(); + }; + match value { + LoroValue::Container(_) => unreachable!(), + LoroValue::List(mut list) => { + if container.get_type() == ContainerType::Tree { + // Each tree node has an associated map container to represent + // the metadata of this node. When the user get the deep value, + // we need to add a field named `meta` to the tree node, + // whose value is deep value of map container. + get_meta_value(list.make_mut(), self); + } else { + if list.iter().all(|x| !x.is_container()) { + return LoroValue::List(list); + } + + let list_mut = list.make_mut(); + for item in list_mut.iter_mut() { + if item.is_container() { + let container = item.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let value = self.get_container_deep_value(container_idx); + *item = value; + } + } + } + LoroValue::List(list) + } + LoroValue::Map(mut map) => { + // A map's mergeable children are encoded as compact binary markers in the map's + // own value table (loro-dev/loro#759), so they are already present in `map` here. + // Derive the active children straight from that value — re-fetching the + // `MapState` would force the snapshot-backed container to decode and break the + // lazy-value invariant for roots. + let mergeable_children = self + .arena + .idx_to_id(container) + .map(|parent_id| self.mergeable_children_from_value(&parent_id, &map)) + .unwrap_or_default(); + + if mergeable_children.is_empty() && map.iter().all(|x| !x.1.is_container()) { + return LoroValue::Map(map); + } + + let map_mut = map.make_mut(); + for (_key, value) in map_mut.iter_mut() { + if value.is_container() { + let container = value.as_container().unwrap(); + let container_idx = self.arena.register_container(container); + let new_value = self.get_container_deep_value(container_idx); + *value = new_value; + } + } + // Replace each marker with the nested deep value of its resolved child, keyed + // under the same logical key. + for (key, cid) in mergeable_children { + let child_idx = self.arena.register_container(&cid); + let new_value = self.get_container_deep_value(child_idx); + map_mut.insert(key.to_string(), new_value); + } + LoroValue::Map(map) + } + _ => value, + } + } + + pub(crate) fn get_all_alive_containers(&mut self) -> FxHashSet { + let flag = self.store.load_all(); + let mut ans = FxHashSet::default(); + let mut to_visit = self + .arena + .root_containers(flag) + .iter() + .map(|x| self.arena.get_container_id(*x).unwrap()) + .filter(|id| self.store.contains_id(id)) + .collect_vec(); + + while let Some(id) = to_visit.pop() { + self.get_alive_children_of(&id, &mut to_visit); + ans.insert(id); + } + + ans + } + + pub(crate) fn get_alive_children_of(&mut self, id: &ContainerID, ans: &mut Vec) { + let idx = self.arena.register_container(id); + let Some(value) = self.store.get_value(idx) else { + return; + }; + + match value { + LoroValue::Container(_) => unreachable!(), + LoroValue::List(list) => { + if idx.get_type() == ContainerType::Tree { + // Each tree node has an associated map container to represent + // the metadata of this node. When the user get the deep value, + // we need to add a field named `meta` to the tree node, + // whose value is deep value of map container. + let mut list = list.unwrap(); + while let Some(node) = list.pop() { + let map = node.as_map().unwrap(); + let meta = map.get("meta").unwrap(); + let id = meta.as_container().unwrap(); + ans.push(id.clone()); + let children = map.get("children").unwrap(); + let children = children.as_list().unwrap(); + for child in children.iter() { + list.push(child.clone()); + } + } + } else { + for item in list.iter() { + if let LoroValue::Container(id) = item { + ans.push(id.clone()); + } + } + } + } + LoroValue::Map(map) => { + for (_key, value) in map.iter() { + if let LoroValue::Container(id) = value { + ans.push(id.clone()); + } + } + // Mergeable children are resolved from compact binary markers stored in this + // map's own value table (loro-dev/loro#759): the active child at each key is the + // deterministic cid for whichever marker the map's regular LWW resolved to. + // Derive them from the value we already fetched lazily above — re-reading the + // decoded `MapState` would force a snapshot-backed map to materialize its full + // state. Pull them in so alive-container walks (notably shallow snapshot export) + // include mergeable cids and don't filter their KV out by `retain_keys`. + let mergeable_cids: Vec = self + .arena + .idx_to_id(idx) + .map(|parent_id| { + self.mergeable_children_from_value(&parent_id, &map) + .into_iter() + .map(|(_key, cid)| cid) + .collect() + }) + .unwrap_or_default(); + for cid in mergeable_cids { + self.arena.register_container(&cid); + ans.push(cid); + } + } + _ => {} + } + } + + // Because we need to calculate path based on [DocState], so we cannot extract + // the event recorder to a separate module. + /// Flowstate §perf: pair composition for the event fold. External text + /// diffs take the LINEAR walk below; everything else keeps the stock + /// `DiffVariant::compose`. + fn compose_pair(first: crate::event::DiffVariant, second: crate::event::DiffVariant) -> crate::event::DiffVariant { + use crate::event::{Diff, DiffVariant}; + match (first, second) { + (DiffVariant::External(Diff::Text(a)), DiffVariant::External(Diff::Text(b))) => { + DiffVariant::External(Diff::Text(Self::compose_text_linear(&a, &b))) + } + (first, second) => first.compose(second).unwrap(), + } + } + + /// Flowstate §perf: LINEAR two-way composition for external text diffs — + /// the standard O(|a| + |b|) flat-item delta composition. The stock + /// `DeltaRope::compose` degrades quadratically in span count: composing + /// the ~12k-span deltas an undo/redo of a mass styled edit records took + /// ~1s per pair (a multi-minute UI freeze at document scale; the ctrl-A + + /// undo field bug). Attr semantics mirror `TextMeta::compose` exactly + /// (plain insert-overwrite, later diff wins). + fn compose_text_linear(a: &crate::event::TextDiff, b: &crate::event::TextDiff) -> crate::event::TextDiff { + use crate::event::TextMeta; + use crate::utils::string_slice::StringSlice; + use generic_btree::rle::{HasLength as _, Sliceable as _}; + use loro_delta::{DeltaItem, DeltaRopeBuilder}; + use std::collections::VecDeque; + + enum Item { + Ret { len: usize, meta: TextMeta }, + // `len` caches `value.rle_len()`: the unicode length of a + // `StringSlice` is a full scan, and the compose loop repeatedly + // carves the front off multi-megabyte inserts — recomputing the + // remainder's length each time is quadratic in characters. + Ins { value: StringSlice, len: usize, meta: TextMeta }, + Del { len: usize }, + } + + fn push(items: &mut Vec, item: Item) { + // Coalesce runs so the output shape matches what rope composition + // would produce for the same content. + match (items.last_mut(), &item) { + (Some(Item::Ret { len, meta }), Item::Ret { len: next_len, meta: next_meta }) if meta.0 == next_meta.0 => { + *len += next_len; + } + (Some(Item::Del { len }), Item::Del { len: next_len }) => *len += next_len, + _ => items.push(item), + } + } + + fn explode(diff: &crate::event::TextDiff) -> VecDeque { + let mut out = VecDeque::new(); + for item in diff.iter() { + match item { + DeltaItem::Retain { len, attr } => out.push_back(Item::Ret { + len: *len, + meta: attr.clone(), + }), + DeltaItem::Replace { value, attr, delete } => { + let value_len = value.rle_len(); + if value_len > 0 { + out.push_back(Item::Ins { + value: value.clone(), + len: value_len, + meta: attr.clone(), + }); + } + if *delete > 0 { + out.push_back(Item::Del { len: *delete }); + } + } + } + } + out + } + + fn merge_meta(base: &TextMeta, over: &TextMeta) -> TextMeta { + if over.0.is_empty() { + return base.clone(); + } + let mut merged = base.clone(); + for (key, value) in over.0.iter() { + merged.0.insert(key.clone(), value.clone()); + } + merged + } + + let mut a_items = explode(a); + let mut b_items = explode(b); + let mut out: Vec = Vec::new(); + + while let Some(b_item) = b_items.pop_front() { + match b_item { + Item::Ins { value, len, meta } => push(&mut out, Item::Ins { value, len, meta }), + Item::Ret { mut len, meta } => { + while len > 0 { + match a_items.pop_front() { + None => { + // Retaining original text beyond `a`'s reach. + push(&mut out, Item::Ret { len, meta: meta.clone() }); + len = 0; + } + Some(Item::Del { len: deleted }) => { + // `a`'s deletions are invisible to `b`; pass through. + push(&mut out, Item::Del { len: deleted }); + } + Some(Item::Ret { len: available, meta: a_meta }) => { + let taken = available.min(len); + if available > taken { + a_items.push_front(Item::Ret { + len: available - taken, + meta: a_meta.clone(), + }); + } + push(&mut out, Item::Ret { + len: taken, + meta: merge_meta(&a_meta, &meta), + }); + len -= taken; + } + Some(Item::Ins { + mut value, + len: available, + meta: a_meta, + }) => { + let taken = available.min(len); + if taken < available { + let rest = value.split(taken); + a_items.push_front(Item::Ins { + value: rest, + len: available - taken, + meta: a_meta.clone(), + }); + } + push(&mut out, Item::Ins { + value, + len: taken, + meta: merge_meta(&a_meta, &meta), + }); + len -= taken; + } + } + } + } + Item::Del { mut len } => { + while len > 0 { + match a_items.pop_front() { + None => { + push(&mut out, Item::Del { len }); + len = 0; + } + Some(Item::Del { len: deleted }) => { + push(&mut out, Item::Del { len: deleted }); + } + Some(Item::Ret { len: available, meta: a_meta }) => { + let taken = available.min(len); + if available > taken { + a_items.push_front(Item::Ret { + len: available - taken, + meta: a_meta, + }); + } + push(&mut out, Item::Del { len: taken }); + len -= taken; + } + Some(Item::Ins { + mut value, + len: available, + meta: a_meta, + }) => { + // Inserted by `a`, deleted by `b`: vanishes. + let taken = available.min(len); + if taken < available { + let rest = value.split(taken); + a_items.push_front(Item::Ins { + value: rest, + len: available - taken, + meta: a_meta, + }); + } + len -= taken; + } + } + } + } + } + } + for item in a_items { + push(&mut out, item); + } + + let mut builder: DeltaRopeBuilder = DeltaRopeBuilder::new(); + for item in out { + builder = match item { + Item::Ret { len, meta } => builder.retain(len, meta), + Item::Ins { value, meta, .. } => builder.insert(value, meta), + Item::Del { len } => builder.delete(len), + }; + } + builder.build() + } + + /// Flowstate §perf: compose many sequential diffs of one container in a + /// balanced fold — O(total · log n) instead of the O(n²) + /// clone-and-compose-one-at-a-time accumulation. Diff composition is + /// associative, so pairing adjacent (order-preserving) diffs yields the + /// identical result. + fn balanced_compose(mut diffs: Vec) -> crate::event::DiffVariant { + while diffs.len() > 1 { + let mut next = Vec::with_capacity(diffs.len().div_ceil(2)); + let mut iter = diffs.into_iter(); + while let Some(first) = iter.next() { + match iter.next() { + Some(second) => next.push(Self::compose_pair(first, second)), + None => next.push(first), + } + } + diffs = next; + } + diffs.pop().unwrap() + } + + fn diffs_to_event(&mut self, diffs: Vec>, from: Frontiers) -> DocDiff { + if diffs.is_empty() { + panic!("diffs is empty"); + } + + let triggered_by = diffs[0].by; + debug_assert!(diffs.iter().all(|x| x.by == triggered_by)); + let mut containers = FxHashMap::default(); + let to = (*diffs.last().unwrap().new_version).to_owned(); + let origin = diffs[0].origin.clone(); + for diff in diffs { + #[allow(clippy::unnecessary_to_owned)] + for container_diff in diff.diff.into_owned() { + let Some((pending_container_diffs, _)) = containers.get_mut(&container_diff.idx) else { + if let Some(path) = self.get_path(container_diff.idx) { + containers.insert(container_diff.idx, (vec![container_diff.diff], path)); + } else { + // if we cannot find the path to the container, the container must be overwritten afterwards. + // So we can ignore the diff from it. + let _container_id = self + .arena + .idx_to_id(container_diff.idx) + .map(|x| x.to_string()) + .unwrap_or_else(|| "unknown".to_string()); + #[cfg(feature = "logging")] + loro_common::warn!( + "⚠️ WARNING: ignore event because cannot find its path {:#?} container id:{}", + &container_diff, + _container_id + ); + } + + continue; + }; + // Flowstate §perf: buffer per-container diffs and compose them in a + // balanced fold below. The previous incremental + // `accumulated.clone().compose(next)` was O(n²) in both time and + // allocation once an edit produced many sibling diffs — a + // select-all delete over marked rich text splits into one diff per + // style-anchor segment (12k+ on a real document), which froze the + // editor for minutes inside a single commit. + pending_container_diffs.push(container_diff.diff); + } + } + let mut diff: Vec<_> = containers + .into_iter() + .map(|(container, (diffs, path))| { + let idx = container; + let id = self.arena.get_container_id(idx).unwrap(); + let is_unknown = id.is_unknown(); + + let composed = Self::balanced_compose(diffs); + ContainerDiff { + id, + idx, + diff: composed.into_external().unwrap(), + is_unknown, + path, + } + }) + .collect(); + + // Sort by path length, so caller can apply the diff from the root to the leaf. + // Otherwise, the caller may use a wrong path to apply the diff. + + diff.sort_by_key(|x| { + ( + x.path.len(), + match &x.id { + ContainerID::Root { .. } => 0, + ContainerID::Normal { counter, .. } => *counter + 1, + }, + ) + }); + DocDiff { + from, + to, + origin, + by: triggered_by, + diff, + } + } + + pub(crate) fn get_reachable(&mut self, id: &ContainerID) -> bool { + if id.is_root() && !id.is_mergeable() { + return true; + } + + // If not registered yet, check KV presence for ordinary containers, then register lazily. + // Mergeable children can be active through a parent marker before they have their + // own KV state, so their reachability must be resolved by the logical edge walk below. + if self.arena.id_to_idx(id).is_none() { + if !id.is_mergeable() && !self.does_container_exist(id) { + return false; + } + self.arena.register_container(id); + } + + let mut idx = self.arena.id_to_idx(id).unwrap(); + loop { + let id = self.arena.idx_to_id(idx).unwrap(); + if let Some(parent_idx) = self.arena.get_parent(idx) { + if !self.contains_logical_child(parent_idx, &id) { + return false; + } + idx = parent_idx; + } else { + if id.is_root() && !id.is_mergeable() { + return true; + } + + return false; + } + } + } + + // the container may be override, so it may return None + pub(super) fn get_path(&mut self, idx: ContainerIdx) -> Option> { + let mut ans = Vec::new(); + let mut idx = idx; + loop { + let id = self.arena.idx_to_id(idx).unwrap(); + if let Some(parent_idx) = self.arena.get_parent(idx) { + let Some(prop) = self.get_logical_child_index(parent_idx, &id) else { + tracing::warn!("Missing in parent's children"); + return None; + }; + ans.push((id, prop)); + idx = parent_idx; + } else { + // this container may be deleted + if id.is_mergeable() { + tracing::info!(id = %id, "Missing parent - mergeable container is inactive"); + return None; + } + let Ok(prop) = id.clone().into_root() else { + let id = format!("{}", &id); + tracing::info!(?id, "Missing parent - container is deleted"); + return None; + }; + ans.push((id, Index::Key(prop.0))); + break; + } + } + + ans.reverse(); + + Some(ans) + } + + pub(crate) fn check_before_decode_snapshot(&self) -> LoroResult<()> { + if self.is_in_txn() { + return Err(LoroError::DecodeError( + "State is in txn".to_string().into_boxed_str(), + )); + } + + if !self.can_import_snapshot() { + return Err(LoroError::DecodeError( + "State is not empty, cannot import snapshot directly" + .to_string() + .into_boxed_str(), + )); + } + + Ok(()) + } + + /// Check whether two [DocState]s are the same. Panic if not. + /// + /// Compared to check equality on `get_deep_value`, this function also checks the equality on richtext + /// styles and states that are not reachable from the root. + /// + /// This is only used for test. + pub(crate) fn check_is_the_same(&mut self, other: &mut Self) { + fn get_entries_for_state( + arena: &SharedArena, + state: &mut State, + ) -> Option<(ContainerID, (ContainerIdx, LoroValue))> { + if state.is_state_empty() { + return None; + } + + let id = arena.idx_to_id(state.container_idx()).unwrap(); + let value = match state { + State::RichtextState(s) => s.get_richtext_value(), + _ => state.get_value(), + }; + if match &value { + LoroValue::List(l) => l.is_empty(), + LoroValue::Map(m) => m.is_empty(), + _ => false, + } { + return None; + } + #[cfg(feature = "counter")] + if id.container_type() == ContainerType::Counter { + if let LoroValue::Double(c) = value { + if c.abs() < f64::EPSILON { + return None; + } + } + } + + Some((id, (state.container_idx(), value))) + } + + let self_id_to_states: FxHashMap = self + .store + .iter_and_decode_all() + .filter_map(|state: &mut State| { + let arena = &self.arena; + get_entries_for_state(arena, state) + }) + .collect(); + let mut other_id_to_states: FxHashMap = other + .store + .iter_and_decode_all() + .filter_map(|state: &mut State| { + let arena = &other.arena; + get_entries_for_state(arena, state) + }) + .collect(); + for (id, (idx, this_value)) in self_id_to_states { + let (_, other_value) = match other_id_to_states.remove(&id) { + Some(x) => x, + None => { + panic!( + "id: {:?}, path: {:?} is missing, value={:?}", + id, + self.get_path(idx), + &this_value + ); + } + }; + + pretty_assertions::assert_eq!( + this_value, + other_value, + "[self!=other] id: {:?}, path: {:?}", + id, + self.get_path(idx) + ); + } + + if !other_id_to_states.is_empty() { + panic!("other has more states {:#?}", &other_id_to_states); + } + } + + pub fn create_state(&self, idx: ContainerIdx) -> State { + let config = &self.config; + let peer = self.peer.load(std::sync::atomic::Ordering::Relaxed); + create_state_(idx, config, peer) + } + + pub fn create_unknown_state(&self, idx: ContainerIdx) -> State { + State::UnknownState(UnknownState::new(idx)) + } + + pub fn get_relative_position(&mut self, pos: &Cursor, use_event_index: bool) -> Option { + let idx = self.arena.register_container(&pos.container); + let state = self.store.get_container_mut(idx)?; + if let Some(id) = pos.id { + match state { + State::ListState(s) => s.get_index_of_id(id), + State::RichtextState(s) => s.get_text_index_of_id(id, use_event_index), + State::MovableListState(s) => s.get_index_of_id(id), + State::MapState(_) | State::TreeState(_) | State::UnknownState(_) => unreachable!(), + #[cfg(feature = "counter")] + State::CounterState(_) => unreachable!(), + } + } else { + if matches!(pos.side, crate::cursor::Side::Left) { + return Some(0); + } + + match state { + State::ListState(s) => Some(s.len()), + State::RichtextState(s) => Some(if use_event_index { + s.len_event() + } else { + s.len_unicode() + }), + State::MovableListState(s) => Some(s.len()), + State::MapState(_) | State::TreeState(_) | State::UnknownState(_) => unreachable!(), + #[cfg(feature = "counter")] + State::CounterState(_) => unreachable!(), + } + } + } + + /// §perf (flowstate vendor patch): batch variant of [`Self::get_relative_position`] + /// for a text container — resolves the positions of many live ids in a single + /// pass. `out[i]` is `None` when `ids[i]` is not currently live (caller falls + /// back to per-id resolution), or when `container` is missing / not a text + /// container. + pub fn get_text_relative_positions( + &mut self, + container: &ContainerID, + ids: &[ID], + use_event_index: bool, + ) -> Vec> { + let idx = self.arena.register_container(container); + let Some(state) = self.store.get_container_mut(idx) else { + return vec![None; ids.len()]; + }; + match state { + State::RichtextState(s) => s.get_text_indices_of_ids(ids, use_event_index), + _ => vec![None; ids.len()], + } + } + + pub fn get_value_by_path(&mut self, path: &[Index]) -> Option { + if path.is_empty() { + return None; + } + + enum CurContainer { + Container(ContainerIdx), + TreeNode { + tree: ContainerIdx, + node: Option, + }, + } + + let mut state_idx = { + let root_index = path[0].as_key()?; + CurContainer::Container(self.preferred_root_container_idx_by_key(root_index)?) + }; + + if path.len() == 1 { + if let CurContainer::Container(c) = state_idx { + let cid = self.arena.idx_to_id(c)?; + return Some(LoroValue::Container(cid)); + } + } + + let mut i = 1; + while i < path.len() - 1 { + let index = &path[i]; + match state_idx { + CurContainer::Container(idx) => { + let parent_id = self.arena.idx_to_id(idx); + let parent_state = self.store.get_container_mut(idx)?; + match parent_state { + State::ListState(l) => { + let Some(LoroValue::Container(c)) = l.get(*index.as_seq()?) else { + return None; + }; + state_idx = CurContainer::Container(self.arena.register_container(c)); + } + State::MovableListState(l) => { + let Some(LoroValue::Container(c)) = + l.get(*index.as_seq()?, IndexType::ForUser) + else { + return None; + }; + state_idx = CurContainer::Container(self.arena.register_container(c)); + } + State::MapState(m) => { + let key = index.as_key()?; + let value = m.get(key)?; + let c = match value { + LoroValue::Container(c) => c.clone(), + value => { + let parent_id = parent_id?; + let kind = loro_common::parse_mergeable_marker( + &parent_id, key, value, + )?; + ContainerID::new_mergeable(&parent_id, key, kind) + } + }; + state_idx = CurContainer::Container(self.arena.register_container(&c)); + } + State::RichtextState(_) => return None, + State::TreeState(_) => { + state_idx = CurContainer::TreeNode { + tree: idx, + node: None, + }; + continue; + } + #[cfg(feature = "counter")] + State::CounterState(_) => return None, + State::UnknownState(_) => unreachable!(), + } + } + CurContainer::TreeNode { tree, node } => match index { + Index::Key(internal_string) => { + let node = node?; + let idx = self + .arena + .register_container(&node.associated_meta_container()); + let map = self.store.get_container(idx)?; + let Some(LoroValue::Container(c)) = + map.as_map_state().unwrap().get(internal_string) + else { + return None; + }; + + state_idx = CurContainer::Container(self.arena.register_container(c)); + } + Index::Seq(i) => { + let tree_state = + self.store.get_container_mut(tree)?.as_tree_state().unwrap(); + let parent: TreeParentId = if let Some(node) = node { + node.into() + } else { + TreeParentId::Root + }; + let child = tree_state.get_children(&parent)?.nth(*i)?; + state_idx = CurContainer::TreeNode { + tree, + node: Some(child), + }; + } + Index::Node(tree_id) => { + let tree_state = + self.store.get_container_mut(tree)?.as_tree_state().unwrap(); + if tree_state.parent(tree_id).is_some() { + state_idx = CurContainer::TreeNode { + tree, + node: Some(*tree_id), + } + } else { + return None; + } + } + }, + } + i += 1; + } + + let parent_idx = match state_idx { + CurContainer::Container(container_idx) => container_idx, + CurContainer::TreeNode { tree, node } => { + if let Some(node) = node { + self.arena + .register_container(&node.associated_meta_container()) + } else { + tree + } + } + }; + + let index = path.last().unwrap(); + let parent_id = self.arena.idx_to_id(parent_idx); + let parent_state = self.store.get_or_create_mut(parent_idx); + let value: LoroValue = match parent_state { + State::ListState(l) => l.get(*index.as_seq()?).cloned()?, + State::MovableListState(l) => l.get(*index.as_seq()?, IndexType::ForUser).cloned()?, + State::MapState(m) => { + if let Some(key) = index.as_key() { + let value = m.get(key).cloned()?; + if let Some(parent_id) = &parent_id { + if let Some(kind) = + loro_common::parse_mergeable_marker(parent_id, key, &value) + { + let cid = ContainerID::new_mergeable(parent_id, key, kind); + LoroValue::Container(cid) + } else { + value + } + } else { + value + } + } else if let CurContainer::TreeNode { tree, node } = state_idx { + match index { + Index::Seq(index) => { + let tree_state = + self.store.get_container_mut(tree)?.as_tree_state().unwrap(); + let parent: TreeParentId = if let Some(node) = node { + node.into() + } else { + TreeParentId::Root + }; + let child = tree_state.get_children(&parent)?.nth(*index)?; + child.associated_meta_container().into() + } + Index::Node(id) => id.associated_meta_container().into(), + _ => return None, + } + } else { + return None; + } + } + State::RichtextState(s) => { + let s = s.to_string_mut(); + s.chars() + .nth(*index.as_seq()?) + .map(|c| c.to_string().into())? + } + State::TreeState(_) => { + let id = index.as_node()?; + let cid = id.associated_meta_container(); + cid.into() + } + #[cfg(feature = "counter")] + State::CounterState(_) => unreachable!(), + State::UnknownState(_) => unreachable!(), + }; + + Some(value) + } + + pub(crate) fn shallow_root_store(&self) -> Option<&Arc> { + self.store.shallow_root_store() + } +} + +fn create_state_(idx: ContainerIdx, config: &Configure, peer: u64) -> State { + match idx.get_type() { + ContainerType::Map => State::MapState(Box::new(MapState::new(idx))), + ContainerType::List => State::ListState(Box::new(ListState::new(idx))), + ContainerType::Text => State::RichtextState(Box::new(RichtextState::new( + idx, + config.text_style_config.clone(), + ))), + ContainerType::Tree => State::TreeState(Box::new(TreeState::new(idx, peer))), + ContainerType::MovableList => State::MovableListState(Box::new(MovableListState::new(idx))), + #[cfg(feature = "counter")] + ContainerType::Counter => { + State::CounterState(Box::new(counter_state::CounterState::new(idx))) + } + ContainerType::Unknown(_) => State::UnknownState(UnknownState::new(idx)), + } +} + +fn trigger_on_new_container( + state_diff: &Diff, + mut listener: impl FnMut(ContainerIdx), + arena: &SharedArena, +) { + match state_diff { + Diff::List(list) => { + for delta in list.iter() { + if let DeltaItem::Replace { + value, + attr, + delete: _, + } = delta + { + if attr.from_move { + continue; + } + + for v in value.iter() { + if let ValueOrHandler::Handler(h) = v { + let idx = h.container_idx(); + listener(idx); + } + } + } + } + } + Diff::Map(map) => { + for (_, v) in map.updated.iter() { + if let Some(ValueOrHandler::Handler(h)) = &v.value { + let idx = h.container_idx(); + listener(idx); + } + } + } + Diff::Tree(tree) => { + for item in tree.iter() { + if matches!(item.action, TreeExternalDiff::Create { .. }) { + let id = item.target.associated_meta_container(); + // Ensure registration instead of assuming it's already in arena + listener(arena.register_container(&id)); + } + } + } + _ => {} + }; +} + +#[derive(Default, Clone)] +struct EventRecorder { + recording_diff: bool, + // A batch of diffs will be converted to a event when + // they cannot be merged with the next diff. + diffs: Vec>, + events: Vec, + diff_start_version: Option, +} + +impl EventRecorder { + #[allow(unused)] + pub fn new() -> Self { + Self::default() + } +} + +#[test] +fn test_size() { + println!("Size of State = {}", std::mem::size_of::()); + println!("Size of MapState = {}", std::mem::size_of::()); + println!("Size of ListState = {}", std::mem::size_of::()); + println!( + "Size of TextState = {}", + std::mem::size_of::() + ); + println!("Size of TreeState = {}", std::mem::size_of::()); +} diff --git a/vendor/loro-internal/src/state/analyzer.rs b/vendor/loro-internal/src/state/analyzer.rs new file mode 100644 index 00000000..1ecef1ce --- /dev/null +++ b/vendor/loro-internal/src/state/analyzer.rs @@ -0,0 +1,93 @@ +use crate::{change::Timestamp, LoroDoc}; +use loro_common::ContainerID; +use rle::HasLength; +use rustc_hash::FxHashMap; + +#[derive(Debug, Clone)] +pub struct DocAnalysis { + pub containers: FxHashMap, +} + +#[derive(Debug, Clone)] +pub struct ContainerAnalysisInfo { + pub size: u32, + pub dropped: bool, + pub depth: u32, + pub ops_num: u32, + pub last_edit_time: Timestamp, +} + +impl DocAnalysis { + pub fn analyze(doc: &LoroDoc) -> Self { + let mut ops_nums = FxHashMap::default(); + let mut last_edit_time = FxHashMap::default(); + { + let oplog = doc.oplog().lock(); + oplog.change_store().visit_all_changes(&mut |c| { + for op in c.ops().iter() { + let idx = op.container; + let info = ops_nums.entry(idx).or_insert(0); + *info += op.atom_len(); + + let time = last_edit_time.entry(idx).or_insert(c.timestamp()); + if *time < c.timestamp() { + *time = c.timestamp(); + } + } + }); + } + + let mut containers = FxHashMap::default(); + let mut state = doc.app_state().lock(); + let alive_containers = state.get_all_alive_containers(); + for (idx, c) in state.iter_all_containers_mut() { + let ops_num = ops_nums.get(&idx).unwrap_or(&0); + let id = doc.arena().get_container_id(idx).unwrap(); + let dropped = !alive_containers.contains(&id); + containers.insert( + id, + ContainerAnalysisInfo { + depth: c.depth() as u32, + dropped, + size: c.encode().len() as u32, + ops_num: *ops_num as u32, + last_edit_time: *last_edit_time.get(&idx).unwrap_or(&0), + }, + ); + } + + Self { containers } + } + + #[allow(unused)] + pub fn len(&self) -> usize { + self.containers.len() + } + + #[allow(unused)] + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn dropped_len(&self) -> usize { + self.containers + .iter() + .filter(|(_, info)| info.dropped) + .count() + } + + pub fn tiny_container_len(&self) -> usize { + self.containers + .iter() + .filter(|(_, info)| info.size < 128) + .count() + } + + pub fn large_container_len(&self) -> usize { + self.containers + .iter() + .filter(|(_, info)| info.size >= 1024) + .count() + } +} diff --git a/vendor/loro-internal/src/state/container_store.rs b/vendor/loro-internal/src/state/container_store.rs new file mode 100644 index 00000000..445bc158 --- /dev/null +++ b/vendor/loro-internal/src/state/container_store.rs @@ -0,0 +1,448 @@ +use super::{ContainerCreationContext, State}; +use crate::arena::LoadAllFlag; +use crate::sync::{AtomicU64, Mutex}; +use crate::{ + arena::SharedArena, configure::Configure, container::idx::ContainerIdx, + utils::kv_wrapper::KvWrapper, version::Frontiers, +}; +use bytes::Bytes; +use inner_store::InnerStore; +use loro_common::{ContainerID, InternalString, LoroResult, LoroValue}; +use std::sync::Arc; + +pub(crate) use container_wrapper::ContainerWrapper; + +mod container_wrapper; +mod inner_store; + +/// Encoding Schema for Container Store +/// +/// KV-Store: +/// - Key: Encoded Container ID +/// - Value: Encoded Container State +/// +/// ─ ─ ─ ─ ─ ─ ─ For Each Container Type ─ ─ ─ ─ ─ ─ ─ ─ +/// +/// ┌────────────────┬──────────────────────────────────┐ +/// │ u16 Depth │ ParentID │ +/// └────────────────┴──────────────────────────────────┘ +/// ┌───────────────────────────────────────────────────┐ +/// │ ┌───────────────────────────────────────────────┐ │ +/// │ │ Into │ │ +/// │ └───────────────────────────────────────────────┘ │ +/// │ │ +/// │ Container Specific Encode │ +/// │ │ +/// │ │ +/// │ │ +/// │ │ +/// └───────────────────────────────────────────────────┘ +pub(crate) struct ContainerStore { + arena: SharedArena, + store: InnerStore, + shallow_root_store: Option>, + conf: Configure, + peer: Arc, +} + +pub(crate) const FRONTIERS_KEY: &[u8] = b"fr"; +impl std::fmt::Debug for ContainerStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerStore") + .field("store", &self.store) + .finish() + } +} + +#[derive(Debug)] +pub(crate) struct GcStore { + pub shallow_root_frontiers: Frontiers, + pub encoded_state_bytes: Bytes, + pub store: Mutex, +} + +macro_rules! ctx { + ($self:expr) => { + ContainerCreationContext { + configure: &$self.conf, + peer: $self.peer.load(std::sync::atomic::Ordering::Relaxed), + } + }; +} + +impl ContainerStore { + pub fn new(arena: SharedArena, conf: Configure, peer: Arc) -> Self { + ContainerStore { + store: InnerStore::new(arena.clone(), conf.clone()), + arena, + conf, + shallow_root_store: None, + peer, + } + } + + pub fn can_import_snapshot(&self) -> bool { + if self.shallow_root_store.is_some() { + return false; + } + + self.store.can_import_snapshot() + } + + pub fn get_container_mut(&mut self, idx: ContainerIdx) -> Option<&mut State> { + self.store + .get_mut(idx) + .map(|x| x.get_state_mut(idx, ctx!(self))) + } + + #[allow(unused)] + pub fn get_container(&mut self, idx: ContainerIdx) -> Option<&State> { + self.store + .get_mut(idx) + .map(|x| x.get_state(idx, ctx!(self))) + } + + pub fn shallow_root_store(&self) -> Option<&Arc> { + self.shallow_root_store.as_ref() + } + + pub fn get_value(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.get_value(idx, ctx!(self))) + } + + pub fn map_get(&mut self, idx: ContainerIdx, key: &str) -> Option { + self.store + .with_container_for_read(idx, |c| c.map_get(idx, ctx!(self), key))? + } + + pub fn map_len(&mut self, idx: ContainerIdx) -> usize { + self.store + .with_container_for_read(idx, |c| c.map_len(idx, ctx!(self))) + .unwrap_or(0) + } + + pub fn map_keys(&mut self, idx: ContainerIdx) -> Vec { + self.store + .with_container_for_read(idx, |c| c.map_keys(idx, ctx!(self))) + .unwrap_or_default() + } + + pub fn map_entries(&mut self, idx: ContainerIdx) -> Vec<(InternalString, LoroValue)> { + self.store + .with_container_for_read(idx, |c| c.map_entries(idx, ctx!(self))) + .unwrap_or_default() + } + + pub fn list_get(&mut self, idx: ContainerIdx, index: usize) -> Option { + self.store + .with_container_for_read(idx, |c| c.list_get(idx, ctx!(self), index))? + } + + pub fn list_len(&mut self, idx: ContainerIdx) -> usize { + self.store + .with_container_for_read(idx, |c| c.list_len(idx, ctx!(self))) + .unwrap_or(0) + } + + pub fn list_values(&mut self, idx: ContainerIdx) -> Vec { + self.store + .with_container_for_read(idx, |c| c.list_values(idx, ctx!(self))) + .unwrap_or_default() + } + + pub fn text_unicode_len(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.text_unicode_len(idx, ctx!(self)))? + } + + pub fn text_utf16_len(&mut self, idx: ContainerIdx) -> Option { + self.store + .with_container_for_read(idx, |c| c.text_utf16_len(idx, ctx!(self)))? + } + + pub fn has_decoded_state(&mut self, idx: ContainerIdx) -> bool { + self.store.has_decoded_state(idx) + } + + pub fn encode(&mut self) -> Bytes { + self.store.encode() + } + + pub(crate) fn flush(&mut self) { + self.store.flush() + } + + pub fn shallow_root_frontiers(&self) -> Option<&Frontiers> { + self.shallow_root_store + .as_ref() + .map(|x| &x.shallow_root_frontiers) + } + + pub(crate) fn encode_shallow_root_state(&self) -> Option { + let shallow_root = self.shallow_root_store.as_ref()?; + Some(shallow_root.encoded_state_bytes.clone()) + } + + pub(crate) fn shallow_root_state_for_export(&self) -> Option<(Bytes, KvWrapper)> { + let shallow_root = self.shallow_root_store.as_ref()?; + let shallow_root_kv = shallow_root.store.lock().get_kv_clone(); + Some((shallow_root.encoded_state_bytes.clone(), shallow_root_kv)) + } + + pub(crate) fn decode(&mut self, bytes: Bytes) -> LoroResult> { + self.store.decode(bytes) + } + + pub(crate) fn decode_gc( + &mut self, + shallow_bytes: Bytes, + start_frontiers: Frontiers, + config: Configure, + ) -> LoroResult> { + assert!(self.shallow_root_store.is_none()); + let mut inner = InnerStore::new(self.arena.clone(), config); + let encoded_state_bytes = shallow_bytes.clone(); + let f = inner.decode(shallow_bytes)?; + let encoded_state_bytes = if f.as_ref() == Some(&start_frontiers) { + encoded_state_bytes + } else { + let kv = inner.get_kv_clone(); + kv.insert(FRONTIERS_KEY, start_frontiers.encode().into()); + kv.export() + }; + self.shallow_root_store = Some(Arc::new(GcStore { + shallow_root_frontiers: start_frontiers, + encoded_state_bytes, + store: Mutex::new(inner), + })); + Ok(f) + } + + pub(crate) fn decode_state_by_two_bytes( + &mut self, + shallow_bytes: Bytes, + state_bytes: Bytes, + ) -> LoroResult<()> { + self.store + .decode_twice(shallow_bytes.clone(), state_bytes)?; + Ok(()) + } + + pub fn iter_and_decode_all(&mut self) -> impl Iterator { + self.store.iter_all_containers_mut().map(|(idx, v)| { + v.get_state_mut( + idx, + ContainerCreationContext { + configure: &self.conf, + peer: self.peer.load(std::sync::atomic::Ordering::Relaxed), + }, + ) + }) + } + + pub fn get_kv_clone(&self) -> KvWrapper { + self.store.get_kv_clone() + } + + pub fn contains_id(&mut self, id: &ContainerID) -> bool { + self.store.contains_id(id) + } + + pub fn iter_all_containers( + &mut self, + ) -> impl Iterator { + self.store.iter_all_containers_mut() + } + + pub fn iter_all_container_ids(&mut self) -> impl Iterator + '_ { + self.store.iter_all_container_ids() + } + + pub fn load_all(&mut self) -> LoadAllFlag { + self.store.load_all(); + LoadAllFlag + } + + pub fn load_root_containers(&mut self) -> LoadAllFlag { + self.store.load_roots(); + LoadAllFlag + } + + pub(super) fn get_or_create_mut(&mut self, idx: ContainerIdx) -> &mut State { + self.store + .get_or_insert_with(idx, || { + let state = super::create_state_( + idx, + &self.conf, + self.peer.load(std::sync::atomic::Ordering::Relaxed), + ); + ContainerWrapper::new(state, &self.arena) + }) + .get_state_mut(idx, ctx!(self)) + } + + pub(crate) fn ensure_container(&mut self, id: &loro_common::ContainerID) { + let idx = self.arena.register_container(id); + self.store.ensure_container(idx, || { + let state = super::create_state_( + idx, + &self.conf, + self.peer.load(std::sync::atomic::Ordering::Relaxed), + ); + ContainerWrapper::new(state, &self.arena) + }); + } + + pub(super) fn get_or_create_imm(&mut self, idx: ContainerIdx) -> &State { + self.store + .get_or_insert_with(idx, || { + let state = super::create_state_( + idx, + &self.conf, + self.peer.load(std::sync::atomic::Ordering::Relaxed), + ); + ContainerWrapper::new(state, &self.arena) + }) + .get_state(idx, ctx!(self)) + } + + pub(crate) fn fork( + &mut self, + arena: SharedArena, + peer: Arc, + config: Configure, + ) -> ContainerStore { + ContainerStore { + store: self.store.fork(arena.clone(), &config), + arena, + conf: config, + peer, + shallow_root_store: None, + } + } + + #[allow(unused)] + fn check_eq_after_parsing(&mut self, other: &mut ContainerStore) { + for (idx, container) in self.store.iter_all_containers_mut() { + let id = self.arena.get_container_id(idx).unwrap(); + let other_idx = other.arena.register_container(&id); + let other_container = other + .store + .get_mut(other_idx) + .expect("container not found on other store"); + let other_id = other.arena.get_container_id(other_idx).unwrap(); + assert_eq!( + id, other_id, + "container id mismatch {:?} {:?}", + id, other_id + ); + assert_eq!( + container.get_value(idx, ctx!(self)), + other_container.get_value(other_idx, ctx!(other)), + "value mismatch" + ); + + if container.encode() != other_container.encode() { + panic!( + "container mismatch Origin: {:#?}, New: {:#?}", + &container, &other_container + ); + } + + other_container + .decode_state(other_idx, ctx!(other)) + .unwrap(); + other_container.clear_bytes(); + if container.encode() != other_container.encode() { + panic!( + "container mismatch Origin: {:#?}, New: {:#?}", + &container, &other_container + ); + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{ + cursor::PosType, state::TreeParentId, ContainerType, ListHandler, LoroDoc, MapHandler, + MovableListHandler, + }; + + fn decode_container_store(bytes: Bytes) -> ContainerStore { + let mut new_store = ContainerStore::new( + SharedArena::new(), + Configure::default(), + Arc::new(AtomicU64::new(233)), + ); + + new_store.decode(bytes).unwrap(); + new_store + } + + fn init_doc() -> LoroDoc { + let doc = LoroDoc::new(); + doc.start_auto_commit(); + let text = doc.get_text("text"); + text.insert(0, "hello", PosType::Unicode).unwrap(); + let map = doc.get_map("map"); + map.insert("key", "value").unwrap(); + + let list = doc.get_list("list"); + list.push("item1").unwrap(); + + let tree = doc.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + tree.create_at(TreeParentId::Node(root), 0).unwrap(); + + let movable_list = doc.get_movable_list("movable_list"); + movable_list.insert(0, "movable_item").unwrap(); + + // Create child containers + let child_map = map + .insert_container("child_map", MapHandler::new_detached()) + .unwrap(); + child_map.insert("child_key", "child_value").unwrap(); + + let child_list = list + .insert_container(0, ListHandler::new_detached()) + .unwrap(); + child_list.push("child_item").unwrap(); + let child_movable_list = movable_list + .insert_container(0, MovableListHandler::new_detached()) + .unwrap(); + child_movable_list.insert(0, "child_movable_item").unwrap(); + doc + } + + fn export_container_store(doc: &LoroDoc) -> Bytes { + let mut state = doc.app_state().lock(); + state.ensure_all_alive_containers(); + state.store.encode() + } + + #[test] + fn test_container_store_exports_imports() { + let doc = init_doc(); + let bytes = export_container_store(&doc); + let mut new_store = decode_container_store(bytes); + let mut s = doc.app_state().lock(); + s.store.check_eq_after_parsing(&mut new_store); + } + + #[test] + fn first_lazy_read_caches_value() { + let doc = init_doc(); + let bytes = export_container_store(&doc); + let mut store = decode_container_store(bytes); + let map_id = ContainerID::new_root("map", ContainerType::Map); + let map_idx = store.arena.register_container(&map_id); + + assert!(!store.store.has_cached_value_for_test(map_idx)); + assert_eq!(store.map_len(map_idx), 2); + assert!(store.store.has_cached_value_for_test(map_idx)); + } +} diff --git a/vendor/loro-internal/src/state/container_store/container_wrapper.rs b/vendor/loro-internal/src/state/container_store/container_wrapper.rs new file mode 100644 index 00000000..6c2255fb --- /dev/null +++ b/vendor/loro-internal/src/state/container_store/container_wrapper.rs @@ -0,0 +1,684 @@ +use std::collections::BTreeMap; + +use bytes::Bytes; +use loro_common::{ContainerID, ContainerType, InternalString, LoroError, LoroResult, LoroValue}; +use once_cell::sync::OnceCell; +use tracing::trace; + +#[cfg(feature = "counter")] +use crate::state::counter_state::CounterState; +use crate::{ + arena::SharedArena, + container::idx::ContainerIdx, + state::{ + unknown_state::UnknownState, ContainerCreationContext, ContainerState, FastStateSnapshot, + IndexType, ListState, MapState, MovableListState, RichtextState, State, TreeState, + }, + utils::utf16::{count_unicode_chars, count_utf16_len}, +}; + +#[derive(Debug)] +pub(crate) struct ContainerWrapper { + depth: usize, + kind: ContainerType, + parent: Option, + data: ContainerData, + flushed: bool, +} + +#[derive(Debug)] +enum ContainerData { + State(State), + Lazy(Box), +} + +#[derive(Debug)] +struct LazyContainerData { + /// Lazily decoded snapshot bytes and optional decoded value. + bytes: Option, + value: Option, + bytes_offset_for_value: Option, + bytes_offset_for_state: Option, +} + +#[derive(Debug)] +enum LazyDecodedValue { + Value(LoroValue), + Text { + value: LoroValue, + unicode_len: usize, + utf16_len: usize, + }, + Map { + ordered: BTreeMap, + value: OnceCell, + }, +} + +impl LazyDecodedValue { + fn text(value: LoroValue) -> Self { + let text = value + .as_string() + .expect("decoded text value should be a string"); + Self::Text { + unicode_len: count_unicode_chars(text.as_bytes()), + utf16_len: count_utf16_len(text.as_bytes()), + value, + } + } + + fn to_loro_value(&self) -> LoroValue { + match self { + Self::Value(value) | Self::Text { value, .. } => value.clone(), + Self::Map { ordered, value } => value + .get_or_init(|| { + LoroValue::Map( + ordered + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect::(), + ) + }) + .clone(), + } + } + + fn as_map(&self) -> Option<&BTreeMap> { + match self { + Self::Map { ordered, .. } => Some(ordered), + _ => None, + } + } + + fn unicode_len(&self) -> Option { + match self { + Self::Text { unicode_len, .. } => Some(*unicode_len), + _ => None, + } + } + + fn utf16_len(&self) -> Option { + match self { + Self::Text { utf16_len, .. } => Some(*utf16_len), + _ => None, + } + } +} + +impl ContainerWrapper { + pub fn new(state: State, arena: &SharedArena) -> Self { + let idx = state.container_idx(); + let parent = arena + .get_parent(idx) + .and_then(|p| arena.get_container_id(p)); + let depth = arena.get_depth(idx).unwrap().get() as usize; + Self { + depth, + parent, + kind: idx.get_type(), + data: ContainerData::State(state), + flushed: false, + } + } + + pub fn depth(&self) -> usize { + self.depth + } + + pub(crate) fn kind(&self) -> ContainerType { + self.kind + } + + /// It will not decode the state if it is not decoded + #[allow(unused)] + pub fn try_get_state(&self) -> Option<&State> { + match &self.data { + ContainerData::State(state) => Some(state), + ContainerData::Lazy(_) => None, + } + } + + /// It will decode the state if it is not decoded + pub fn get_state(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> &State { + self.decode_state(idx, ctx).unwrap(); + match &self.data { + ContainerData::State(state) => state, + ContainerData::Lazy(_) => unreachable!("ContainerWrapper state should be decoded"), + } + } + + /// It will decode the state if it is not decoded + pub fn get_state_mut( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> &mut State { + self.decode_state(idx, ctx).unwrap(); + self.flushed = false; + match &mut self.data { + ContainerData::State(state) => state, + ContainerData::Lazy(_) => unreachable!("ContainerWrapper state should be decoded"), + } + } + + pub(crate) fn try_get_value( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> LoroResult { + match &mut self.data { + ContainerData::State(state) => { + trace!("state"); + Ok(state.get_value()) + } + ContainerData::Lazy(lazy) if lazy.value.is_some() => { + trace!("value"); + Ok(lazy.value.as_ref().unwrap().to_loro_value()) + } + ContainerData::Lazy(_) => { + trace!("decode value"); + self.decode_value(idx, ctx)?; + match &mut self.data { + ContainerData::State(state) => Ok(state.get_value()), + ContainerData::Lazy(lazy) => Ok(lazy.value.as_ref().unwrap().to_loro_value()), + } + } + } + } + + pub fn get_value(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> LoroValue { + self.try_get_value(idx, ctx).unwrap() + } + + pub fn map_get( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + key: &str, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => state.as_map_state().unwrap().get(key).cloned(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.as_map()?.get(key).cloned(), + ContainerData::State(state) => state.as_map_state().unwrap().get(key).cloned(), + } + } + } + } + + pub fn map_len(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> usize { + match &mut self.data { + ContainerData::State(state) => state.as_map_state().unwrap().len(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .and_then(|value| value.as_map()) + .map_or(0, BTreeMap::len), + ContainerData::State(state) => state.as_map_state().unwrap().len(), + } + } + } + } + + pub fn map_keys( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Vec { + match &mut self.data { + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter(|(_, value)| value.value.is_some()) + .map(|(key, _)| key.clone()) + .collect(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .and_then(|value| value.as_map()) + .map(|map| map.keys().map(|key| key.as_str().into()).collect()) + .unwrap_or_default(), + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter(|(_, value)| value.value.is_some()) + .map(|(key, _)| key.clone()) + .collect(), + } + } + } + } + + pub fn map_entries( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Vec<(InternalString, LoroValue)> { + match &mut self.data { + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter_map(|(key, value)| value.value.clone().map(|value| (key.clone(), value))) + .collect(), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &self.data { + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .and_then(|value| value.as_map()) + .map(|map| { + map.iter() + .map(|(key, value)| (key.as_str().into(), value.clone())) + .collect() + }) + .unwrap_or_default(), + ContainerData::State(state) => state + .as_map_state() + .unwrap() + .iter() + .filter_map(|(key, value)| { + value.value.clone().map(|value| (key.clone(), value)) + }) + .collect(), + } + } + } + } + + pub fn text_unicode_len( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_unicode()) + } + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_unicode()) + } + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.unicode_len(), + } + } + } + } + + pub fn text_utf16_len( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => Some(state.as_richtext_state_mut().unwrap().len_utf16()), + ContainerData::Lazy(_) => { + self.decode_value(idx, ctx).unwrap(); + match &mut self.data { + ContainerData::State(state) => { + Some(state.as_richtext_state_mut().unwrap().len_utf16()) + } + ContainerData::Lazy(lazy) => lazy.value.as_ref()?.utf16_len(), + } + } + } + } + + pub fn list_get( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + index: usize, + ) -> Option { + match &mut self.data { + ContainerData::State(state) => match self.kind { + ContainerType::List => state.as_list_state().unwrap().get(index).cloned(), + ContainerType::MovableList => state + .as_movable_list_state() + .unwrap() + .get(index, IndexType::ForUser) + .cloned(), + _ => None, + }, + ContainerData::Lazy(_) => match self.get_value(idx, ctx) { + LoroValue::List(list) => list.get(index).cloned(), + _ => None, + }, + } + } + + pub fn list_len(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> usize { + match &mut self.data { + ContainerData::State(state) => match self.kind { + ContainerType::List => state.as_list_state().unwrap().len(), + ContainerType::MovableList => state.as_movable_list_state().unwrap().len(), + _ => 0, + }, + ContainerData::Lazy(_) => match self.get_value(idx, ctx) { + LoroValue::List(list) => list.len(), + _ => 0, + }, + } + } + + pub fn list_values( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> Vec { + match &mut self.data { + ContainerData::State(state) => match self.kind { + ContainerType::List => state.as_list_state().unwrap().iter().cloned().collect(), + ContainerType::MovableList => state + .as_movable_list_state() + .unwrap() + .iter() + .cloned() + .collect(), + _ => Vec::new(), + }, + ContainerData::Lazy(_) => match self.get_value(idx, ctx) { + LoroValue::List(list) => list.iter().cloned().collect(), + _ => Vec::new(), + }, + } + } + + pub fn encode(&mut self) -> Bytes { + let ContainerData::State(state) = &mut self.data else { + let lazy = match &self.data { + ContainerData::Lazy(lazy) => lazy, + ContainerData::State(_) => unreachable!(), + }; + assert!(self.flushed, "lazy container should be flushed"); + return lazy.bytes.as_ref().unwrap().clone(); + }; + + // ContainerType + // Depth + // ParentID + // StateSnapshot + let mut output = Vec::new(); + output.push(self.kind.to_u8()); + leb128::write::unsigned(&mut output, self.depth as u64).unwrap(); + postcard::to_io(&self.parent, &mut output).unwrap(); + state.encode_snapshot_fast(&mut output); + output.into() + } + + #[allow(unused)] + pub fn decode_parent(b: &[u8]) -> Option { + let mut bytes = &b[1..]; + let _depth = leb128::read::unsigned(&mut bytes).unwrap(); + let (parent, _bytes) = postcard::take_from_bytes::>(bytes).unwrap(); + parent + } + + pub fn new_from_bytes(bytes: Bytes) -> Self { + Self::try_new_from_bytes(bytes).unwrap() + } + + pub fn try_new_from_bytes(bytes: Bytes) -> LoroResult { + if bytes.is_empty() { + return Err(LoroError::DecodeError( + "Decode container state failed".to_string().into_boxed_str(), + )); + } + + let kind = ContainerType::try_from_u8(bytes[0])?; + let mut reader = &bytes[1..]; + let depth = leb128::read::unsigned(&mut reader).map_err(|_| { + LoroError::DecodeError("Decode container state failed".to_string().into_boxed_str()) + })?; + let (parent, reader) = postcard::take_from_bytes(reader).map_err(|_| { + LoroError::DecodeError("Decode container state failed".to_string().into_boxed_str()) + })?; + let size = bytes.len() - reader.len(); + Ok(Self { + depth: depth as usize, + kind, + parent, + data: ContainerData::Lazy(Box::new(LazyContainerData { + value: None, + bytes: Some(bytes.clone()), + bytes_offset_for_value: Some(size), + bytes_offset_for_state: None, + })), + flushed: true, + }) + } + + fn decode_value(&mut self, idx: ContainerIdx, ctx: ContainerCreationContext) -> LoroResult<()> { + if matches!(self.data, ContainerData::State(_)) { + return Ok(()); + } + + if matches!(&self.data, ContainerData::Lazy(lazy) if lazy.value.is_some()) { + return Ok(()); + } + + let (v, state_offset, decoded_state) = self.decode_value_from_bytes(idx, ctx)?; + if let Some(state) = decoded_state { + self.data = ContainerData::State(state); + return Ok(()); + } + + let ContainerData::Lazy(lazy) = &mut self.data else { + unreachable!(); + }; + lazy.value = Some(v); + lazy.bytes_offset_for_state = Some(state_offset); + Ok(()) + } + + pub(super) fn has_cached_value(&self) -> bool { + match &self.data { + ContainerData::State(_) => true, + ContainerData::Lazy(lazy) => lazy.value.is_some(), + } + } + + #[cfg(test)] + pub(super) fn has_cached_value_for_test(&self) -> bool { + self.has_cached_value() + } + + fn value_bytes_and_offset(&mut self) -> Option<(Bytes, usize)> { + let ContainerData::Lazy(lazy) = &mut self.data else { + return None; + }; + + let bytes = lazy.bytes.as_ref()?.clone(); + if lazy.bytes_offset_for_value.is_none() { + let mut reader: &[u8] = &bytes; + reader = &reader[1..]; + let _depth = leb128::read::unsigned(&mut reader).unwrap(); + let (_parent, reader) = + postcard::take_from_bytes::>(reader).unwrap(); + // SAFETY: bytes is a slice of b + let size = bytes.len() - reader.len(); + lazy.bytes_offset_for_value = Some(size); + } + + Some((bytes, lazy.bytes_offset_for_value.unwrap())) + } + + fn decode_value_from_bytes( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> LoroResult<(LazyDecodedValue, usize, Option)> { + let Some((bytes, value_offset)) = self.value_bytes_and_offset() else { + return Ok((LazyDecodedValue::Value(self.kind.default_value()), 0, None)); + }; + let b = &bytes[value_offset..]; + + let mut decoded_state = None; + let (v, state_offset) = match self.kind { + ContainerType::Text => { + let (v, rest) = RichtextState::decode_value(b)?; + ( + LazyDecodedValue::text(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::Map => { + let (v, rest) = MapState::decode_value_as_btree_map(b)?; + ( + LazyDecodedValue::Map { + ordered: v, + value: OnceCell::new(), + }, + b.len() - rest.len() + value_offset, + ) + } + ContainerType::List => { + let (v, rest) = ListState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::MovableList => { + let (v, rest) = MovableListState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::Tree => { + let mut state = TreeState::decode_snapshot_fast(idx, (LoroValue::Null, b), ctx)?; + let value = state.get_value(); + decoded_state = Some(State::TreeState(Box::new(state))); + (LazyDecodedValue::Value(value), value_offset) + } + #[cfg(feature = "counter")] + ContainerType::Counter => { + let (v, rest) = CounterState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + ContainerType::Unknown(_) => { + let (v, rest) = UnknownState::decode_value(b)?; + ( + LazyDecodedValue::Value(v), + b.len() - rest.len() + value_offset, + ) + } + }; + + Ok((v, state_offset, decoded_state)) + } + + pub(super) fn decode_state( + &mut self, + idx: ContainerIdx, + ctx: ContainerCreationContext, + ) -> LoroResult<()> { + if matches!(self.data, ContainerData::State(_)) { + return Ok(()); + } + + let need_value = match &self.data { + ContainerData::Lazy(lazy) => lazy.value.is_none(), + ContainerData::State(_) => false, + }; + if need_value { + self.decode_value(idx, ctx)?; + } + + if matches!(self.data, ContainerData::State(_)) { + return Ok(()); + } + + let ContainerData::Lazy(lazy) = &self.data else { + unreachable!(); + }; + let bytes = lazy.bytes.as_ref().unwrap(); + let offset = lazy.bytes_offset_for_state.unwrap(); + let b = &bytes[offset..]; + let v = lazy.value.as_ref().unwrap().to_loro_value(); + let state: State = match self.kind { + ContainerType::Text => RichtextState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::Map => MapState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::List => ListState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::MovableList => { + MovableListState::decode_snapshot_fast(idx, (v, b), ctx)?.into() + } + ContainerType::Tree => TreeState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + #[cfg(feature = "counter")] + ContainerType::Counter => CounterState::decode_snapshot_fast(idx, (v, b), ctx)?.into(), + ContainerType::Unknown(_) => { + UnknownState::decode_snapshot_fast(idx, (v, b), ctx)?.into() + } + }; + self.data = ContainerData::State(state); + Ok(()) + } + + #[allow(unused)] + pub(crate) fn is_state_empty(&self) -> bool { + match &self.data { + ContainerData::State(state) => state.is_state_empty(), + ContainerData::Lazy(lazy) => { + // FIXME: it's not very accurate... + lazy.bytes.as_ref().unwrap().len() > 10 + } + } + } + + pub(crate) fn is_deleted_root_value_cleared(&mut self) -> bool { + fn value_is_cleared(kind: ContainerType, value: &LoroValue) -> bool { + match kind { + ContainerType::Text => value.as_string().is_some_and(|value| value.is_empty()), + ContainerType::Map | ContainerType::List | ContainerType::MovableList => { + value.is_empty_collection() + } + ContainerType::Tree => value.as_list().is_some_and(|value| value.is_empty()), + #[cfg(feature = "counter")] + ContainerType::Counter => value.as_double().is_some_and(|value| *value == 0.0), + ContainerType::Unknown(_) => false, + } + } + + match &mut self.data { + ContainerData::State(state) => value_is_cleared(self.kind, &state.get_value()), + ContainerData::Lazy(lazy) => lazy + .value + .as_ref() + .is_some_and(|value| value_is_cleared(self.kind, &value.to_loro_value())), + } + } + + pub(crate) fn clear_bytes(&mut self) { + assert!(matches!(self.data, ContainerData::State(_))); + } + + pub(crate) fn is_flushed(&self) -> bool { + self.flushed + } + + pub(crate) fn set_flushed(&mut self, flushed: bool) { + self.flushed = flushed; + } + + #[allow(unused)] + pub(crate) fn parent(&self) -> Option<&ContainerID> { + self.parent.as_ref() + } +} diff --git a/vendor/loro-internal/src/state/container_store/inner_store.rs b/vendor/loro-internal/src/state/container_store/inner_store.rs new file mode 100644 index 00000000..895db89e --- /dev/null +++ b/vendor/loro-internal/src/state/container_store/inner_store.rs @@ -0,0 +1,436 @@ +use crate::{ + arena::SharedArena, configure::Configure, container::idx::ContainerIdx, + state::container_store::FRONTIERS_KEY, utils::kv_wrapper::KvWrapper, version::Frontiers, +}; +use bytes::Bytes; +use loro_common::ContainerID; +use std::ops::Bound; + +use super::ContainerWrapper; + +/// The invariants about this struct: +/// +/// - `kv` is either the same or older than `store`. +/// - if `load_state` is `AllLoaded`, then `store` contains all the entries from `kv` +/// +/// Invariants: it should be agnostic to the users of this struct whether a container is stored in `kv` or `store` +pub(crate) struct InnerStore { + arena: SharedArena, + store: Vec>, + kv: KvWrapper, + load_state: LoadState, + config: Configure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoadState { + Lazy, + RootsLoaded, + AllLoaded, +} + +impl std::fmt::Debug for InnerStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("InnerStore").finish() + } +} + +/// This impl block contains all the mutation code that may break the invariants of this struct +impl InnerStore { + #[inline] + fn slot(idx: ContainerIdx) -> usize { + idx.to_index() as usize + } + + #[inline] + fn get_entry_mut_in( + store: &mut [Option], + idx: ContainerIdx, + ) -> Option<&mut ContainerWrapper> { + let entry = store.get_mut(Self::slot(idx))?.as_mut()?; + debug_assert_eq!(entry.kind(), idx.get_type()); + Some(entry) + } + + #[inline] + fn get_entry_mut(&mut self, idx: ContainerIdx) -> Option<&mut ContainerWrapper> { + Self::get_entry_mut_in(&mut self.store, idx) + } + + #[inline] + fn contains_idx_in(store: &[Option], idx: ContainerIdx) -> bool { + store + .get(Self::slot(idx)) + .and_then(|entry| entry.as_ref()) + .is_some_and(|entry| entry.kind() == idx.get_type()) + } + + #[inline] + fn contains_idx(&self, idx: ContainerIdx) -> bool { + Self::contains_idx_in(&self.store, idx) + } + + fn insert_entry( + store: &mut Vec>, + idx: ContainerIdx, + container: ContainerWrapper, + ) -> Option { + let slot = Self::slot(idx); + if store.len() <= slot { + store.resize_with(slot + 1, || None); + } + + store[slot].replace(container) + } + + pub(super) fn get_or_insert_with( + &mut self, + idx: ContainerIdx, + f: impl FnOnce() -> ContainerWrapper, + ) -> &mut ContainerWrapper { + if self.get_entry_mut(idx).is_none() { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + let container = if self.load_state != LoadState::AllLoaded { + self.kv + .get(&key) + .map(ContainerWrapper::new_from_bytes) + .unwrap_or_else(f) + } else { + f() + }; + Self::insert_entry(&mut self.store, idx, container); + } + + self.get_entry_mut(idx).unwrap() + } + + pub(super) fn ensure_container( + &mut self, + idx: ContainerIdx, + f: impl FnOnce() -> ContainerWrapper, + ) { + if self.contains_idx(idx) { + return; + } + + if self.load_state != LoadState::AllLoaded { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + if let Some(v) = self.kv.get(&key) { + let c = ContainerWrapper::new_from_bytes(v); + Self::insert_entry(&mut self.store, idx, c); + return; + } + } + + let c = f(); + Self::insert_entry(&mut self.store, idx, c); + } + + pub(crate) fn get_mut(&mut self, idx: ContainerIdx) -> Option<&mut ContainerWrapper> { + if self.get_entry_mut(idx).is_none() && self.load_state != LoadState::AllLoaded { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + if let Some(v) = self.kv.get(&key) { + let c = ContainerWrapper::new_from_bytes(v); + Self::insert_entry(&mut self.store, idx, c); + } + } + + self.get_entry_mut(idx) + } + + pub(crate) fn with_container_for_read( + &mut self, + idx: ContainerIdx, + f: impl FnOnce(&mut ContainerWrapper) -> R, + ) -> Option { + if let Some(entry) = self.get_entry_mut(idx) { + return Some(f(entry)); + } + + if self.load_state != LoadState::AllLoaded { + let id = self.arena.get_container_id(idx).unwrap(); + let key = id.to_bytes(); + if let Some(v) = self.kv.get(&key) { + let mut container = ContainerWrapper::new_from_bytes(v); + let ans = f(&mut container); + if container.has_cached_value() { + Self::insert_entry(&mut self.store, idx, container); + } + return Some(ans); + } + } + + None + } + + pub(crate) fn has_decoded_state(&mut self, idx: ContainerIdx) -> bool { + self.get_entry_mut(idx) + .is_some_and(|entry| entry.try_get_state().is_some()) + } + + pub(crate) fn contains_id(&mut self, id: &ContainerID) -> bool { + if let Some(idx) = self.arena.id_to_idx(id) { + if self.contains_idx(idx) { + return true; + } + } + + if self.load_state != LoadState::AllLoaded { + let key = id.to_bytes(); + return self.kv.contains_key(&key); + } + + false + } + + pub(crate) fn iter_all_containers_mut( + &mut self, + ) -> impl Iterator { + self.load_all(); + self.store + .iter_mut() + .enumerate() + .filter_map(|(slot, entry)| { + entry.as_mut().map(|container| { + ( + ContainerIdx::from_index_and_type(slot as u32, container.kind()), + container, + ) + }) + }) + } + + pub(crate) fn iter_all_container_ids(&mut self) -> impl Iterator + '_ { + // PERF: we don't need to load all the containers here + self.load_all(); + self.store.iter().enumerate().filter_map(|(slot, entry)| { + entry.as_ref().map(|container| { + let idx = ContainerIdx::from_index_and_type(slot as u32, container.kind()); + self.arena.get_container_id(idx).unwrap() + }) + }) + } + + pub(crate) fn encode(&mut self) -> Bytes { + self.flush(); + self.kv.export() + } + + pub(crate) fn flush(&mut self) { + let deleted = self.config.deleted_root_containers.lock(); + let mut updates = Vec::new(); + let mut deleted_roots = Vec::new(); + + for (slot, entry) in self.store.iter_mut().enumerate() { + let Some(c) = entry.as_mut() else { + continue; + }; + let idx = ContainerIdx::from_index_and_type(slot as u32, c.kind()); + let cid = self.arena.get_container_id(idx).unwrap(); + if cid.is_root() && deleted.contains(&cid) && c.is_deleted_root_value_cleared() { + deleted_roots.push(cid.to_bytes()); + c.set_flushed(true); + continue; + } + + if c.is_flushed() { + continue; + } + + let cid: Bytes = cid.to_bytes().into(); + let value = c.encode(); + c.set_flushed(true); + updates.push((cid, value)); + } + + drop(deleted); + for cid in deleted_roots { + self.kv.remove(&cid); + } + self.kv.set_all(updates.into_iter()); + } + + pub(crate) fn get_kv_clone(&self) -> KvWrapper { + self.kv.clone() + } + + pub(crate) fn decode( + &mut self, + bytes: bytes::Bytes, + ) -> Result, loro_common::LoroError> { + assert!(self.kv.is_empty()); + let mut fr = None; + self.kv + .import(bytes) + .map_err(|e| loro_common::LoroError::DecodeError(e.into_boxed_str()))?; + if let Some(f) = self.kv.remove(FRONTIERS_KEY) { + fr = Some(Frontiers::decode(&f)?); + } + + let kv = self.kv.arc_clone(); + self.arena + .set_parent_resolver(Some(move |child_id: ContainerID| { + let k = child_id.to_bytes(); + let v = kv.get(&k)?; + let c = ContainerWrapper::new_from_bytes(v); + c.parent().cloned() + })); + + self.store.clear(); + self.load_state = LoadState::Lazy; + Ok(fr) + } + + pub(crate) fn decode_twice( + &mut self, + bytes_a: bytes::Bytes, + bytes_b: bytes::Bytes, + ) -> Result<(), loro_common::LoroError> { + assert!(self.kv.is_empty()); + // TODO: add assert that all containers in the store should be empty right now + self.kv + .import(bytes_a) + .map_err(|e| loro_common::LoroError::DecodeError(e.into_boxed_str()))?; + self.kv + .import(bytes_b) + .map_err(|e| loro_common::LoroError::DecodeError(e.into_boxed_str()))?; + self.kv.remove(FRONTIERS_KEY); + + // FLOWSTATE vendor patch #27 (§A14.4): the shallow-snapshot two-byte + // decode used to EAGERLY register every container (header parse + + // arena registration + store insert for all N containers) — ~48ms on + // the impact doc, on the open critical path now that shallow-open is + // the default. The single-blob `decode()` path instead goes LAZY: a + // parent-resolver closure reads containers from the merged kv on + // demand. The two byte layers are already merged into ONE kv above + // (bytes_b overwrites bytes_a per key, same as the eager scan saw), + // so the identical lazy resolver is correct here. `load_all()` + // materializes on first whole-doc need exactly as it does after + // `decode()`. `FLOWSTATE_EAGER_STATE_DECODE=1` restores the eager + // scan. + static EAGER: std::sync::OnceLock = std::sync::OnceLock::new(); + let eager = *EAGER + .get_or_init(|| std::env::var_os("FLOWSTATE_EAGER_STATE_DECODE").is_some()); + if eager { + let store = &mut self.store; + let arena = &self.arena; + self.kv.with_kv(|kv| { + arena.with_guards(|guards| { + let iter = kv.scan(Bound::Unbounded, Bound::Unbounded); + for (k, v) in iter { + let cid = ContainerID::from_bytes(&k); + let c = ContainerWrapper::new_from_bytes(v); + let parent = c.parent(); + let idx = guards.register_container(&cid); + let p = parent.as_ref().map(|p| guards.register_container(p)); + guards.set_parent(idx, p); + if Self::insert_entry(store, idx, c).is_some() {} + } + }); + }); + self.load_state = LoadState::AllLoaded; + return Ok(()); + } + + let kv = self.kv.arc_clone(); + self.arena + .set_parent_resolver(Some(move |child_id: ContainerID| { + let k = child_id.to_bytes(); + let v = kv.get(&k)?; + let c = ContainerWrapper::new_from_bytes(v); + c.parent().cloned() + })); + + self.store.clear(); + self.load_state = LoadState::Lazy; + Ok(()) + } + + pub fn load_all(&mut self) { + if self.load_state == LoadState::AllLoaded { + return; + } + + let store = &mut self.store; + let arena = &self.arena; + self.kv.with_kv(|kv| { + let iter = kv.scan(Bound::Unbounded, Bound::Unbounded); + arena.with_guards(|guards| { + for (k, v) in iter { + let cid = ContainerID::from_bytes(&k); + let idx = guards.register_container(&cid); + if Self::contains_idx_in(store, idx) { + // the container is already loaded + // the content in `store` is guaranteed to be newer than the content in `kv` + continue; + } + + let container = ContainerWrapper::new_from_bytes(v); + Self::insert_entry(store, idx, container); + } + }); + }); + + self.load_state = LoadState::AllLoaded; + } + + pub fn load_roots(&mut self) { + if self.load_state != LoadState::Lazy { + return; + } + + let arena = &self.arena; + self.kv.with_kv(|kv| { + let iter = kv.scan(Bound::Unbounded, Bound::Unbounded); + arena.with_guards(|guards| { + for (k, _) in iter { + let cid = ContainerID::from_bytes(&k); + if cid.is_root() { + guards.register_container(&cid); + } + } + }); + }); + self.load_state = LoadState::RootsLoaded; + } + + pub(crate) fn can_import_snapshot(&self) -> bool { + if !self.kv.is_empty() { + return false; + } + + self.store + .iter() + .filter_map(|entry| entry.as_ref()) + .all(|c| c.is_state_empty()) + } + + #[cfg(test)] + pub(super) fn has_cached_value_for_test(&mut self, idx: ContainerIdx) -> bool { + self.get_entry_mut(idx) + .is_some_and(|entry| entry.has_cached_value_for_test()) + } +} + +impl InnerStore { + pub(crate) fn new(arena: SharedArena, config: Configure) -> Self { + Self { + arena, + store: Vec::new(), + kv: KvWrapper::new_mem(), + load_state: LoadState::AllLoaded, + config, + } + } + + pub(crate) fn fork(&mut self, arena: SharedArena, config: &Configure) -> InnerStore { + // PERF: we can try to reuse + let bytes = self.encode(); + let mut new_store = Self::new(arena, config.clone()); + new_store.decode(bytes).unwrap(); + new_store + } +} diff --git a/vendor/loro-internal/src/state/counter_state.rs b/vendor/loro-internal/src/state/counter_state.rs new file mode 100644 index 00000000..332b2f52 --- /dev/null +++ b/vendor/loro-internal/src/state/counter_state.rs @@ -0,0 +1,134 @@ +use std::sync::Weak; + +use loro_common::{ContainerID, LoroResult, LoroValue}; + +use crate::{ + configure::Configure, + container::idx::ContainerIdx, + event::{Diff, Index, InternalDiff}, + op::{Op, RawOp, RawOpContent}, + LoroDocInner, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug, Clone)] +pub struct CounterState { + idx: ContainerIdx, + value: f64, +} + +impl CounterState { + pub(crate) fn new(idx: ContainerIdx) -> Self { + Self { idx, value: 0. } + } +} + +impl ContainerState for CounterState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + false + } + + fn apply_diff_and_convert(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> Diff { + if let InternalDiff::Counter(diff) = diff { + self.value += diff; + Diff::Counter(diff) + } else { + unreachable!() + } + } + + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + let _ = self.apply_diff_and_convert(diff, ctx); + Ok(()) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, _op: &Op) -> LoroResult { + if let RawOpContent::Counter(diff) = raw_op.content { + self.value += diff; + Ok(Default::default()) + } else { + unreachable!() + } + } + + #[doc = " Convert a state to a diff, such that an empty state will be transformed into the same as this state when it\'s applied."] + fn to_diff(&mut self, _doc: &Weak) -> Diff { + Diff::Counter(self.value) + } + + fn get_value(&mut self) -> LoroValue { + LoroValue::Double(self.value) + } + + #[doc = " Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + None + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + vec![] + } + + fn contains_child(&self, _id: &ContainerID) -> bool { + false + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +mod snapshot { + use crate::state::FastStateSnapshot; + + use super::*; + + impl FastStateSnapshot for CounterState { + fn encode_snapshot_fast(&mut self, mut w: W) { + let bytes = self.value.to_le_bytes(); + w.write_all(&bytes).unwrap(); + } + + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> { + // Builds without the counter feature can re-export an untouched counter as header only. + if bytes.is_empty() { + return Ok((LoroValue::Double(0.0), bytes)); + } + + if bytes.len() != 8 { + return Err(loro_common::LoroError::DecodeError( + "Decode counter value failed".to_string().into_boxed_str(), + )); + } + + let value = f64::from_le_bytes(bytes[..8].try_into().unwrap()); + Ok((LoroValue::Double(value), &bytes[8..])) + } + + fn decode_snapshot_fast( + idx: ContainerIdx, + v: (LoroValue, &[u8]), + _ctx: crate::state::ContainerCreationContext, + ) -> LoroResult + where + Self: Sized, + { + if !v.1.is_empty() { + return Err(loro_common::LoroError::DecodeError( + "Decode counter state failed".to_string().into_boxed_str(), + )); + } + + let mut counter = CounterState::new(idx); + counter.value = *v.0.as_double().unwrap(); + Ok(counter) + } + } +} diff --git a/vendor/loro-internal/src/state/dead_containers_cache.rs b/vendor/loro-internal/src/state/dead_containers_cache.rs new file mode 100644 index 00000000..9eb25e6a --- /dev/null +++ b/vendor/loro-internal/src/state/dead_containers_cache.rs @@ -0,0 +1,242 @@ +use super::DocState; +use crate::container::idx::ContainerIdx; +use rustc_hash::FxHashMap; + +#[derive(Default, Debug, Clone)] +pub(super) struct DeadContainersCache { + cache: FxHashMap, +} + +impl DeadContainersCache { + pub fn clear(&mut self) { + self.cache.clear(); + } + + pub(crate) fn clear_alive(&mut self) { + self.cache.retain(|_, is_deleted| *is_deleted); + } +} + +impl DocState { + pub(crate) fn is_deleted(&mut self, idx: ContainerIdx) -> bool { + #[cfg(not(debug_assertions))] + { + // Cache stores only deleted containers. + if self.dead_containers_cache.cache.contains_key(&idx) { + return true; + } + } + + let mut visited = vec![idx]; + let mut idx = idx; + let mut depends_on_mergeable_edge = false; + let is_deleted = loop { + let id = self.arena.idx_to_id(idx).unwrap(); + if id.is_mergeable() { + depends_on_mergeable_edge = true; + } + if let Some(parent_idx) = self.arena.get_parent(idx) { + if !self.contains_logical_child(parent_idx, &id) { + break true; + } + + idx = parent_idx; + visited.push(idx); + } else { + // No parent in the arena: top-level Roots are always alive; anything else + // (including a mergeable Root whose parent edge was never wired) is treated + // as deleted. + break !id.is_root() || id.is_mergeable(); + } + }; + + #[cfg(debug_assertions)] + { + if !depends_on_mergeable_edge { + if let Some(cached_is_deleted) = self.dead_containers_cache.cache.get(&idx) { + assert_eq!(is_deleted, *cached_is_deleted); + } + } + } + + // A mergeable ancestor can be deleted and later reactivated by changing the parent map's + // marker. Do not cache deletion for any descendant whose liveness depends on that + // logical edge, including ordinary children nested inside a mergeable map. + if depends_on_mergeable_edge { + if !is_deleted { + for idx in visited { + self.dead_containers_cache.cache.remove(&idx); + } + } + return is_deleted; + } + + if is_deleted { + for idx in visited { + self.dead_containers_cache.cache.insert(idx, true); + } + } else { + for idx in visited { + self.dead_containers_cache.cache.remove(&idx); + } + } + + is_deleted + } + + #[cfg(test)] + pub(crate) fn dead_cache_entry(&self, idx: ContainerIdx) -> Option { + self.dead_containers_cache.cache.get(&idx).copied() + } +} + +#[cfg(test)] +#[cfg(feature = "counter")] +mod tests { + use loro_common::ContainerID; + + use crate::{cursor::PosType, HandlerTrait, LoroDoc, TextHandler}; + + /// A mergeable child can be deleted and then reactivated: `delete(key)` clears its + /// marker (child unreachable), and a later `ensure_mergeable_(key)` writes the + /// marker back (child reachable again). While the child is unreachable, querying its + /// liveness must not cache a `deleted` entry because that answer depends on the mutable + /// mergeable marker edge. + /// + /// The scenario: + /// 1. Create the mergeable counter and capture its container idx. + /// 2. Delete the key, then query `is_deleted()`. + /// 3. Re-get the counter to rewrite the marker and reactivate the child. + /// 4. Assert the cache never held a `deleted` entry for that idx. + /// + /// It asserts the cache contents directly because `is_deleted()` only trusts the cache via a + /// release-only early return; inspecting the cache makes stale-cache regressions fail in both + /// debug and release builds. + #[test] + fn reactivated_mergeable_child_has_no_stale_dead_cache_entry() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + + let cid: ContainerID = counter.id(); + let idx = doc.state.lock().arena.id_to_idx(&cid).unwrap(); + + root.delete("revision").unwrap(); + doc.commit_then_renew(); + assert!(counter.is_deleted()); + assert_eq!( + doc.state.lock().dead_cache_entry(idx), + None, + "mergeable-dependent deletion must not be cached" + ); + + root.ensure_mergeable_counter("revision").unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.state.lock().dead_cache_entry(idx), + None, + "reactivation must drop the stale deleted-cache entry" + ); + } + + /// The no-cache rule also applies to ordinary descendants under a mergeable ancestor. A + /// regular child container can look cache-safe by cid shape, but its liveness still depends on + /// the ancestor's marker edge. + #[test] + fn ordinary_child_under_reactivated_mergeable_map_has_no_stale_dead_cache_entry() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let root = doc.get_map("state"); + let profile = root.ensure_mergeable_map("profile").unwrap(); + let text = profile + .insert_container("bio", TextHandler::new_detached()) + .unwrap(); + text.insert(0, "Ada", PosType::Unicode).unwrap(); + doc.commit_then_renew(); + + let text_id: ContainerID = text.id(); + let text_idx = doc.state.lock().arena.id_to_idx(&text_id).unwrap(); + + root.delete("profile").unwrap(); + doc.commit_then_renew(); + assert!(text.is_deleted()); + assert_eq!( + doc.state.lock().dead_cache_entry(text_idx), + None, + "ordinary descendants behind a mergeable edge must not be cached as deleted" + ); + + root.ensure_mergeable_map("profile").unwrap(); + doc.commit_then_renew(); + assert!(!text.is_deleted()); + assert_eq!( + doc.state.lock().dead_cache_entry(text_idx), + None, + "reactivated ordinary descendant must not leave a stale cache entry" + ); + } + + /// The same stale-cache hazard exists when reactivation arrives from a *peer* via import, + /// not just from a local `ensure_mergeable_*` call. The importing peer must not cache the + /// mergeable-dependent deleted result before the reactivation update arrives. + /// + /// The scenario, with two peers A (author) and B (importer): + /// 1. A creates the mergeable counter and deletes the key, then exports. + /// 2. B imports A's updates so the child exists but is unreachable, and queries + /// `is_deleted()`. + /// 3. A re-gets the counter (rewriting the marker, reactivating the child) and + /// exports just that new update. + /// 4. B imports the reactivation update. + /// 5. Assert B's cache never held a `deleted` entry for that idx. + /// + /// Like the local-reactivation test, this asserts cache contents directly rather than + /// through `is_deleted()`, because `is_deleted()` only trusts the cache via a release-only + /// early return; a public-API assertion would pass in debug even with the bug present. + #[test] + fn imported_mergeable_child_reactivation_clears_dead_cache() { + use crate::loro::ExportMode; + + let doc_a = LoroDoc::new_auto_commit(); + doc_a.set_peer_id(1).unwrap(); + let root_a = doc_a.get_map("state"); + let counter_a = root_a.ensure_mergeable_counter("revision").unwrap(); + counter_a.increment(1.0).unwrap(); + doc_a.commit_then_renew(); + root_a.delete("revision").unwrap(); + doc_a.commit_then_renew(); + + let cid: ContainerID = counter_a.id(); + + // B imports A's history: the child exists but is unreachable (marker cleared). + let doc_b = LoroDoc::new_auto_commit(); + doc_b.set_peer_id(2).unwrap(); + doc_b + .import(&doc_a.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + + let idx = doc_b.state.lock().arena.id_to_idx(&cid).unwrap(); + assert!(doc_b.state.lock().is_deleted(idx)); + assert_eq!( + doc_b.state.lock().dead_cache_entry(idx), + None, + "imported mergeable-dependent deletion must not be cached" + ); + + // A reactivates the child locally and exports just the new update. + let vv_before = doc_a.oplog_vv(); + root_a.ensure_mergeable_counter("revision").unwrap(); + doc_a.commit_then_renew(); + let reactivation = doc_a.export(ExportMode::updates(&vv_before)).unwrap(); + + // B imports the reactivation. There must be no stale deleted-cache entry left to mask it. + doc_b.import(&reactivation).unwrap(); + assert_eq!( + doc_b.state.lock().dead_cache_entry(idx), + None, + "imported reactivation must drop the stale deleted-cache entry" + ); + } +} diff --git a/vendor/loro-internal/src/state/list_state.rs b/vendor/loro-internal/src/state/list_state.rs new file mode 100644 index 00000000..474d4b11 --- /dev/null +++ b/vendor/loro-internal/src/state/list_state.rs @@ -0,0 +1,991 @@ +use std::{io::Write, ops::RangeBounds, sync::Weak}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext, FastStateSnapshot}; +use crate::{ + configure::Configure, + container::{idx::ContainerIdx, list::list_op::ListOp, ContainerID}, + event::{Diff, Index, InternalDiff, ListDiff}, + handler::ValueOrHandler, + op::{ListSlice, Op, RawOp, RawOpContent}, + LoroDocInner, LoroValue, +}; + +use generic_btree::{ + iter, + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, Cursor, LeafIndex, LengthFinder, UseLengthFinder, +}; +use loro_common::{IdFull, LoroError, LoroResult, ID}; +use loro_delta::array_vec::ArrayVec; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +const TINY_LIST_MAX: usize = 4; + +#[derive(Debug)] +pub struct ListState { + idx: ContainerIdx, + list: ListEntries, +} + +impl Clone for ListState { + fn clone(&self) -> Self { + Self { + idx: self.idx, + list: self.list.clone(), + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct Elem { + pub v: LoroValue, + pub id: IdFull, +} + +impl HasLength for Elem { + fn rle_len(&self) -> usize { + 1 + } +} + +impl Sliceable for Elem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert_eq!(range.start, 0); + assert_eq!(range.end, 1); + self.clone() + } + + fn split(&mut self, _pos: usize) -> Self { + unreachable!() + } +} + +impl Mergeable for Elem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } +} + +impl TryInsert for Elem { + fn try_insert(&mut self, _pos: usize, _elem: Self) -> Result<(), Self> { + Err(_elem) + } +} + +impl CanRemove for Elem { + fn can_remove(&self) -> bool { + false + } +} + +struct ListImpl; +impl BTreeTrait for ListImpl { + type Elem = Elem; + type Cache = isize; + type CacheDiff = isize; + const USE_DIFF: bool = true; + + #[inline(always)] + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let mut new_cache = 0; + for child in caches { + new_cache += child.cache; + } + + let diff = new_cache - *cache; + *cache = new_cache; + diff + } + + #[inline(always)] + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += diff; + } + + #[inline(always)] + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += diff2 + } + + #[inline(always)] + fn get_elem_cache(_elem: &Self::Elem) -> Self::Cache { + 1 + } + + #[inline(always)] + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + cache_lhs - cache_rhs + } +} + +impl UseLengthFinder for ListImpl { + fn get_len(cache: &isize) -> usize { + *cache as usize + } +} + +#[derive(Debug, Clone)] +enum ListEntries { + Tiny { + entries: SmallVec<[Elem; 1]>, + child_container_to_index: FxHashMap, + }, + Tree { + tree: BTree, + child_container_to_index: FxHashMap, + }, +} + +impl ListEntries { + fn new_tiny() -> Self { + Self::Tiny { + entries: SmallVec::new(), + child_container_to_index: Default::default(), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Tiny { entries, .. } => entries.is_empty(), + Self::Tree { tree, .. } => tree.is_empty(), + } + } + + fn len(&self) -> usize { + match self { + Self::Tiny { entries, .. } => entries.len(), + Self::Tree { tree, .. } => *tree.root_cache() as usize, + } + } + + fn iter(&self) -> Box + '_> { + match self { + Self::Tiny { entries, .. } => Box::new(entries.iter()), + Self::Tree { tree, .. } => Box::new(tree.iter()), + } + } +} + +impl ListState { + pub fn new(idx: ContainerIdx) -> Self { + Self { + idx, + list: ListEntries::new_tiny(), + } + } + + fn rebuild_tiny_child_index(&mut self) { + let ListEntries::Tiny { + entries, + child_container_to_index, + } = &mut self.list + else { + return; + }; + + child_container_to_index.clear(); + for (index, elem) in entries.iter().enumerate() { + if let LoroValue::Container(id) = &elem.v { + child_container_to_index.insert(id.clone(), index); + } + } + } + + fn upgrade_to_tree(&mut self) { + let ListEntries::Tiny { entries, .. } = + std::mem::replace(&mut self.list, ListEntries::new_tiny()) + else { + return; + }; + + let mut tree = BTree::new(); + let mut child_container_to_index = FxHashMap::default(); + for elem in entries { + let container = elem.v.as_container().cloned(); + let leaf = tree.push(elem); + if let Some(container) = container { + child_container_to_index.insert(container, leaf.leaf); + } + } + + self.list = ListEntries::Tree { + tree, + child_container_to_index, + }; + } + + fn downgrade_tree_if_tiny(&mut self) { + let ListEntries::Tree { tree, .. } = &self.list else { + return; + }; + + if *tree.root_cache() as usize > TINY_LIST_MAX { + return; + } + + let entries = tree.iter().cloned().collect(); + self.list = ListEntries::Tiny { + entries, + child_container_to_index: Default::default(), + }; + self.rebuild_tiny_child_index(); + } + + pub fn contains_child_container(&self, id: &ContainerID) -> bool { + match &self.list { + ListEntries::Tiny { + entries, + child_container_to_index, + } => child_container_to_index.get(id).is_some_and(|index| { + entries + .get(*index) + .is_some_and(|elem| elem.v.as_container() == Some(id)) + }), + ListEntries::Tree { + tree, + child_container_to_index, + } => child_container_to_index + .get(id) + .is_some_and(|leaf| tree.get_elem(*leaf).is_some()), + } + } + + pub fn get_child_container_index(&self, id: &ContainerID) -> Option { + let (leaf, list) = match &self.list { + ListEntries::Tiny { + entries, + child_container_to_index, + } => { + let index = *child_container_to_index.get(id)?; + entries.get(index)?; + return Some(index); + } + ListEntries::Tree { + tree, + child_container_to_index, + } => (*child_container_to_index.get(id)?, tree), + }; + + list.get_elem(leaf)?; + let mut index = 0; + list.visit_previous_caches(Cursor { leaf, offset: 0 }, |cache| match cache { + generic_btree::PreviousCache::NodeCache(cache) => { + index += *cache; + } + generic_btree::PreviousCache::PrevSiblingElem(..) => { + index += 1; + } + generic_btree::PreviousCache::ThisElemAndOffset { .. } => {} + }); + + Some(index as usize) + } + + pub fn insert(&mut self, index: usize, value: LoroValue, id: IdFull) { + if index > self.len() { + panic!("Index {index} out of range. The length is {}", self.len()); + } + + if let ListEntries::Tiny { entries, .. } = &mut self.list { + if entries.len() < TINY_LIST_MAX { + entries.insert( + index, + Elem { + v: value.clone(), + id, + }, + ); + self.rebuild_tiny_child_index(); + return; + } + + self.upgrade_to_tree(); + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let (leaf, data) = tree.insert::( + &index, + Elem { + v: value.clone(), + id, + }, + ); + + if value.is_container() { + child_container_to_index.insert(value.into_container().unwrap(), leaf.leaf); + } + + assert!(data.arr.is_empty()); + } + + pub fn push(&mut self, value: LoroValue, id: IdFull) { + if let ListEntries::Tiny { entries, .. } = &mut self.list { + if entries.len() < TINY_LIST_MAX { + entries.push(Elem { + v: value.clone(), + id, + }); + self.rebuild_tiny_child_index(); + return; + } + + self.upgrade_to_tree(); + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let leaf = tree.push(Elem { + v: value.clone(), + id, + }); + + if value.is_container() { + child_container_to_index.insert(value.into_container().unwrap(), leaf.leaf); + } + } + + pub fn delete(&mut self, index: usize) -> LoroValue { + if let ListEntries::Tiny { entries, .. } = &mut self.list { + let elem = entries.remove(index); + self.rebuild_tiny_child_index(); + return elem.v; + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let leaf = tree.query::(&index); + let leaf = tree.remove_leaf(leaf.unwrap().cursor).unwrap(); + if leaf.v.is_container() { + child_container_to_index.remove(leaf.v.as_container().unwrap()); + } + self.downgrade_tree_if_tiny(); + leaf.v + } + + pub fn delete_range( + &mut self, + range: impl RangeBounds, + mut notify_deletion: Option<&mut Vec>, + ) { + let start: usize = match range.start_bound() { + std::ops::Bound::Included(x) => *x, + std::ops::Bound::Excluded(x) => *x + 1, + std::ops::Bound::Unbounded => 0, + }; + let end: usize = match range.end_bound() { + std::ops::Bound::Included(x) => *x + 1, + std::ops::Bound::Excluded(x) => *x, + std::ops::Bound::Unbounded => self.len(), + }; + + if let ListEntries::Tiny { entries, .. } = &mut self.list { + for elem in entries.drain(start..end) { + if let LoroValue::Container(c) = elem.v { + if let Some(notify_deletion) = &mut notify_deletion { + notify_deletion.push(c); + } + } + } + self.rebuild_tiny_child_index(); + return; + } + + if end - start == 1 { + if let LoroValue::Container(c) = self.delete(start) { + if let Some(notify_deletion) = &mut notify_deletion { + notify_deletion.push(c); + } + } + return; + } + + let ListEntries::Tree { + tree, + child_container_to_index, + } = &mut self.list + else { + unreachable!() + }; + let q = start..end; + let start1 = tree.query::(&q.start); + let end1 = tree.query::(&q.end); + for v in iter::Drain::new(tree, start1, end1) { + if v.v.is_container() { + child_container_to_index.remove(v.v.as_container().unwrap()); + if let Some(notify_deletion) = &mut notify_deletion { + notify_deletion.push(v.v.into_container().unwrap()); + } + } + } + self.downgrade_tree_if_tiny(); + } + + // PERF: use &[LoroValue] + // PERF: batch + pub fn insert_batch(&mut self, index: usize, values: Vec, start_id: IdFull) { + let mut id = start_id; + for (i, value) in values.into_iter().enumerate() { + self.insert(index + i, value, id); + id = id.inc(1); + } + } + + pub fn iter(&self) -> impl Iterator { + self.iter_with_id().map(|x| &x.v) + } + + #[allow(unused)] + pub(crate) fn iter_with_id(&self) -> Box + '_> { + self.list.iter() + } + + pub fn len(&self) -> usize { + self.list.len() + } + + fn to_vec(&self) -> Vec { + let mut ans = Vec::with_capacity(self.len()); + for value in self.list.iter() { + ans.push(value.v.clone()); + } + ans + } + + pub fn get(&self, index: usize) -> Option<&LoroValue> { + match &self.list { + ListEntries::Tiny { entries, .. } => entries.get(index).map(|elem| &elem.v), + ListEntries::Tree { tree, .. } => { + let result = tree.query::(&index)?; + if result.found { + Some(&result.elem(tree).unwrap().v) + } else { + None + } + } + } + } + + pub fn get_id_at(&self, index: usize) -> Option { + match &self.list { + ListEntries::Tiny { entries, .. } => entries.get(index).map(|elem| elem.id), + ListEntries::Tree { tree, .. } => { + let result = tree.query::(&index)?; + if result.found { + Some(result.elem(tree).unwrap().id) + } else { + None + } + } + } + } + + #[allow(unused)] + pub(crate) fn check(&self) { + for value in self.iter() { + if let LoroValue::Container(c) = value { + self.get_child_index(c).unwrap(); + } + } + } + + pub fn get_index_of_id(&self, id: ID) -> Option { + for (i, elem) in self.iter_with_id().enumerate() { + if elem.id.id() == id { + return Some(i); + } + } + None + } +} + +impl ContainerState for ListState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.list.is_empty() + } + + fn apply_diff_and_convert( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, .. }: DiffApplyContext, + ) -> Diff { + let InternalDiff::ListRaw(delta) = diff else { + unreachable!() + }; + let mut ans: ListDiff = ListDiff::default(); + let mut index = 0; + let doc = &doc.upgrade().unwrap(); + for span in delta.iter() { + match span { + crate::delta::DeltaItem::Retain { retain: len, .. } => { + index += len; + ans.push_retain(*len, Default::default()); + } + crate::delta::DeltaItem::Insert { insert: value, .. } => { + let mut arr = Vec::new(); + match &value.values { + either::Either::Left(range) => { + for i in range.to_range() { + let value = doc.arena.get_value(i).unwrap(); + arr.push(value); + } + } + either::Either::Right(v) => arr.push(v.clone()), + } + for arr in ArrayVec::from_many( + arr.iter() + .map(|v| ValueOrHandler::from_value(v.clone(), doc)), + ) { + ans.push_insert(arr, Default::default()); + } + let len = arr.len(); + self.insert_batch(index, arr, value.id); + index += len; + } + crate::delta::DeltaItem::Delete { delete: len, .. } => { + self.delete_range(index..index + len, None); + ans.push_delete(*len); + } + } + } + + Diff::List(ans) + } + + fn validate_diff(&self, diff: &InternalDiff) -> LoroResult<()> { + let InternalDiff::ListRaw(delta) = diff else { + unreachable!() + }; + + let mut cursor = 0usize; + let mut projected = self.len(); + for span in delta.iter() { + match span { + crate::delta::DeltaItem::Retain { retain: len, .. } => { + cursor += len; + if cursor > projected { + return Err(LoroError::internal(format!( + "list diff retains {cursor} items but state only has {projected}", + ))); + } + } + crate::delta::DeltaItem::Insert { insert: value, .. } => { + let len = match &value.values { + either::Either::Left(range) => range.to_range().len(), + either::Either::Right(_) => 1, + }; + if cursor > projected { + return Err(LoroError::internal(format!( + "list diff inserts at {cursor} but state only has {projected}", + ))); + } + cursor += len; + projected += len; + } + crate::delta::DeltaItem::Delete { delete: len, .. } => { + if cursor + len > projected { + return Err(LoroError::internal(format!( + "list diff deletes {len} at {cursor} but state only has {projected}", + ))); + } + projected -= len; + } + } + } + + Ok(()) + } + + fn apply_diff( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, .. }: DiffApplyContext, + ) -> LoroResult<()> { + let doc = &doc.upgrade().unwrap(); + match diff { + InternalDiff::ListRaw(delta) => { + let mut index = 0; + for span in delta.iter() { + match span { + crate::delta::DeltaItem::Retain { retain: len, .. } => { + index += len; + } + crate::delta::DeltaItem::Insert { insert: value, .. } => { + let mut arr = Vec::new(); + match &value.values { + either::Either::Left(range) => { + for i in range.to_range() { + let value = doc.arena.get_value(i).unwrap(); + arr.push(value); + } + } + either::Either::Right(v) => arr.push(v.clone()), + } + + let len = arr.len(); + self.insert_batch(index, arr, value.id); + index += len; + } + crate::delta::DeltaItem::Delete { delete: len, .. } => { + self.delete_range(index..index + len, None); + } + } + } + } + _ => unreachable!(), + } + Ok(()) + } + + fn apply_local_op(&mut self, op: &RawOp, _: &Op) -> LoroResult { + let mut ans: ApplyLocalOpReturn = Default::default(); + match &op.content { + RawOpContent::List(list) => match list { + ListOp::Insert { slice, pos } => match slice { + ListSlice::RawData(list) => match list { + std::borrow::Cow::Borrowed(list) => { + self.insert_batch(*pos, list.to_vec(), op.id_full()); + } + std::borrow::Cow::Owned(list) => { + self.insert_batch(*pos, list.clone(), op.id_full()); + } + }, + _ => unreachable!(), + }, + ListOp::Delete(del) => { + self.delete_range(del.span.to_urange(), Some(&mut ans.deleted_containers)); + } + ListOp::Move { .. } => { + unreachable!() + } + ListOp::StyleStart { .. } => unreachable!(), + ListOp::StyleEnd { .. } => unreachable!(), + ListOp::Set { .. } => { + unreachable!() + } + }, + _ => unreachable!(), + } + Ok(ans) + } + + #[doc = " Convert a state to a diff that when apply this diff on a empty state,"] + #[doc = " the state will be the same as this state."] + fn to_diff(&mut self, doc: &Weak) -> Diff { + let doc = &doc.upgrade().unwrap(); + Diff::List(ListDiff::from_many( + self.to_vec() + .into_iter() + .map(|v| ValueOrHandler::from_value(v, doc)), + )) + } + + fn get_value(&mut self) -> LoroValue { + let ans = self.to_vec(); + LoroValue::List(ans.into()) + } + + fn get_child_index(&self, id: &ContainerID) -> Option { + self.get_child_container_index(id).map(Index::Seq) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.contains_child_container(id) + } + + fn get_child_containers(&self) -> Vec { + let mut ans = Vec::new(); + for elem in self.iter_with_id() { + if elem.v.is_container() { + ans.push(elem.v.as_container().unwrap().clone()); + } + } + ans + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +mod snapshot { + use loro_common::PeerID; + use serde_columnar::columnar; + + use crate::{ + encoding::value_register::ValueRegister, + state::{ + decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, ContainerCreationContext, + }, + }; + + use super::*; + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedListId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + lamport_sub_counter: i32, + } + + #[columnar(ser, de)] + struct EncodedListIds { + #[columnar(class = "vec", iter = "EncodedListId")] + ids: Vec, + } + + impl FastStateSnapshot for ListState { + /// Encodes the ListState snapshot in a compact binary format: + /// 1. Encodes the list values using postcard serialization + /// 2. Encodes a table of unique peer IDs + /// 3. For each element, encodes its ID as: + /// - Index of the peer ID in the table (LEB128) + /// - Counter (LEB128) + /// - Lamport timestamp (LEB128) + fn encode_snapshot_fast(&mut self, mut w: W) { + let value = self.get_value().into_list().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + let mut peers: ValueRegister = ValueRegister::new(); + let mut ids = Vec::with_capacity(self.len()); + for elem in self.iter_with_id() { + let id = elem.id; + let peer_idx = peers.register(&id.peer); + ids.push(EncodedListId { + peer_idx, + counter: id.counter, + lamport_sub_counter: (id.lamport as i32 - id.counter), + }); + } + + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let id_bytes = serde_columnar::to_vec(&EncodedListIds { ids }).unwrap(); + w.write_all(&id_bytes).unwrap(); + } + fn decode_value(bytes: &[u8]) -> LoroResult<(LoroValue, &[u8])> { + let (value, bytes) = postcard::take_from_bytes(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode list value failed".to_string().into_boxed_str(), + ) + })?; + let value: Vec = value; + Ok((LoroValue::List(value.into()), bytes)) + } + + fn decode_snapshot_fast( + idx: ContainerIdx, + (v, mut bytes): (LoroValue, &[u8]), + _ctx: ContainerCreationContext, + ) -> LoroResult + where + Self: Sized, + { + let peers = decode_peer_table(&mut bytes, "Decode list state failed")?; + + let EncodedListIds { ids } = serde_columnar::from_bytes(bytes).map_err(|err| { + state_decode_error(format!("Decode list state failed: invalid id table: {err}")) + })?; + + let list = v.as_list().unwrap(); + if ids.len() != list.len() { + return Err(state_decode_error( + "Decode list state failed: id/value length mismatch", + )); + } + + let mut ans = Self::new(idx); + for (i, id) in ids.into_iter().enumerate() { + let peer = decode_peer_from_table(&peers, id.peer_idx, "Decode list state failed")?; + ans.insert( + i, + list[i].clone(), + IdFull::new( + peer, + id.counter, + decode_lamport_from_delta( + id.counter, + id.lamport_sub_counter, + "Decode list state failed", + )?, + ), + ); + } + + Ok(ans) + } + } + + #[cfg(test)] + mod tests { + use loro_common::LoroValue; + + use crate::{container::idx::ContainerIdx, state::ContainerCreationContext}; + + use super::*; + + #[test] + fn list_fast_snapshot_rejects_negative_counter() { + let mut bytes = Vec::new(); + postcard::to_io(&vec![LoroValue::I64(1)], &mut bytes).unwrap(); + leb128::write::unsigned(&mut bytes, 1).unwrap(); + bytes.extend_from_slice(&1_u64.to_le_bytes()); + bytes.extend_from_slice( + &serde_columnar::to_vec(&EncodedListIds { + ids: vec![EncodedListId { + peer_idx: 0, + counter: -1, + lamport_sub_counter: 1, + }], + }) + .unwrap(), + ); + + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List); + let (value, state_bytes) = ListState::decode_value(&bytes).unwrap(); + assert!(ListState::decode_snapshot_fast( + idx, + (value, state_bytes), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .is_err()); + } + } +} + +#[cfg(test)] +mod test { + use itertools::Itertools; + use loro_common::{Counter, Lamport}; + + use crate::state::ContainerCreationContext; + + use super::*; + + #[test] + fn test() { + let mut list = ListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::List, + )); + fn id(name: &str) -> ContainerID { + ContainerID::new_root(name, crate::ContainerType::List) + } + list.insert(0, LoroValue::Container(id("abc")), IdFull::new(0, 0, 0)); + list.insert(0, LoroValue::Container(id("x")), IdFull::new(0, 0, 0)); + assert_eq!(list.get_child_container_index(&id("x")), Some(0)); + assert_eq!(list.get_child_container_index(&id("abc")), Some(1)); + list.insert(1, LoroValue::Bool(false), IdFull::new(0, 0, 0)); + assert_eq!(list.get_child_container_index(&id("x")), Some(0)); + assert_eq!(list.get_child_container_index(&id("abc")), Some(2)); + } + + #[test] + fn test_list_fast_snapshot() { + let mut list = ListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::List, + )); + let mut w = Vec::new(); + list.encode_snapshot_fast(&mut w); + println!("Empty: {}", w.len()); + + list.insert(0, LoroValue::I64(0), IdFull::new(0, 0, 0)); + list.insert(1, LoroValue::I64(2), IdFull::new(1, 1, 1)); + list.insert(2, LoroValue::I64(4), IdFull::new(1, 2, 2)); + let mut w = Vec::new(); + list.encode_snapshot_fast(&mut w); + assert!(w.len() <= 39, "w.len() = {}", w.len()); + let (v, left) = ListState::decode_value(&w).unwrap(); + let mut new_list = ListState::decode_snapshot_fast( + ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List), + (v.clone(), left), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .unwrap(); + new_list.check(); + assert_eq!( + new_list.get_value(), + vec![LoroValue::I64(0), LoroValue::I64(2), LoroValue::I64(4)].into() + ); + assert_eq!(new_list.get_value(), v); + let v = new_list.list.iter().collect_vec(); + assert_eq!(v[0].id.peer, 0); + assert_eq!(v[0].id.counter, 0 as Counter); + assert_eq!(v[0].id.lamport, 0 as Lamport); + + assert_eq!(v[1].id.peer, 1); + assert_eq!(v[1].id.counter, 1 as Counter); + assert_eq!(v[1].id.lamport, 1 as Lamport); + + assert_eq!(v[2].id.peer, 1); + assert_eq!(v[2].id.counter, 2 as Counter); + assert_eq!(v[2].id.lamport, 2 as Lamport); + } + + #[test] + fn list_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::List); + let ctx = ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }; + + let value = LoroValue::from(vec![LoroValue::I64(1)]); + assert!(ListState::decode_snapshot_fast(idx, (value.clone(), &[1]), ctx).is_err()); + + let mut empty = ListState::new(idx); + let mut bytes = Vec::new(); + empty.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = ListState::decode_value(&bytes).unwrap(); + assert!(ListState::decode_snapshot_fast(idx, (value, state_bytes), ctx).is_err()); + } +} diff --git a/vendor/loro-internal/src/state/map_state.rs b/vendor/loro-internal/src/state/map_state.rs new file mode 100644 index 00000000..947eb6bc --- /dev/null +++ b/vendor/loro-internal/src/state/map_state.rs @@ -0,0 +1,723 @@ +use std::{collections::BTreeMap, sync::Weak}; + +use loro_common::{ContainerID, IdLp, LoroResult, PeerID}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; + +use crate::{ + configure::Configure, + container::{idx::ContainerIdx, map::MapSet}, + delta::{MapValue, ResolvedMapDelta, ResolvedMapValue}, + diff_calc::DiffMode, + event::{Diff, Index, InternalDiff}, + handler::ValueOrHandler, + op::{Op, RawOp, RawOpContent}, + InternalString, LoroDocInner, LoroValue, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +const TINY_MAP_MAX: usize = 4; + +/// Apply the mergeable marker → `Container` boundary translation when a parent cid is known. +/// +/// `MapState` is used both with and without a registered parent cid (e.g. during decode), so the +/// translation no-ops when the parent is missing — markers stay as their stored `Binary` form. +fn translate_with_parent( + parent_id: Option<&ContainerID>, + key: &InternalString, + value: LoroValue, +) -> LoroValue { + match parent_id { + Some(parent) => { + loro_common::translate_mergeable_marker_value(parent, key.as_ref(), value) + } + None => value, + } +} + +#[derive(Debug, Clone)] +enum MapEntries { + SortedTiny(SmallVec<[(InternalString, MapValue); 1]>), + Tree(BTreeMap), +} + +impl Default for MapEntries { + fn default() -> Self { + Self::SortedTiny(SmallVec::new()) + } +} + +enum MapEntriesIter<'a> { + SortedTiny(std::slice::Iter<'a, (InternalString, MapValue)>), + Tree(std::collections::btree_map::Iter<'a, InternalString, MapValue>), +} + +impl<'a> Iterator for MapEntriesIter<'a> { + type Item = (&'a InternalString, &'a MapValue); + + fn next(&mut self) -> Option { + match self { + Self::SortedTiny(iter) => iter.next().map(|(key, value)| (key, value)), + Self::Tree(iter) => iter.next(), + } + } +} + +impl MapEntries { + fn debug_assert_sorted_tiny(entries: &[(InternalString, MapValue)]) { + debug_assert!(entries.windows(2).all(|pair| { + let left = &pair[0].0; + let right = &pair[1].0; + left < right + })); + } + + fn is_empty(&self) -> bool { + match self { + Self::SortedTiny(entries) => entries.is_empty(), + Self::Tree(map) => map.is_empty(), + } + } + + fn get(&self, key: &InternalString) -> Option<&MapValue> { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + entries + .binary_search_by(|(entry_key, _)| entry_key.cmp(key)) + .ok() + .map(|index| &entries[index].1) + } + Self::Tree(map) => map.get(key), + } + } + + fn insert(&mut self, key: InternalString, value: MapValue) -> Option { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + match entries.binary_search_by(|(entry_key, _)| entry_key.cmp(&key)) { + Ok(index) => return Some(std::mem::replace(&mut entries[index].1, value)), + Err(index) if entries.len() < TINY_MAP_MAX => { + entries.insert(index, (key, value)); + Self::debug_assert_sorted_tiny(entries); + return None; + } + Err(_) => {} + } + + let mut map = BTreeMap::new(); + for (key, value) in entries.drain(..) { + map.insert(key, value); + } + let result = map.insert(key, value); + *self = Self::Tree(map); + result + } + Self::Tree(map) => map.insert(key, value), + } + } + + fn remove(&mut self, key: &InternalString) -> Option { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + entries + .binary_search_by(|(entry_key, _)| entry_key.cmp(key)) + .ok() + .map(|index| entries.remove(index).1) + } + Self::Tree(map) => { + let result = map.remove(key); + if result.is_some() && map.len() <= TINY_MAP_MAX { + let entries = std::mem::take(map).into_iter().collect(); + *self = Self::SortedTiny(entries); + } + result + } + } + } + + fn iter(&self) -> MapEntriesIter<'_> { + match self { + Self::SortedTiny(entries) => { + Self::debug_assert_sorted_tiny(entries); + MapEntriesIter::SortedTiny(entries.iter()) + } + Self::Tree(map) => MapEntriesIter::Tree(map.iter()), + } + } +} + +#[derive(Debug, Clone)] +enum ChildContainers { + Tiny(SmallVec<[(ContainerID, InternalString); 1]>), + Map(FxHashMap), +} + +impl Default for ChildContainers { + fn default() -> Self { + Self::Tiny(SmallVec::new()) + } +} + +impl ChildContainers { + fn get(&self, id: &ContainerID) -> Option<&InternalString> { + match self { + Self::Tiny(entries) => entries + .iter() + .find_map(|(entry_id, key)| (entry_id == id).then_some(key)), + Self::Map(map) => map.get(id), + } + } + + fn contains_key(&self, id: &ContainerID) -> bool { + self.get(id).is_some() + } + + fn insert(&mut self, id: ContainerID, key: InternalString) -> Option { + match self { + Self::Tiny(entries) => { + if let Some((_, old_key)) = entries.iter_mut().find(|(entry_id, _)| entry_id == &id) + { + return Some(std::mem::replace(old_key, key)); + } + + if entries.len() < TINY_MAP_MAX { + entries.push((id, key)); + return None; + } + + let mut map = FxHashMap::default(); + for (id, key) in entries.drain(..) { + map.insert(id, key); + } + let result = map.insert(id, key); + *self = Self::Map(map); + result + } + Self::Map(map) => map.insert(id, key), + } + } + + fn remove(&mut self, id: &ContainerID) -> Option { + match self { + Self::Tiny(entries) => entries + .iter() + .position(|(entry_id, _)| entry_id == id) + .map(|index| entries.swap_remove(index).1), + Self::Map(map) => { + let result = map.remove(id); + if result.is_some() && map.len() <= TINY_MAP_MAX { + let entries = std::mem::take(map).into_iter().collect(); + *self = Self::Tiny(entries); + } + result + } + } + } +} + +#[derive(Debug, Clone)] +pub struct MapState { + idx: ContainerIdx, + map: MapEntries, + /// Parent-edge index for regular child containers (`MapValue::Container`). Mergeable child + /// edges are resolved by `DocState` from their deterministic cid and this map's marker + /// value, not stored here. + child_containers: ChildContainers, + size: usize, +} + +impl ContainerState for MapState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.map.is_empty() + } + + fn apply_diff_and_convert( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, mode }: DiffApplyContext, + ) -> Diff { + let InternalDiff::Map(delta) = diff else { + unreachable!() + }; + let doc = &doc.upgrade().unwrap(); + // The parent cid is the binding side of a mergeable child marker. Cached here so we + // translate marker values into Container values once per delta apply instead of per entry. + let parent_id = doc.arena.idx_to_id(self.idx); + let force = matches!(mode, DiffMode::Checkout | DiffMode::Linear); + let mut resolved_delta = ResolvedMapDelta::new(); + for (key, value) in delta.updated.into_iter() { + let Some(value) = value else { + // uncreate op + assert_eq!(mode, DiffMode::Checkout); + self.remove(&key); + resolved_delta = resolved_delta.with_entry(key, ResolvedMapValue::new_unset()); + continue; + }; + + let mut changed = false; + if force { + self.insert(key.clone(), value.clone()); + changed = true; + } else { + match self.map.get(&key) { + Some(old_value) if old_value > &value => {} + _ => { + self.insert(key.clone(), value.clone()); + changed = true; + } + } + } + + if changed { + resolved_delta = resolved_delta.with_entry( + key.clone(), + ResolvedMapValue { + idlp: IdLp::new(value.peer, value.lamp), + value: value + .value + .map(|v| translate_with_parent(parent_id.as_ref(), &key, v)) + .map(|v| ValueOrHandler::from_value(v, doc)), + }, + ) + } + } + + Diff::Map(resolved_delta) + } + + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + let _ = self.apply_diff_and_convert(diff, ctx); + Ok(()) + } + + fn apply_local_op(&mut self, op: &RawOp, _: &Op) -> LoroResult { + let mut ans: ApplyLocalOpReturn = Default::default(); + match &op.content { + RawOpContent::Map(MapSet { key, value }) => { + let prev = self.insert( + key.clone(), + MapValue { + lamp: op.lamport, + peer: op.id.peer, + value: value.clone(), + }, + ); + + if let Some(MapValue { + value: Some(LoroValue::Container(c)), + .. + }) = prev + { + ans.deleted_containers.push(c); + } + } + _ => unreachable!(), + } + + Ok(ans) + } + + #[doc = " Convert a state to a diff that when apply this diff on a empty state,"] + #[doc = " the state will be the same as this state."] + fn to_diff(&mut self, doc: &Weak) -> Diff { + let doc_strong = doc.upgrade().unwrap(); + let parent_id = doc_strong.arena.idx_to_id(self.idx); + // Build resolved values directly instead of going through + // `ResolvedMapValue::from_map_value`: that path forced a full `MapValue` clone per entry + // just to overwrite `.value` with the translated form. Cloning only the inner + // `Option` is enough and saves an alloc per entry on every state-to-diff. + Diff::Map(ResolvedMapDelta { + updated: self + .map + .iter() + .map(|(k, v)| { + let value = v + .value + .clone() + .map(|val| translate_with_parent(parent_id.as_ref(), k, val)) + .map(|val| ValueOrHandler::from_value(val, &doc_strong)); + ( + k.clone(), + ResolvedMapValue { + idlp: IdLp::new(v.peer, v.lamp), + value, + }, + ) + }) + .collect::>(), + }) + } + + fn get_value(&mut self) -> LoroValue { + let ans = self.to_map(); + LoroValue::Map(ans.into()) + } + + fn get_child_index(&self, id: &ContainerID) -> Option { + self.child_containers.get(id).map(|x| Index::Key(x.clone())) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.child_containers.contains_key(id) + } + + fn get_child_containers(&self) -> Vec { + let mut ans = Vec::new(); + for (_, value) in self.map.iter() { + if let Some(LoroValue::Container(x)) = &value.value { + ans.push(x.clone()); + } + } + ans + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +impl MapState { + pub fn new(idx: ContainerIdx) -> Self { + Self { + idx, + map: Default::default(), + child_containers: Default::default(), + size: 0, + } + } + + pub fn insert(&mut self, key: InternalString, value: MapValue) -> Option { + let value_yes = value.value.is_some(); + if let Some(LoroValue::Container(id)) = &value.value { + self.child_containers.insert(id.clone(), key.clone()); + } + + let result = self.map.insert(key.clone(), value); + if let Some(Some(LoroValue::Container(c))) = result.as_ref().map(|x| &x.value) { + self.child_containers.remove(c); + } + + match (&result, value_yes) { + (Some(x), true) if x.value.is_none() => { + self.size += 1; + } + (None, true) => { + self.size += 1; + } + (Some(x), false) if x.value.is_some() => { + self.size -= 1; + } + _ => {} + }; + + result + } + + pub fn remove(&mut self, key: &InternalString) { + let result = self.map.remove(key); + if let Some(x) = result { + if x.value.is_some() { + self.size -= 1; + } + if let Some(LoroValue::Container(id)) = x.value { + self.child_containers.remove(&id); + } + }; + } + + pub fn iter(&self) -> impl Iterator { + self.map.iter() + } + + fn to_map(&self) -> FxHashMap { + let mut ans = FxHashMap::with_capacity_and_hasher(self.len(), Default::default()); + for (key, value) in self.map.iter() { + if value.value.is_none() { + continue; + } + + ans.insert(key.to_string(), value.value.as_ref().cloned().unwrap()); + } + + ans + } + + pub fn get(&self, k: &str) -> Option<&LoroValue> { + match self.map.get(&k.into()) { + Some(value) => match &value.value { + Some(v) => Some(v), + None => None, + }, + None => None, + } + } + + pub fn len(&self) -> usize { + self.size + } + + pub fn get_last_edit_peer(&self, key: &str) -> Option { + self.map.get(&key.into()).map(|v| v.peer) + } +} + +mod snapshot { + + use std::collections::BTreeMap; + + use loro_common::{InternalString, LoroValue}; + use rustc_hash::{FxHashMap, FxHashSet}; + use serde_columnar::Itertools; + + use crate::{ + delta::MapValue, + encoding::value_register::ValueRegister, + state::{ + decode_peer_from_table, decode_peer_table, read_state_leb_u64, state_decode_error, + ContainerCreationContext, ContainerState, FastStateSnapshot, + }, + }; + + use super::MapState; + + impl FastStateSnapshot for MapState { + fn encode_snapshot_fast(&mut self, mut w: W) { + // 1. LoroValue + // 2. Vec keys_with_none_value + // 3. leb128 peer_num + peers (in u64) + // 3. Groups of (leb128 peer_idx, leb128 lamport), each has a respective map entry + // from either 1 or 2 when they all sorted by the key strings + let value = self.get_value().into_map().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + + let keys_with_none_value = self + .map + .iter() + .filter_map(|(k, v)| if v.value.is_some() { None } else { Some(k) }) + .collect_vec(); + postcard::to_io(&keys_with_none_value, &mut w).unwrap(); + let mut peer_register = ValueRegister::new(); + for (_, v) in self.map.iter() { + peer_register.register(&v.peer); + } + + leb128::write::unsigned(&mut w, peer_register.vec().len() as u64).unwrap(); + for p in peer_register.vec() { + w.write_all(&p.to_le_bytes()).unwrap(); + } + let mut keys: Vec<&InternalString> = self.map.iter().map(|(key, _)| key).collect(); + keys.sort_unstable(); + for key in keys.into_iter() { + let value = self.map.get(key).unwrap(); + let peer_idx = peer_register.register(&value.peer); + leb128::write::unsigned(&mut w, peer_idx as u64).unwrap(); + leb128::write::unsigned(&mut w, value.lamp as u64).unwrap(); + } + } + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + let (value, bytes) = postcard::take_from_bytes::>(bytes) + .map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode map value failed".to_string().into_boxed_str(), + ) + })?; + Ok((LoroValue::Map(value.into()), bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (value, bytes): (loro_common::LoroValue, &[u8]), + _ctx: ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let value = value.into_map().unwrap(); + // keys_with_none_value + let (keys_with_none_value, mut bytes) = + postcard::take_from_bytes::>(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode map keys_with_none_value failed" + .to_string() + .into_boxed_str(), + ) + })?; + let keys_with_none_value: FxHashSet<_> = keys_with_none_value.into_iter().collect(); + + // peers + let peers = decode_peer_table(&mut bytes, "Decode map state failed")?; + + // + let mut ans = MapState::new(idx); + let mut keys: Vec<_> = value.keys().map(|x| x.as_str().into()).collect(); + keys.extend(keys_with_none_value.iter().cloned()); + keys.sort_unstable(); + + for key in keys { + let peer_idx = + usize::try_from(read_state_leb_u64(&mut bytes, "Decode map state failed")?) + .map_err(|_| { + state_decode_error("Decode map state failed: peer index overflow") + })?; + let lamp = + u32::try_from(read_state_leb_u64(&mut bytes, "Decode map state failed")?) + .map_err(|_| { + state_decode_error("Decode map state failed: lamport overflow") + })?; + let peer = decode_peer_from_table(&peers, peer_idx, "Decode map state failed")?; + + if keys_with_none_value.contains(&key) { + ans.insert( + key, + MapValue { + value: None, + lamp, + peer, + }, + ); + } else { + let value = value.get(&*key).unwrap(); + ans.insert( + key, + MapValue { + value: Some(value.clone()), + lamp, + peer, + }, + ); + } + } + + if !bytes.is_empty() { + return Err(loro_common::LoroError::DecodeError( + "Decode map state failed".to_string().into_boxed_str(), + )); + } + + Ok(ans) + } + } + + impl MapState { + pub(crate) fn decode_value_as_btree_map( + bytes: &[u8], + ) -> loro_common::LoroResult<(BTreeMap, &[u8])> { + let (value, bytes) = postcard::take_from_bytes::>(bytes) + .map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode map value failed".to_string().into_boxed_str(), + ) + })?; + Ok((value, bytes)) + } + } + + #[cfg(test)] + mod map_snapshot_test { + use loro_common::LoroValue; + + use crate::container::idx::ContainerIdx; + + use super::*; + + #[test] + fn map_fast_snapshot() { + let mut map = MapState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::Map, + )); + map.insert( + "1".into(), + MapValue { + value: None, + lamp: 1, + peer: 1, + }, + ); + map.insert( + "2".into(), + MapValue { + value: Some(LoroValue::I64(0)), + lamp: 2, + peer: 2, + }, + ); + map.insert( + "3".into(), + MapValue { + value: Some(LoroValue::Double(1.0)), + lamp: 3, + peer: 3, + }, + ); + + let mut bytes = Vec::new(); + map.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 50); + + let (value, bytes) = MapState::decode_value(&bytes).unwrap(); + { + let m = value.clone().into_map().unwrap(); + assert_eq!(m.len(), 2); + assert_eq!(m.get("2").unwrap(), &LoroValue::I64(0)); + assert_eq!(m.get("3").unwrap(), &LoroValue::Double(1.0)); + } + + let new_map = MapState::decode_snapshot_fast( + ContainerIdx::from_index_and_type(0, loro_common::ContainerType::Map), + (value, bytes), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .unwrap(); + let v = new_map.map.get(&"2".into()).unwrap(); + assert_eq!( + v, + &MapValue { + value: Some(LoroValue::I64(0)), + lamp: 2, + peer: 2, + } + ); + let v = new_map.map.get(&"1".into()).unwrap(); + assert_eq!( + v, + &MapValue { + value: None, + lamp: 1, + peer: 1, + } + ); + } + + #[test] + fn map_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::Map); + let ctx = ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }; + let value = LoroValue::from(std::collections::HashMap::from([( + "key".to_string(), + LoroValue::I64(1), + )])); + + let mut empty = MapState::new(idx); + let mut bytes = Vec::new(); + empty.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = MapState::decode_value(&bytes).unwrap(); + assert!(MapState::decode_snapshot_fast(idx, (value, state_bytes), ctx).is_err()); + } + } +} diff --git a/vendor/loro-internal/src/state/mergeable.rs b/vendor/loro-internal/src/state/mergeable.rs new file mode 100644 index 00000000..f41080c2 --- /dev/null +++ b/vendor/loro-internal/src/state/mergeable.rs @@ -0,0 +1,93 @@ +//! Mergeable-container edge resolution on `DocState`. +//! +//! Mergeable child containers (created via `MapHandler::ensure_mergeable_*`) use deterministic +//! `ContainerID::Root` ids in a reserved namespace. The cid encodes `(parent, key, kind)`, while +//! the parent map's compact binary child ref is the single source of truth for whether that child +//! is currently active. +//! +//! The ref is intentionally stored as a specially constructed binary marker value. This keeps +//! user-editable strings from being reinterpreted as internal container topology, and lets the +//! resolver fail closed unless the ref's digest matches the exact `(parent, key, kind)` being +//! resolved. + +use loro_common::{ContainerID, ContainerType, InternalString, LoroMapValue}; + +use crate::{container::idx::ContainerIdx, event::Index}; + +use super::{ContainerState, DocState}; + +impl DocState { + /// Resolve a child edge using the document-level logical edge model. + /// + /// Ordinary child containers use the materialized child index stored in their parent state. + /// Mergeable children are resolved lazily from their deterministic cid plus the parent map's + /// current binary child ref, so snapshot/update import does not need any eager mergeable-edge + /// rebuild. + pub(super) fn get_logical_child_index( + &mut self, + parent_idx: ContainerIdx, + child_id: &ContainerID, + ) -> Option { + if let Some((parent_id, key, kind)) = child_id.parse_mergeable() { + return self.resolve_mergeable_child_index(parent_idx, &parent_id, &key, kind); + } + + self.store + .get_container_mut(parent_idx) + .and_then(|parent_state| parent_state.get_child_index(child_id)) + } + + pub(super) fn contains_logical_child( + &mut self, + parent_idx: ContainerIdx, + child_id: &ContainerID, + ) -> bool { + self.get_logical_child_index(parent_idx, child_id).is_some() + } + + fn resolve_mergeable_child_index( + &mut self, + parent_idx: ContainerIdx, + encoded_parent_id: &ContainerID, + key: &str, + kind: ContainerType, + ) -> Option { + if parent_idx.get_type() != ContainerType::Map { + return None; + } + + let actual_parent_id = self.arena.idx_to_id(parent_idx)?; + if &actual_parent_id != encoded_parent_id { + return None; + } + + let value = self.store.map_get(parent_idx, key)?; + if loro_common::parse_mergeable_marker(&actual_parent_id, key, &value) == Some(kind) { + Some(Index::Key(key.into())) + } else { + None + } + } + + /// Derive `(key, cid)` pairs for the active mergeable children of a map directly from its + /// current value table. + /// + /// This is used by deep-value and alive-container walks. It mirrors regular map semantics: + /// the current value at a key determines which child is visible. For mergeable children, that + /// value is a compact binary child ref rather than `LoroValue::Container`. + pub(super) fn mergeable_children_from_value( + &self, + parent_id: &ContainerID, + map_value: &LoroMapValue, + ) -> Vec<(InternalString, ContainerID)> { + let mut ans = Vec::new(); + for (key, value) in map_value.iter() { + if let Some(kind) = loro_common::parse_mergeable_marker(parent_id, key, value) { + let key_istr: InternalString = key.as_str().into(); + let cid = ContainerID::new_mergeable(parent_id, key, kind); + ans.push((key_istr, cid)); + } + } + ans + } +} diff --git a/vendor/loro-internal/src/state/movable_list_state.rs b/vendor/loro-internal/src/state/movable_list_state.rs new file mode 100644 index 00000000..ea8f89ba --- /dev/null +++ b/vendor/loro-internal/src/state/movable_list_state.rs @@ -0,0 +1,2032 @@ +use itertools::Itertools; +use loro_delta::{array_vec::ArrayVec, DeltaRope, DeltaRopeBuilder}; +use serde_columnar::columnar; +use std::sync::Weak; +use tracing::{instrument, warn}; + +use generic_btree::BTree; +use loro_common::{CompactIdLp, ContainerID, IdFull, IdLp, LoroResult, LoroValue, PeerID, ID}; +use rustc_hash::FxHashMap; + +use crate::{ + configure::Configure, + container::{idx::ContainerIdx, list::list_op::ListOp}, + delta::DeltaItem, + diff_calc::DiffMode, + event::{Diff, Index, InternalDiff, ListDeltaMeta}, + handler::ValueOrHandler, + op::{ListSlice, Op, RawOp}, + ListDiff, LoroDocInner, +}; + +use self::{ + inner::{InnerState, UpdateResultFromPosChange}, + list_item_tree::{MovableListTreeTrait, OpLenQuery, UserLenQuery}, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug, Clone)] +pub struct MovableListState { + idx: ContainerIdx, + inner: InnerState, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ListItem { + pointed_by: Option, + pub(crate) id: IdFull, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Element { + pub(crate) value: LoroValue, + pub(crate) value_id: IdLp, + pub(crate) pos: IdLp, +} + +impl Element { + pub(crate) fn value(&self) -> &LoroValue { + &self.value + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum IndexType { + /// This includes the ones that are not being pointed at, which may not be visible to users + ForOp, + /// This only includes the ones that are being pointed at, which means visible to users + ForUser, +} + +mod list_item_tree { + use std::{ + iter::Sum, + ops::{Add, AddAssign, Sub}, + }; + + use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTreeTrait, + }; + + use crate::utils::query_by_len::{IndexQuery, QueryByLen}; + + use super::ListItem; + + impl HasLength for ListItem { + fn rle_len(&self) -> usize { + 1 + } + } + + impl Mergeable for ListItem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } + } + + impl Sliceable for ListItem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert_eq!(range.len(), 1); + self.clone() + } + } + + impl CanRemove for ListItem { + fn can_remove(&self) -> bool { + false + } + } + + impl TryInsert for ListItem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } + } + + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] + pub(super) struct Cache { + /// This length info include the ones that were deleted. + /// It's used in op and diff calculation. + pub include_dead_len: i32, + /// This length info does not include the ones that were deleted. + /// So it's facing the users. + pub user_len: i32, + } + + impl Add for Cache { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self { + include_dead_len: self.include_dead_len + rhs.include_dead_len, + user_len: self.user_len + rhs.user_len, + } + } + } + + impl AddAssign for Cache { + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } + } + + impl Sub for Cache { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self { + include_dead_len: self.include_dead_len - rhs.include_dead_len, + user_len: self.user_len - rhs.user_len, + } + } + } + + impl Sum for Cache { + fn sum>(iter: I) -> Self { + iter.fold(Self::default(), |acc, x| acc + x) + } + } + + impl CanRemove for Cache { + fn can_remove(&self) -> bool { + self.include_dead_len == 0 + } + } + + pub(super) struct MovableListTreeTrait; + + impl BTreeTrait for MovableListTreeTrait { + type Elem = ListItem; + + type Cache = Cache; + + type CacheDiff = Cache; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + let new: Cache = caches.iter().map(|x| *x.cache()).sum(); + let diff = new - *cache; + *cache = new; + diff + } + + fn apply_cache_diff(cache: &mut Self::Cache, diff: &Self::CacheDiff) { + *cache += *diff; + } + + fn merge_cache_diff(diff1: &mut Self::CacheDiff, diff2: &Self::CacheDiff) { + *diff1 += *diff2; + } + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + if elem.pointed_by.is_some() { + Cache { + include_dead_len: 1, + user_len: 1, + } + } else { + Cache { + include_dead_len: 1, + user_len: 0, + } + } + } + + fn new_cache_to_diff(cache: &Self::Cache) -> Self::CacheDiff { + *cache + } + + fn sub_cache(cache_lhs: &Self::Cache, cache_rhs: &Self::Cache) -> Self::CacheDiff { + *cache_lhs - *cache_rhs + } + } + + pub(crate) struct UserLenQueryT; + pub(crate) type UserLenQuery = IndexQuery; + + impl QueryByLen for UserLenQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.user_len as usize + } + + fn get_elem_len(elem: &::Elem) -> usize { + if elem.pointed_by.is_some() { + 1 + } else { + 0 + } + } + + fn get_offset_and_found( + left: usize, + elem: &::Elem, + ) -> (usize, bool) { + if left == 0 && elem.pointed_by.is_some() { + (0, true) + } else { + (1, false) + } + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.user_len as usize + } + } + + pub(crate) struct IncludeDeadLenQueryT; + pub(crate) type OpLenQuery = IndexQuery; + + impl QueryByLen for IncludeDeadLenQueryT { + fn get_cache_len(cache: &::Cache) -> usize { + cache.include_dead_len as usize + } + + fn get_elem_len(_elem: &::Elem) -> usize { + 1 + } + + fn get_offset_and_found( + left: usize, + _elem: &::Elem, + ) -> (usize, bool) { + if left == 0 { + return (0, true); + } + + (1, false) + } + + fn get_cache_entity_len(cache: &::Cache) -> usize { + cache.include_dead_len as usize + } + } +} + +/// The inner state of the list. +/// +/// The operations inside this mod need to ensure certain invariants. +/// +/// - list items' `pointed_by` must be consistent with the `elements`' `pos`. +/// - `id_to_list_leaf` must be consistent with the list. +/// - `child_container_to_elem` must be consistent with the element. +mod inner { + use generic_btree::{BTree, Cursor, LeafIndex, Query}; + use loro_common::{CompactIdLp, ContainerID, IdFull, IdLp, LoroValue, PeerID}; + use rustc_hash::{FxHashMap, FxHashSet}; + use tracing::error; + + use super::{ + list_item_tree::{MovableListTreeTrait, OpLenQuery, UserLenQuery}, + Element, IndexType, ListItem, + }; + + #[derive(Debug, Clone)] + pub(super) struct InnerState { + list: BTree, + id_to_list_leaf: FxHashMap, + elements: FxHashMap, + /// This mapping may be out of date when elem is removed/updated. + /// But it's guaranteed that if there is a ContainerID in the actual list, + /// it will be mapped correctly. + child_container_to_elem: FxHashMap, + } + + impl PartialEq for InnerState { + fn eq(&self, other: &Self) -> bool { + let v = self.id_to_list_leaf == other.id_to_list_leaf + && self.elements == other.elements + && self.child_container_to_elem == other.child_container_to_elem; + if !v { + false + } else { + self.list.iter().zip(other.list.iter()).all(|(a, b)| a == b) + } + } + } + + fn eq(a: T, b: T) -> Result<(), ()> { + if a == b { + Ok(()) + } else { + Err(()) + } + } + + impl InnerState { + pub fn new() -> Self { + Self { + list: BTree::new(), + id_to_list_leaf: FxHashMap::default(), + elements: FxHashMap::default(), + child_container_to_elem: FxHashMap::default(), + } + } + + #[inline] + pub fn child_container_to_elem(&self) -> &FxHashMap { + &self.child_container_to_elem + } + + #[inline] + pub fn elements(&self) -> &FxHashMap { + &self.elements + } + + #[inline] + pub fn list(&self) -> &BTree { + &self.list + } + + pub fn remove_elem_by_id(&mut self, elem_id: &CompactIdLp) { + self.elements.remove(elem_id); + } + + #[allow(dead_code)] + pub fn check_consistency(&self) { + if !cfg!(debug_assertions) { + return; + } + + let mut failed = false; + if self.check_list_item_consistency().is_err() { + error!("list item consistency check failed, self={:#?}", self); + failed = true; + } + + if self.check_child_container_to_elem_consistency().is_err() { + error!( + "child container to elem consistency check failed, self={:#?}", + self + ); + failed = true; + } + + if failed { + panic!("consistency check failed"); + } + } + + fn check_list_item_consistency(&self) -> Result<(), ()> { + let mut visited_ids = FxHashSet::default(); + for list_item in self.list.iter() { + { + if visited_ids.contains(&list_item.id.idlp()) { + error!("duplicate list item id"); + return Err(()); + } + visited_ids.insert(list_item.id.idlp()); + } + let leaf = self.id_to_list_leaf.get(&list_item.id.idlp()).unwrap(); + let list_item_found = self.list.get_elem(*leaf).unwrap(); + eq(list_item_found, list_item)?; + if let Some(pointed_by) = list_item_found.pointed_by { + let elem = self.elements.get(&pointed_by).unwrap(); + eq(elem.pos, list_item.id.idlp())?; + } + } + + for (elem_id, elem) in self.elements.iter() { + if let Some(item) = self.get_list_item_by_id(elem.pos) { + eq(item.pointed_by, Some(*elem_id))?; + } + } + + Ok(()) + } + + fn check_child_container_to_elem_consistency(&self) -> Result<(), ()> { + for (elem_id, elem) in self.elements.iter() { + if let LoroValue::Container(c) = &elem.value { + eq(self.child_container_to_elem.get(c), Some(elem_id))?; + } + } + + Ok(()) + } + + pub fn get_list_item_index(&self, id: IdLp, kind: IndexType) -> Option { + self.id_to_list_leaf + .get(&id) + .map(|leaf| self.get_index_of(*leaf, kind) as usize) + } + + pub fn get_index_of(&self, leaf: LeafIndex, kind: IndexType) -> i32 { + let mut ans = 0; + self.list + .visit_previous_caches(Cursor { leaf, offset: 0 }, |cache| match cache { + generic_btree::PreviousCache::NodeCache(c) => { + if matches!(kind, IndexType::ForUser) { + ans += c.user_len; + } else { + ans += c.include_dead_len; + } + } + generic_btree::PreviousCache::PrevSiblingElem(p) => { + if matches!(kind, IndexType::ForUser) { + ans += if p.pointed_by.is_some() { 1 } else { 0 }; + } else { + ans += 1; + } + } + generic_btree::PreviousCache::ThisElemAndOffset { .. } => {} + }); + ans + } + + #[inline] + pub fn get_list_item_at(&self, pos: usize, index_type: IndexType) -> Option<&ListItem> { + let index = match index_type { + IndexType::ForUser => self.list.query::(&pos)?, + IndexType::ForOp => self.list.query::(&pos)?, + }; + self.list.get_elem(index.leaf()) + } + + pub fn get_child_index(&self, id: &ContainerID, index_type: IndexType) -> Option { + let ans = self.child_container_to_elem.get(id).and_then(|eid| { + let this = &self; + let elem_id = eid.to_id().compact(); + let elem = this.elements.get(&elem_id)?; + if elem.value.as_container() != Some(id) { + // TODO: may be better to find a way clean up these invalid elements? + return None; + } + if let Some(leaf) = self.id_to_list_leaf.get(&elem.pos) { + let list_item = self.list.get_elem(*leaf)?; + assert_eq!(list_item.pointed_by, Some(elem_id)); + } else { + return None; + } + this.get_list_item_index(elem.pos, index_type) + }); + + ans + } + + pub fn contains_child_container(&self, id: &ContainerID) -> bool { + self.get_child_index(id, IndexType::ForUser).is_some() + } + + #[inline] + pub fn get_list_item_by_id(&self, id: IdLp) -> Option<&ListItem> { + self.id_to_list_leaf + .get(&id) + .map(|&leaf| self.list.get_elem(leaf).unwrap()) + } + + pub fn len_kind(&self, kind: IndexType) -> usize { + match kind { + IndexType::ForUser => self.list.root_cache().user_len as usize, + IndexType::ForOp => self.list.root_cache().include_dead_len as usize, + } + } + + /// Insert a new list item at the given position (op index). + /// Return the value if the insertion activated an element. + pub fn insert_list_item(&mut self, pos: usize, list_item_id: IdFull) { + let c = self + .list + .insert::( + &pos, + ListItem { + pointed_by: None, + id: list_item_id, + }, + ) + .0; + self.id_to_list_leaf.insert(list_item_id.idlp(), c.leaf); + } + + /// Drain the list items in the given range (op index). + /// + /// We also remove the elements that are pointed by the list items. + pub fn list_drain( + &mut self, + range: std::ops::Range, + mut on_elem_id: impl FnMut(CompactIdLp, &Element), + ) { + for item in Self::drain_by_query::(&mut self.list, range) { + self.id_to_list_leaf.remove(&item.id.idlp()); + if let Some(elem_id) = &item.pointed_by { + let elem = self.elements.get(elem_id).unwrap(); + on_elem_id(*elem_id, elem); + self.elements.remove(elem_id); + } + } + } + + fn drain_by_query>( + list: &mut BTree, + range: std::ops::Range, + ) -> generic_btree::iter::Drain<'_, MovableListTreeTrait> { + let start = list.query::(&range.start); + let end = list.query::(&range.end); + generic_btree::iter::Drain::new(list, start, end) + } + + /// Update the pos pointer of an element and the corresponding list items. + /// + /// The old list item will be removed if `remove_old` is true; Otherwise, + /// it will be updated with its `pointed_by` removed. + /// + #[must_use] + pub fn update_pos( + &mut self, + elem_id: CompactIdLp, + new_pos: IdLp, + remove_old: bool, + ) -> UpdateResultFromPosChange { + let mut old_item_id = None; + if let Some(element) = self.elements.get_mut(&elem_id) { + if element.pos != new_pos { + old_item_id = Some(element.pos); + element.pos = new_pos; + } else { + return UpdateResultFromPosChange { + activate_new_list_item: false, + new_list_item_leaf: None, + removed_old_list_item_leaf: None, + }; + } + } else { + self.elements.insert( + elem_id, + Element { + value: LoroValue::Null, + value_id: IdLp::NONE_ID, + pos: new_pos, + }, + ); + } + + let mut ans = UpdateResultFromPosChange { + activate_new_list_item: true, + new_list_item_leaf: None, + removed_old_list_item_leaf: None, + }; + if let Some(leaf) = self.id_to_list_leaf.get(&new_pos) { + ans.new_list_item_leaf = Some(*leaf); + self.list.update_leaf(*leaf, |list_item| { + debug_assert!( + list_item.pointed_by.is_none(), + "list_item was pointed by {:?} but need to be changed to {:?}", + list_item.pointed_by, + elem_id + ); + ans.activate_new_list_item = list_item.pointed_by.is_none(); + list_item.pointed_by = Some(elem_id); + (true, None, None) + }); + } else { + ans.activate_new_list_item = false; + } + + if let Some(old) = old_item_id { + if old.is_none() || old == new_pos { + return ans; + } + + if remove_old { + let leaf = self.id_to_list_leaf.remove(&old).unwrap(); + let elem = self.list.remove_leaf(Cursor { leaf, offset: 0 }).unwrap(); + assert_eq!(elem.pointed_by, Some(elem_id)); + } else if let Some(leaf) = self.id_to_list_leaf.get(&old) { + ans.removed_old_list_item_leaf = Some(*leaf); + let (still_valid, split) = self.list.update_leaf(*leaf, |item| { + item.pointed_by = None; + (true, None, None) + }); + assert!(still_valid); + assert!(split.arr.is_empty()); + } + } + + ans + } + + /// The update can reflected on an event exposed to users. This method calculates the + /// corresponding event position information. + pub fn convert_update_to_event_pos( + &self, + update: UpdateResultFromPosChange, + ) -> EventPosInfo { + if update.activate_new_list_item { + let new = update.new_list_item_leaf.unwrap(); + if let Some(del) = update.removed_old_list_item_leaf { + let mut insert_pos = self.get_index_of(new, IndexType::ForUser) as usize; + let del_pos = self.get_index_of(del, IndexType::ForUser) as usize; + if insert_pos > del_pos { + insert_pos += 1; + } + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: Some(insert_pos), + delete: Some(del_pos), + } + } else { + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: Some(self.get_index_of(new, IndexType::ForUser) as usize), + delete: None, + } + } + } else if let Some(del) = update.removed_old_list_item_leaf { + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: None, + delete: Some(self.get_index_of(del, IndexType::ForUser) as usize), + } + } else { + EventPosInfo { + activate_new_list_item: update.activate_new_list_item, + insert: None, + delete: None, + } + } + } + + pub fn update_value( + &mut self, + elem_id: CompactIdLp, + new_value: LoroValue, + value_id: IdLp, + ) -> Option { + debug_assert!(elem_id.peer != PeerID::MAX); + debug_assert!(!value_id.is_none()); + if let LoroValue::Container(c) = &new_value { + self.child_container_to_elem.insert(c.clone(), elem_id); + } + + if let Some(element) = self.elements.get_mut(&elem_id) { + if let LoroValue::Container(c) = &element.value { + if element.value != new_value { + self.child_container_to_elem.remove(c); + } + } + let old_value = std::mem::replace(&mut element.value, new_value); + element.value_id = value_id; + Some(old_value) + } else { + self.elements.insert( + elem_id, + Element { + value: new_value, + value_id, + pos: IdLp::NONE_ID, + }, + ); + None + } + } + + /// push a new elem into the list + pub fn push_inner(&mut self, list_item_id: IdFull, elem: Option) { + let pointed_by = elem.as_ref().map(|x| x.elem_id); + if let Some(elem) = elem { + if let LoroValue::Container(c) = &elem.value { + self.child_container_to_elem.insert(c.clone(), elem.elem_id); + } + debug_assert!(!elem.last_set_id.is_none()); + self.elements.insert( + elem.elem_id, + Element { + value: elem.value, + value_id: elem.last_set_id, + pos: list_item_id.idlp(), + }, + ); + } + let cursor = self.list.push(ListItem { + pointed_by, + id: list_item_id, + }); + self.id_to_list_leaf + .insert(list_item_id.idlp(), cursor.leaf); + } + } + + pub(super) struct PushElemInfo { + pub elem_id: CompactIdLp, + pub value: LoroValue, + pub last_set_id: IdLp, + } + + pub(super) struct UpdateResultFromPosChange { + /// Whether the list item is activated by this change. + /// If so, the length of the list will be increased by 1 from the users' perspective. + pub activate_new_list_item: bool, + pub new_list_item_leaf: Option, + /// Remove from the users' perspective. But they are still in the state. + pub removed_old_list_item_leaf: Option, + } + + /// This struct assumes we apply insert first and then delete. + pub(super) struct EventPosInfo { + pub activate_new_list_item: bool, + pub insert: Option, + pub delete: Option, + } +} + +impl MovableListState { + pub fn new(idx: ContainerIdx) -> Self { + Self { + idx, + inner: InnerState::new(), + } + } + + #[inline] + fn list(&self) -> &BTree { + self.inner.list() + } + + #[inline] + pub(crate) fn elements(&self) -> &FxHashMap { + self.inner.elements() + } + + /// Return whether the list item is activated by this change. + /// + /// If so, the length of the list will be increased by 1 from the users' perspective. + /// + /// It's false when the list item is already activated or deleted. + #[must_use] + fn create_new_elem( + &mut self, + elem_id: CompactIdLp, + new_pos: IdLp, + new_value: LoroValue, + value_id: IdLp, + ) -> UpdateResultFromPosChange { + self.inner.update_value(elem_id, new_value, value_id); + self.inner.update_pos(elem_id, new_pos, false) + } + + /// Return the values that are activated by the insertions of the list items, and their elem ids. + /// + /// The activation means there were elements that already points to the list items + /// inserted by this op. + fn list_insert_batch(&mut self, index: usize, items: impl Iterator) { + for (i, item) in items.enumerate() { + self.inner.insert_list_item(index + i, item); + } + } + + /// Get the length defined by op, where the length includes the ones that are not being pointed at (moved, invisible to users). + #[allow(unused)] + fn op_len(&self) -> usize { + self.inner.list().root_cache().include_dead_len as usize + } + + fn mov( + &mut self, + from_index: usize, + to_index: usize, + elem_id: IdLp, + new_pos_id: IdFull, + kind: IndexType, + ) { + // PERF: can be optimized by inlining try update + if cfg!(debug_assertions) { + let item = self.inner.get_list_item_at(from_index, kind).unwrap(); + assert_eq!(item.pointed_by, Some(elem_id.compact())); + } + + self.inner.insert_list_item( + if to_index > from_index { + to_index + 1 + } else { + to_index + }, + new_pos_id, + ); + let _ = self + .inner + .update_pos(elem_id.compact(), new_pos_id.idlp(), true); + } + + pub(crate) fn get(&self, index: usize, kind: IndexType) -> Option<&LoroValue> { + if index >= self.len() { + return None; + } + + let item = self.inner.get_list_item_at(index, kind)?; + let elem = item.pointed_by?; + self.inner.elements().get(&elem).map(|x| &x.value) + } + + pub(crate) fn get_elem_at_given_pos( + &self, + index: usize, + kind: IndexType, + ) -> Option<(CompactIdLp, &Element)> { + self.inner.get_list_item_at(index, kind).and_then(|x| { + x.pointed_by + .map(|pointed_by| (pointed_by, self.inner.elements().get(&pointed_by).unwrap())) + }) + } + + pub(crate) fn get_list_id_at(&self, index: usize, kind: IndexType) -> Option { + self.inner.get_list_item_at(index, kind).map(|x| x.id.id()) + } + + pub(crate) fn get_elem_id_at(&self, index: usize, kind: IndexType) -> Option { + self.inner + .get_list_item_at(index, kind) + .and_then(|x| x.pointed_by) + } + + pub(crate) fn convert_index( + &self, + index: usize, + from: IndexType, + to: IndexType, + ) -> Option { + let len = self.inner.len_kind(from); + if index == len { + return Some(self.inner.len_kind(to)); + } + + if index > len { + return None; + } + + let c = match from { + IndexType::ForOp => self.inner.list().query::(&index).unwrap(), + IndexType::ForUser => self.inner.list().query::(&index).unwrap(), + }; + + Some(self.inner.get_index_of(c.cursor.leaf, to) as usize) + } + + pub(crate) fn get_list_item(&self, id: IdLp) -> Option<&ListItem> { + self.inner.get_list_item_by_id(id) + } + + /// Get the user index of elem + /// + /// If we cannot find the list item in the list, we will return None. + fn get_index_of_elem(&self, elem_id: CompactIdLp) -> Option { + let elem = self.inner.elements().get(&elem_id)?; + self.inner.get_list_item_index(elem.pos, IndexType::ForUser) + } + + /// Debug check the consistency between the list and the elements + /// + /// We need to ensure that: + /// - Every element's pos is in the list, and it has a `pointed_by` value that points + /// back to the element + /// - Every list item's `pointed_by` value points to an element in the elements, and + /// the element has the correct `pos` value + #[cfg(any(debug_assertions, test))] + #[allow(unused)] + pub(crate) fn check_consistency(&self) { + for (id, elem) in self.inner.elements().iter() { + let item = self + .get_list_item(id.to_id()) + .expect("Elem's pos should be in the list"); + assert_eq!(item.pointed_by.unwrap(), *id); + } + + for item in self.inner.list().iter() { + if let Some(elem_id) = item.pointed_by { + let elem = self.inner.elements().get(&elem_id).unwrap(); + assert_eq!(elem.pos, item.id.idlp()); + } + } + } + + pub fn iter(&self) -> impl Iterator { + // PERF: can be optimized + (0..self.len()).map(move |i| self.get(i, IndexType::ForUser).unwrap()) + } + + pub fn iter_with_last_move_id_and_elem_id( + &self, + ) -> impl Iterator { + self.inner.list().iter().filter_map(|list_item| { + if let Some(elem_id) = list_item.pointed_by.as_ref() { + let elem = self.inner.elements().get(elem_id).unwrap(); + Some((list_item.id, *elem_id, &elem.value)) + } else { + None + } + }) + } + + pub fn len(&self) -> usize { + self.inner.list().root_cache().user_len as usize + } + + fn to_vec(&self) -> Vec { + self.iter().cloned().collect_vec() + } + + pub(crate) fn get_index_of_id(&self, id: ID) -> Option { + let mut user_index = 0; + for item in self.list().iter() { + if item.id.peer == id.peer && item.id.counter == id.counter { + return Some(user_index); + } + + user_index += if item.pointed_by.is_some() { 1 } else { 0 }; + } + None + } + + fn get_value_inner(&self) -> Vec { + let list = self + .inner + .list() + .iter_with_filter(|x| (x.user_len > 0, 0)) + .filter_map(|(_, item)| { + item.pointed_by + .map(|eid| self.elements()[&eid].value.clone()) + }) + .collect(); + list + } + + pub(crate) fn get_list_item_id_at(&self, pos: usize) -> Option { + let item = self.inner.get_list_item_at(pos, IndexType::ForUser); + item.map(|x| x.id) + } + + #[allow(unused)] + fn check_get_child_index_correctly(&mut self) { + let value = self.get_value(); + let list = value.into_list().unwrap(); + for (i, item) in list.iter().enumerate() { + if let LoroValue::Container(c) = item { + let child_index = self.get_child_index(c).expect("cannot find child index"); + assert_eq!(child_index.into_seq().unwrap(), i); + } + } + } + + pub fn get_creator_at(&self, pos: usize) -> Option { + self.inner + .get_list_item_at(pos, IndexType::ForUser) + .and_then(|x| x.pointed_by.map(|x| x.peer)) + } + + pub fn get_last_mover_at(&self, pos: usize) -> Option { + self.inner + .get_list_item_at(pos, IndexType::ForUser) + .map(|x| x.id.peer) + } + + pub fn get_last_editor_at(&self, pos: usize) -> Option { + self.inner + .get_list_item_at(pos, IndexType::ForUser) + .and_then(|x| { + x.pointed_by + .and_then(|x| self.inner.elements().get(&x).map(|x| x.value_id.peer)) + }) + } +} + +impl ContainerState for MovableListState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.list().is_empty() && self.elements().is_empty() + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + #[instrument(skip_all)] + fn apply_diff_and_convert( + &mut self, + diff: InternalDiff, + DiffApplyContext { doc, mode }: DiffApplyContext, + ) -> Diff { + let InternalDiff::MovableList(mut diff) = diff else { + unreachable!() + }; + + // let start_value = self.get_value(); + if cfg!(debug_assertions) { + self.inner.check_consistency(); + } + + let mut event: ListDiff = DeltaRope::new(); + let mut maybe_moved: FxHashMap = FxHashMap::default(); + let need_compare = matches!(mode, DiffMode::Import); + + { + // apply deletions and calculate `maybe_moved` + let mut index = 0; + for delta_item in diff.list.iter() { + match delta_item { + DeltaItem::Retain { + retain, + attributes: _, + } => { + index += retain; + } + DeltaItem::Insert { .. } => {} + DeltaItem::Delete { + delete, + attributes: _, + } => { + let mut user_index = self + .convert_index(index, IndexType::ForOp, IndexType::ForUser) + .unwrap(); + let user_index_end = self + .convert_index(index + delete, IndexType::ForOp, IndexType::ForUser) + .unwrap(); + event.compose( + &DeltaRopeBuilder::new() + .retain(user_index, Default::default()) + .delete(user_index_end - user_index) + .build(), + ); + self.inner + .list_drain(index..index + delete, |elem_id, elem| { + maybe_moved.insert(elem_id, (user_index, elem.value.clone())); + if !matches!(mode, DiffMode::Checkout) { + if let Some(new_elem) = diff.elements.get_mut(&elem_id) { + if new_elem.value_id.is_none() { + new_elem.value = elem.value.clone(); + new_elem.value_id = Some(elem.value_id); + new_elem.value_updated = false; + } + + if new_elem.pos.is_none() { + new_elem.pos = Some(elem.pos); + } + } + } + + user_index += 1; + }); + assert_eq!(user_index, user_index_end); + } + } + } + } + + { + // apply list item changes + let mut index = 0; + for delta_item in diff.list.into_iter() { + match delta_item { + DeltaItem::Retain { + retain, + attributes: _, + } => { + index += retain; + } + DeltaItem::Insert { + insert, + attributes: _, + } => { + let len = insert.len(); + self.list_insert_batch(index, insert.into_iter()); + index += len; + } + DeltaItem::Delete { .. } => {} + } + } + } + + { + let doc = &doc.upgrade().unwrap(); + // Apply element changes + // + // In this block, we need to handle the events generated from the following sources: + // + // - The change of elem's pos that activated the list item. This creates an insert event. + // - The change of elem's value. This create a deletion and an insert event. + // + // It doesn't need to worry about the deletion of the list item, because it's handled by the list diff. + + for (elem_id, delta_item) in diff.elements.into_iter() { + let crate::delta::ElementDelta { + pos, + value, + value_updated, + value_id, + } = delta_item; + // Element may be dropped after snapshot encoding, + // so we need to check which kind of update we need to do + + match self.inner.elements().get(&elem_id).cloned() { + Some(elem) => { + // Update value if needed + if let Some(value_id) = value_id { + if elem.value != value && (!need_compare || elem.value_id < value_id) { + maybe_moved.remove(&elem_id); + self.inner.update_value(elem_id, value.clone(), value_id); + let index = self.get_index_of_elem(elem_id); + if let Some(index) = index { + event.compose( + &DeltaRopeBuilder::new() + .retain(index, Default::default()) + .delete(1) + .insert( + ArrayVec::from([ValueOrHandler::from_value( + value, doc, + )]), + ListDeltaMeta { from_move: false }, + ) + .build(), + ) + } + } + } + + // Update pos if needed + if let Some(pos) = pos { + if elem.pos != pos && (!need_compare || elem.pos < pos) { + // don't need to update old list item, because it's handled by list diff already + let result = self.inner.update_pos(elem_id, pos, false); + let result = self.inner.convert_update_to_event_pos(result); + if let Some(new_index) = result.insert { + let new_value = + self.elements().get(&elem_id).unwrap().value.clone(); + let from_delete = if let Some((_elem_index, elem_old_value)) = + maybe_moved.remove(&elem_id) + { + elem_old_value == new_value + } else { + false + }; + let new_delta: ListDiff = DeltaRopeBuilder::new() + .retain(new_index, Default::default()) + .insert( + ArrayVec::from([ValueOrHandler::from_value( + new_value, doc, + )]), + ListDeltaMeta { + from_move: (result.delete.is_some() + && !value_updated) + || from_delete, + }, + ) + .build(); + event.compose(&new_delta); + } + if let Some(del_index) = result.delete { + event.compose( + &DeltaRopeBuilder::new() + .retain(del_index, Default::default()) + .delete(1) + .build(), + ); + } + if !result.activate_new_list_item { + // not matched list item found, remove directly + self.inner.remove_elem_by_id(&elem_id); + } + } + } + } + None => { + // Need to create new element + let result = self.create_new_elem( + elem_id, + pos.unwrap(), + value.clone(), + value_id.unwrap(), + ); + // Composing events + let result = self.inner.convert_update_to_event_pos(result); + // Create event for pos change and value change + if let Some(index) = result.insert { + let from_delete = if let Some((_elem_index, elem_old_value)) = + maybe_moved.remove(&elem_id) + { + elem_old_value == value + } else { + false + }; + event.compose( + &DeltaRopeBuilder::new() + .retain(index, Default::default()) + .insert( + ArrayVec::from([ValueOrHandler::from_value(value, doc)]), + ListDeltaMeta { + from_move: (result.delete.is_some() && !value_updated) + || from_delete, + }, + ) + .build(), + ) + } + if let Some(index) = result.delete { + event.compose( + &DeltaRopeBuilder::new() + .retain(index, Default::default()) + .delete(1) + .build(), + ); + } + if !result.activate_new_list_item { + // not matched list item found, remove directly + self.inner.remove_elem_by_id(&elem_id); + } + } + } + } + } + + { + // Remove redundant elements + + // We now know that the elements inside `maybe_moved` are actually deleted. + // So we can safely remove them. + let redundant = maybe_moved; + for (elem_id, _) in redundant.iter() { + self.inner.remove_elem_by_id(elem_id); + } + } + + // if cfg!(debug_assertions) { + // self.inner.check_consistency(); + // let mut end_value = start_value.clone(); + // end_value.apply_diff_shallow(&[Diff::List(event.clone())]); + // let cur_value = self.get_value(); + // assert_eq!( + // end_value, cur_value, + // "start_value={:#?} event={:#?} new_state={:#?} but the end_value={:#?}", + // start_value, event, cur_value, end_value + // ); + // self.check_get_child_index_correctly(); + // } + + Diff::List(event) + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + let _ = self.apply_diff_and_convert(diff, ctx); + Ok(()) + } + + #[instrument(skip_all)] + fn apply_local_op(&mut self, op: &RawOp, _: &Op) -> LoroResult { + let mut ans: ApplyLocalOpReturn = Default::default(); + match op.content.as_list().unwrap() { + ListOp::Insert { slice, pos } => match slice { + ListSlice::RawData(list) => { + for (i, x) in list.as_ref().iter().enumerate() { + let elem_id = op.idlp().inc(i as i32).try_into().unwrap(); + let pos_id = op.id_full().inc(i as i32); + self.inner.insert_list_item(*pos + i, pos_id); + let _ = self.create_new_elem( + elem_id, + pos_id.idlp(), + x.clone(), + elem_id.to_id(), + ); + } + } + _ => unreachable!(), + }, + ListOp::Delete(span) => { + self.inner + .list_drain(span.start() as usize..span.end() as usize, |_, elem| { + if let Some(c) = elem.value.as_container() { + ans.deleted_containers.push(c.clone()); + } + }); + } + ListOp::Move { from, to, elem_id } => { + self.mov( + *from as usize, + *to as usize, + *elem_id, + op.id_full(), + IndexType::ForOp, + ); + } + ListOp::Set { elem_id, value } => { + let old_value = + self.inner + .update_value(elem_id.compact(), value.clone(), op.idlp()); + if let Some(LoroValue::Container(c)) = old_value { + ans.deleted_containers.push(c); + } + } + ListOp::StyleStart { .. } | ListOp::StyleEnd => unreachable!(), + } + + Ok(ans) + } + + fn to_diff(&mut self, doc: &Weak) -> Diff { + let doc = &doc.upgrade().unwrap(); + Diff::List( + DeltaRopeBuilder::new() + .insert_many( + self.to_vec() + .into_iter() + .map(|v| ValueOrHandler::from_value(v, doc)), + Default::default(), + ) + .build(), + ) + } + + fn get_value(&mut self) -> LoroValue { + let list = self.get_value_inner(); + LoroValue::List(list.into()) + } + + /// Get the index of the child container + fn get_child_index(&self, id: &ContainerID) -> Option { + self.inner + .get_child_index(id, IndexType::ForUser) + .map(Index::Seq) + } + + fn contains_child(&self, id: &ContainerID) -> bool { + self.inner.contains_child_container(id) + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + self.inner + .child_container_to_elem() + .iter() + .filter_map(|(c, elem_id)| { + let elem = self.elements().get(elem_id)?; + if elem.value.as_container() != Some(c) { + None + } else { + Some(c.clone()) + } + }) + .collect_vec() + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +struct EncodedId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + lamport: u32, +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone, Copy)] +struct EncodedItemForFastSnapshot { + #[columnar(strategy = "DeltaRle")] + invisible_list_item: usize, + #[columnar(strategy = "BoolRle")] + pos_id_eq_elem_id: bool, + #[columnar(strategy = "BoolRle")] + elem_id_eq_last_set_id: bool, +} + +#[columnar(vec, ser, de, iterable)] +#[derive(Debug, Clone)] +struct EncodedIdFull { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + lamport_sub_counter: i32, +} + +#[columnar(ser, de)] +struct EncodedFastSnapshot { + #[columnar(class = "vec", iter = "EncodedItemForFastSnapshot")] + items: Vec, + #[columnar(class = "vec", iter = "EncodedIdFull")] + list_item_ids: Vec, + #[columnar(class = "vec", iter = "EncodedId")] + elem_ids: Vec, + #[columnar(class = "vec", iter = "EncodedId")] + last_set_ids: Vec, +} + +mod snapshot { + use loro_common::{IdFull, IdLp, LoroValue, PeerID}; + + use crate::{ + encoding::value_register::ValueRegister, + state::{ + decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, ContainerCreationContext, ContainerState, FastStateSnapshot, + }, + }; + + use super::{ + inner::PushElemInfo, EncodedFastSnapshot, EncodedId, EncodedIdFull, + EncodedItemForFastSnapshot, MovableListState, + }; + + impl FastStateSnapshot for MovableListState { + /// Encodes the MovableListState into a compact binary format for fast snapshot storage and retrieval. + /// + /// The encoding format consists of: + /// 1. The full value of the MovableListState, encoded using postcard serialization. + /// 2. A series of EncodedItemForFastSnapshot structs representing each visible list item: + /// - invisible_list_item: Count of invisible items before this item (RLE encoded) + /// - pos_id_eq_elem_id: Boolean indicating if position ID equals element ID (RLE encoded) + /// - elem_id_eq_last_set_id: Boolean indicating if element ID equals last set ID (RLE encoded) + /// 3. A series of EncodedIdFull structs for list item IDs: + /// - peer_idx: Index of the peer ID in a value register (delta-RLE encoded) + /// - counter: Operation counter (delta-RLE encoded) + /// - lamport: Lamport timestamp (delta-RLE encoded) + /// 4. EncodedId structs for element IDs (when different from position ID) + /// 5. EncodedId structs for last set IDs (when different from element ID) + /// 6. A list of unique peer IDs used in the encoding + fn encode_snapshot_fast(&mut self, mut w: W) { + let value = self.get_value().into_list().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + let mut peers: ValueRegister = ValueRegister::new(); + let len = self.len(); + let mut items = Vec::with_capacity(len); + // starts with a sentinel value. The num of `invisible_list_item` may be updated later + items.push(EncodedItemForFastSnapshot { + pos_id_eq_elem_id: true, + invisible_list_item: 0, + elem_id_eq_last_set_id: true, + }); + + let mut list_item_ids = Vec::with_capacity(self.len()); + let mut elem_ids = Vec::new(); + let mut last_set_ids = Vec::new(); + for item in self.list().iter() { + if let Some(elem_id) = item.pointed_by { + let elem = self.elements().get(&elem_id).unwrap(); + let elem_eq_list_item = elem_id.to_id() == item.id.idlp(); + let elem_id_eq_last_set_id = elem.value_id.compact() == elem_id; + items.push(EncodedItemForFastSnapshot { + invisible_list_item: 0, + pos_id_eq_elem_id: elem_eq_list_item, + elem_id_eq_last_set_id, + }); + + list_item_ids.push(super::EncodedIdFull { + peer_idx: peers.register(&item.id.peer), + counter: item.id.counter, + lamport_sub_counter: (item.id.lamport as i32 - item.id.counter), + }); + if !elem_eq_list_item { + elem_ids.push(super::EncodedId { + peer_idx: peers.register(&elem_id.peer), + lamport: elem_id.lamport.get(), + }) + } + if !elem_id_eq_last_set_id { + last_set_ids.push(super::EncodedId { + peer_idx: peers.register(&elem.value_id.peer), + lamport: elem.value_id.lamport, + }) + } + } else { + items.last_mut().unwrap().invisible_list_item += 1; + list_item_ids.push(super::EncodedIdFull { + peer_idx: peers.register(&item.id.peer), + counter: item.id.counter, + lamport_sub_counter: (item.id.lamport as i32 - item.id.counter), + }); + } + } + + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let v = serde_columnar::to_vec(&EncodedFastSnapshot { + items, + list_item_ids, + elem_ids, + last_set_ids, + }) + .unwrap(); + w.write_all(&v).unwrap(); + } + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + let (list_value, bytes) = + postcard::take_from_bytes::>(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode list value failed".to_string().into_boxed_str(), + ) + })?; + Ok((list_value.into(), bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (list_value, mut bytes): (loro_common::LoroValue, &[u8]), + _ctx: ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let peers = decode_peer_table(&mut bytes, "Decode movable list state failed")?; + + let mut ans = MovableListState::new(idx); + + let iters = + serde_columnar::iter_from_bytes::(bytes).map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid metadata: {err}" + )) + })?; + let mut elem_iter = iters.elem_ids; + let item_iter = iters.items; + let mut list_item_id_iter = iters.list_item_ids; + let mut last_set_id_iter = iters.last_set_ids; + let mut is_first = true; + + let list_value = list_value.into_list().map_err(|_| { + state_decode_error("Decode movable list state failed: value is not a list") + })?; + let mut list_value_iter = list_value.iter(); + for item in item_iter { + let EncodedItemForFastSnapshot { + invisible_list_item, + pos_id_eq_elem_id, + elem_id_eq_last_set_id, + } = item.map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid item: {err}" + )) + })?; + + if !is_first { + let EncodedIdFull { + peer_idx, + counter, + lamport_sub_counter, + } = list_item_id_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing list item id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid list item id: {err}" + )) + })?; + let peer = decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?; + let id_full = IdFull::new( + peer, + counter, + decode_lamport_from_delta( + counter, + lamport_sub_counter, + "Decode movable list state failed", + )?, + ); + let elem_id = if pos_id_eq_elem_id { + id_full.idlp() + } else { + let EncodedId { peer_idx, lamport } = elem_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing element id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid element id: {err}" + )) + })?; + IdLp::new( + decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?, + lamport, + ) + }; + + let last_set_id = if elem_id_eq_last_set_id { + elem_id + } else { + let EncodedId { peer_idx, lamport } = last_set_id_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing last set id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid last set id: {err}" + )) + })?; + IdLp::new( + decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?, + lamport, + ) + }; + + let value = list_value_iter.next().ok_or_else(|| { + state_decode_error("Decode movable list state failed: missing list value") + })?; + ans.inner.push_inner( + id_full, + Some(PushElemInfo { + elem_id: elem_id.compact(), + value: value.clone(), + last_set_id, + }), + ) + } + + is_first = false; + for _ in 0..invisible_list_item { + let EncodedIdFull { + peer_idx, + counter, + lamport_sub_counter, + } = list_item_id_iter + .next() + .ok_or_else(|| { + state_decode_error( + "Decode movable list state failed: missing invisible list item id", + ) + })? + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid invisible list item id: {err}" + )) + })?; + let peer = decode_peer_from_table( + &peers, + peer_idx, + "Decode movable list state failed", + )?; + let id_full = IdFull::new( + peer, + counter, + decode_lamport_from_delta( + counter, + lamport_sub_counter, + "Decode movable list state failed", + )?, + ); + ans.inner.push_inner(id_full, None); + } + } + + if is_first { + return Err(state_decode_error( + "Decode movable list state failed: missing sentinel item", + )); + } + if elem_iter + .next() + .transpose() + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid extra element id: {err}" + )) + })? + .is_some() + { + return Err(state_decode_error( + "Decode movable list state failed: unused element id", + )); + } + if list_item_id_iter + .next() + .transpose() + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid extra list item id: {err}" + )) + })? + .is_some() + { + return Err(state_decode_error( + "Decode movable list state failed: unused list item id", + )); + } + if last_set_id_iter + .next() + .transpose() + .map_err(|err| { + state_decode_error(format!( + "Decode movable list state failed: invalid extra last set id: {err}" + )) + })? + .is_some() + { + return Err(state_decode_error( + "Decode movable list state failed: unused last set id", + )); + } + if list_value_iter.next().is_some() { + return Err(state_decode_error( + "Decode movable list state failed: unused list value", + )); + } + + Ok(ans) + } + } + + #[cfg(test)] + mod test { + + use loro_common::{CompactIdLp, ContainerID, LoroValue, ID}; + + use crate::container::idx::ContainerIdx; + + use super::*; + + #[test] + fn test_movable_list_snapshot() { + let mut list = MovableListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::MovableList, + )); + + list.inner.insert_list_item(0, IdFull::new(9, 9, 9)); + list.inner.insert_list_item(1, IdFull::new(0, 0, 0)); + let _ = list.create_new_elem( + CompactIdLp::new(10, 10), + IdLp { + lamport: 0, + peer: 0, + }, + LoroValue::Container(ContainerID::new_normal( + ID::new(10, 10), + loro_common::ContainerType::Text, + )), + IdLp { + lamport: 1, + peer: 2, + }, + ); + list.inner.insert_list_item(2, IdFull::new(1, 1, 1)); + list.inner.insert_list_item(3, IdFull::new(2, 2, 2)); + list.inner.insert_list_item(4, IdFull::new(3, 3, 3)); + let _ = list.create_new_elem( + CompactIdLp::new(3, 8), + IdLp { + lamport: 3, + peer: 3, + }, + LoroValue::String("abc".into()), + IdLp { + lamport: 4, + peer: 5, + }, + ); + + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 117, "{}", bytes.len()); + + let (v, bytes) = MovableListState::decode_value(&bytes).unwrap(); + assert_eq!( + v, + vec![ + LoroValue::Container(ContainerID::new_normal( + ID::new(10, 10), + loro_common::ContainerType::Text, + )), + LoroValue::String("abc".into()), + ] + .into() + ); + let mut list2 = MovableListState::decode_snapshot_fast( + ContainerIdx::from_index_and_type(0, loro_common::ContainerType::MovableList), + (v.clone(), bytes), + ContainerCreationContext { + configure: &Default::default(), + peer: 0, + }, + ) + .unwrap(); + assert_eq!(&list2.get_value(), &v); + assert_eq!(&list.inner, &list2.inner); + } + + #[test] + fn test_movable_list_snapshot_size() { + let mut list = MovableListState::new(ContainerIdx::from_index_and_type( + 0, + loro_common::ContainerType::MovableList, + )); + + list.inner.insert_list_item(0, IdFull::new(0, 0, 0)); + let _ = list.create_new_elem( + CompactIdLp::new(0, 0), + IdLp { + lamport: 0, + peer: 0, + }, + LoroValue::I64(0), + IdLp { + lamport: 0, + peer: 0, + }, + ); + + list.inner.insert_list_item(1, IdFull::new(0, 1, 1)); + let _ = list.create_new_elem( + CompactIdLp::new(0, 1), + IdLp { + peer: 0, + lamport: 1, + }, + LoroValue::I64(0), + IdLp { + peer: 0, + lamport: 1, + }, + ); + + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 42, "{}", bytes.len()); + + list.inner.insert_list_item(2, IdFull::new(0, 1, 2)); + let _ = list.create_new_elem( + CompactIdLp::new(0, 2), + IdLp { + peer: 0, + lamport: 2, + }, + LoroValue::I64(0), + IdLp { + peer: 0, + lamport: 2, + }, + ); + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + assert!(bytes.len() <= 47, "{}", bytes.len()); + } + + #[test] + fn movable_list_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, loro_common::ContainerType::MovableList); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + assert!(MovableListState::decode_snapshot_fast( + idx, + (Vec::::new().into(), &[1]), + ctx, + ) + .is_err()); + + let mut list = MovableListState::new(idx); + let mut bytes = Vec::new(); + list.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = MovableListState::decode_value(&bytes).unwrap(); + assert!(MovableListState::decode_snapshot_fast( + idx, + (vec![LoroValue::I64(1)].into(), state_bytes), + ctx, + ) + .is_err()); + } + } +} + +#[cfg(test)] +mod test { + use crate::{loro::ExportMode, HandlerTrait, LoroDoc, ToJson}; + use serde_json::json; + + #[test] + fn basic_handler_ops() { + let doc = LoroDoc::new_auto_commit(); + let list = doc.get_movable_list("list"); + list.insert(0, 0).unwrap(); + list.insert(1, 1).unwrap(); + list.insert(2, 2).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([0, 1, 2])); + list.mov(0, 1).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([1, 0, 2])); + list.mov(2, 0).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([2, 1, 0])); + list.delete(0, 2).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([0])); + list.insert(0, 9).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([9, 0])); + list.delete(0, 2).unwrap(); + assert_eq!(list.get_value().to_json_value(), json!([])); + } + + #[test] + fn basic_sync() { + let doc = LoroDoc::new_auto_commit(); + let list = doc.get_movable_list("list"); + list.insert(0, 1).unwrap(); + list.insert(1, 0).unwrap(); + list.mov(0, 1).unwrap(); + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [0, 1] + }) + ); + } + list.mov(1, 0).unwrap(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ + "list": [1, 0] + }) + ); + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [1, 0] + }) + ); + } + list.mov(0, 1).unwrap(); + list.insert(2, 3).unwrap(); + list.set(2, 2).unwrap(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ + "list": [0, 1, 2] + }) + ); + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [0, 1, 2] + }) + ); + } + { + let doc_b = LoroDoc::new_auto_commit(); + doc_b + .import(&doc.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + assert_eq!( + doc_b.get_deep_value().to_json_value(), + json!({ + "list": [0, 1, 2] + }) + ); + } + } +} diff --git a/vendor/loro-internal/src/state/richtext_state.rs b/vendor/loro-internal/src/state/richtext_state.rs new file mode 100644 index 00000000..ac25d483 --- /dev/null +++ b/vendor/loro-internal/src/state/richtext_state.rs @@ -0,0 +1,1825 @@ +use generic_btree::{rle::HasLength, rle::Sliceable as _, Cursor}; +use loro_common::{ContainerID, Counter, InternalString, LoroError, LoroResult, LoroValue, PeerID, ID}; +use loro_delta::{DeltaRope, DeltaRopeBuilder}; +use rustc_hash::{FxHashMap, FxHashSet}; +use smallvec::SmallVec; +use std::ops::Range; +use std::sync::{Arc, Weak}; + +use crate::{ + container::{ + idx::ContainerIdx, + list::list_op, + richtext::{ + config::StyleConfigMap, + richtext_state::{ + DrainInfo, EntityRangeInfo, IterRangeItem, PosType, RichtextStateChunk, + }, + AnchorType, RichtextState as InnerState, StyleKey, StyleOp, Styles, + }, + }, + delta::{StyleMeta, StyleMetaItem}, + event::{Diff, Index, InternalDiff, TextDiff}, + handler::TextDelta, + op::{Op, RawOp}, + sync::RwLock, + utils::{lazy::LazyLoad, string_slice::StringSlice}, + LoroDocInner, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +/// §perf-heaven T1: whether the `Src`-loader richtext-value fast path is active +/// (default ON; set `FLOWSTATE_RICHTEXT_NO_FASTPATH` to fall back to the full +/// `Src -> Dst` build). Read once and cached. +fn richtext_src_fastpath_enabled() -> bool { + static ENABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *ENABLED.get_or_init(|| std::env::var_os("FLOWSTATE_RICHTEXT_NO_FASTPATH").is_none()) +} + +/// §perf-heaven T1: whether to ALSO build the full state and assert the fast +/// path matches it (the fuzz-guided recovery guard). On in debug builds — where +/// the convergence/intent fuzz runs — and in any build via +/// `FLOWSTATE_RICHTEXT_VERIFY`, so `heaven.sh` can validate over the real corpus +/// in release. When on, `get_richtext_value` returns the BUILT value, so debug +/// behaviour is identical to before the patch regardless of the fast path. +fn richtext_verify_enabled() -> bool { + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + *VERIFY.get_or_init(|| cfg!(debug_assertions) || std::env::var_os("FLOWSTATE_RICHTEXT_VERIFY").is_some()) +} + +#[derive(Debug)] +pub struct RichtextState { + idx: ContainerIdx, + config: Arc>, + state: LazyLoad, + /// This is used to indicate whether the richtext state is changed, so the downstream has an easy way to cache + /// NOTE: We need to ensure the invariance that the version id is always increased when the richtext state is changed + version_id: usize, +} + +struct Pos { + entity_index: usize, + event_index: usize, +} + +impl RichtextState { + #[inline] + pub fn new(idx: ContainerIdx, config: Arc>) -> Self { + Self { + idx, + config, + state: LazyLoad::Src(Default::default()), + version_id: 0, + } + } + + #[inline] + fn update_version(&mut self) { + self.version_id = self.version_id.wrapping_add(1); + } + + /// Get the version id of the richtext + /// + /// This can be used to detect whether the richtext is changed + #[inline] + pub fn get_version_id(&self) -> usize { + self.version_id + } + + /// Get the text content of the richtext + /// + /// This uses `mut` because we may need to build the state from snapshot + #[inline] + pub fn to_string_mut(&mut self) -> String { + self.state.get_mut().to_string() + } + + #[allow(unused)] + #[inline(always)] + pub(crate) fn is_empty(&self) -> bool { + match &self.state { + LazyLoad::Src(s) => s.elements.is_empty(), + LazyLoad::Dst(d) => d.is_empty(), + } + } + + pub(crate) fn diagnose(&self) { + match &self.state { + LazyLoad::Src(_) => {} + LazyLoad::Dst(d) => d.diagnose(), + } + } + + #[allow(unused)] + pub(crate) fn get_text_slice_by_event_index( + &mut self, + pos: usize, + len: usize, + ) -> LoroResult { + self.state.get_mut().get_text_slice_by_event_index(pos, len) + } + + pub(crate) fn slice_delta( + &mut self, + start_index: usize, + end_index: usize, + pos_type: PosType, + ) -> LoroResult> { + self.state + .get_mut() + .slice_delta(start_index, end_index, pos_type) + } + + #[allow(unused)] + pub(crate) fn get_char_by_event_index(&mut self, pos: usize) -> Result { + self.state.get_mut().get_char_by_event_index(pos) + } + + /// §perf-heaven T1/T7.3: serve the char straight from the Src loader when + /// the index space maps to unicode chunks, so object-anchor validation and + /// event-index lookups don't force the `Src -> Dst` build. On non-wasm the + /// EVENT index IS the unicode index, so `Event` reuses the same net-guarded + /// walker as `Unicode` (guarded by the corpus Src-path equivalence net, + /// T7.26). `Utf16` and wasm `Event` still fall through to the built state: + /// there is no net exercising a utf16 chunk walk, so adding one would be + /// unguarded dead code (the T1 trap). + pub(crate) fn char_at_pos(&mut self, pos: usize, pos_type: PosType) -> Result { + if let LazyLoad::Src(loader) = &self.state { + let char_at = match pos_type { + PosType::Unicode => Some(loader.char_at_unicode(pos)), + PosType::Event => Some(if cfg!(feature = "wasm") { + loader.char_at_utf16(pos) + } else { + loader.char_at_unicode(pos) + }), + PosType::Utf16 => Some(loader.char_at_utf16(pos)), + _ => None, + }; + if let Some(char_at) = char_at { + return char_at.ok_or(()); + } + } + let event_pos = match pos_type { + PosType::Event => pos, + _ => self.index_to_event_index(pos, pos_type), + }; + self.get_char_by_event_index(event_pos) + } + + pub(crate) fn iter(&mut self, mut callback: impl FnMut(&str) -> bool) { + for span in self.state.get_mut().iter_chunk() { + match span { + RichtextStateChunk::Text(text_chunk) => { + if !callback(text_chunk.as_str()) { + return; + } + } + RichtextStateChunk::Style { .. } => {} + } + } + } + + pub(crate) fn iter_raw(&self, callback: &mut dyn FnMut(&RichtextStateChunk)) { + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + for c in iter { + callback(c); + } + } + + fn get_style_start( + &mut self, + style_starts: &mut FxHashMap, Pos>, + style: &Arc, + ) -> Pos { + self.update_version(); + match style_starts.remove(style) { + Some(x) => x, + None => { + // this should happen rarely, so it should be fine to scan + let mut pos = Pos { + entity_index: 0, + event_index: 0, + }; + + for c in self.state.get_mut().iter_chunk() { + match c { + RichtextStateChunk::Style { + style: s, + anchor_type: AnchorType::Start, + } if style == s => { + break; + } + RichtextStateChunk::Text(t) => { + pos.entity_index += t.unicode_len() as usize; + pos.event_index += t.event_len() as usize; + } + RichtextStateChunk::Style { .. } => { + pos.entity_index += 1; + } + } + } + pos + } + } + } + + pub fn get_index_of_id(&self, id: ID) -> Option { + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut index = 0; + for elem in iter { + let span = elem.get_id_span(); + if span.contains(id) { + return Some(index + (id.counter - span.counter.start) as usize); + } + + index += elem.rle_len(); + } + + None + } + + pub fn get_text_index_of_id(&self, id: ID, use_event_index: bool) -> Option { + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut index = 0; + for elem in iter { + let span = elem.get_id_span(); + if span.contains(id) { + match elem { + RichtextStateChunk::Text(t) => { + if use_event_index { + let event_offset = t.convert_unicode_offset_to_event_offset( + (id.counter - span.counter.start) as usize, + ); + return Some(index + event_offset); + } else { + return Some(index + (id.counter - span.counter.start) as usize); + } + } + RichtextStateChunk::Style { .. } => { + return Some(index); + } + } + } + + index += match elem { + RichtextStateChunk::Text(t) => { + if use_event_index { + t.event_len() as usize + } else { + t.unicode_len() as usize + } + } + RichtextStateChunk::Style { .. } => 0, + }; + } + + None + } + + /// §perf (flowstate vendor patch): batch variant of [`Self::get_text_index_of_id`]. + /// Resolves the positions of many ids in a SINGLE pass over the chunks instead + /// of O(elements) per id, so a caller resolving one id per record (e.g. a + /// document projection resolving every paragraph boundary) becomes + /// O(elements + ids·log ids) rather than O(elements · ids). `out[i]` is the + /// position of `ids[i]`, or `None` if that id is not present in the current + /// state (deleted) — callers that need history-traced resolution for those + /// should fall back to the per-id `get_cursor_pos`. Each resolved entry is + /// bit-identical to calling [`Self::get_text_index_of_id`] on that id. + pub fn get_text_indices_of_ids(&self, ids: &[ID], use_event_index: bool) -> Vec> { + let mut out = vec![None; ids.len()]; + if ids.is_empty() { + return out; + } + // Group query slots by peer, each sorted by counter, so a chunk's + // contiguous (forward) id-span selects its matching queries by binary + // search — keeping the whole resolution to a single chunk pass even when + // every id shares one peer. + let mut by_peer: FxHashMap> = FxHashMap::default(); + for (i, id) in ids.iter().enumerate() { + by_peer.entry(id.peer).or_default().push((id.counter, i)); + } + for slots in by_peer.values_mut() { + slots.sort_unstable_by_key(|(counter, _)| *counter); + } + let mut remaining = ids.len(); + + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut index = 0; + for elem in iter { + if remaining == 0 { + break; + } + let span = elem.get_id_span(); + if let Some(slots) = by_peer.get(&span.peer) { + // Chunk id-spans are always forward (start < end), matching the + // `span.contains(id)` window `start <= counter < end` used by the + // per-id path above. Select that counter window by binary search. + let start = span.counter.start; + let end = span.counter.end; + let lo = slots.partition_point(|(counter, _)| *counter < start); + for &(counter, slot) in &slots[lo..] { + if counter >= end { + break; + } + if out[slot].is_none() { + let pos = match elem { + RichtextStateChunk::Text(t) => { + if use_event_index { + index + t.convert_unicode_offset_to_event_offset((counter - start) as usize) + } else { + index + (counter - start) as usize + } + } + RichtextStateChunk::Style { .. } => index, + }; + out[slot] = Some(pos); + remaining -= 1; + } + } + } + index += match elem { + RichtextStateChunk::Text(t) => { + if use_event_index { + t.event_len() as usize + } else { + t.unicode_len() as usize + } + } + RichtextStateChunk::Style { .. } => 0, + }; + } + + out + } + + pub(crate) fn get_delta(&mut self) -> Vec { + let mut delta = Vec::new(); + for span in self.state.get_mut().iter() { + let next_attr = span.attributes.to_option_map(); + match delta.last_mut() { + Some(TextDelta::Insert { insert, attributes }) if &next_attr == attributes => { + insert.push_str(span.text.as_str()); + continue; + } + _ => {} + } + delta.push(TextDelta::Insert { + insert: span.text.as_str().to_string(), + attributes: next_attr, + }) + } + delta + } +} + +#[cfg(test)] +mod tests { + use crate::{container::richtext::StyleKey, cursor::PosType, handler::HandlerTrait, LoroDoc}; + + #[test] + fn has_style_key_in_entity_range_basic() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "abcdef", PosType::Unicode).unwrap(); + text.mark(1, 3, "bold", true.into(), PosType::Unicode) + .unwrap(); + + let bold_key = StyleKey::Key("bold".into()); + let has_style = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(1..3, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(has_style); + + let missing = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(4..5, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(!missing); + } + + #[test] + fn has_style_key_in_entity_range_spans_elements() { + let loro = LoroDoc::new_auto_commit(); + let text = loro.get_text("text"); + text.insert(0, "abcdefgh", PosType::Unicode).unwrap(); + text.mark(0, 2, "bold", true.into(), PosType::Unicode) + .unwrap(); + text.mark(3, 5, "bold", true.into(), PosType::Unicode) + .unwrap(); + + let bold_key = StyleKey::Key("bold".into()); + + let has_style_across_segments = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(0..5, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(has_style_across_segments); + + let gap_has_style = text + .with_state(|state| { + let st = state.as_richtext_state_mut().unwrap(); + let (entity_range, _) = + st.get_entity_range_and_styles_at_range(6..7, PosType::Unicode); + Ok(st.has_style_key_in_entity_range(entity_range, &bold_key)) + }) + .unwrap(); + assert!(!gap_has_style); + } +} + +impl Clone for RichtextState { + fn clone(&self) -> Self { + Self { + idx: self.idx, + config: self.config.clone(), + state: self.state.clone(), + version_id: self.version_id, + } + } +} + +impl ContainerState for RichtextState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + match &self.state { + LazyLoad::Src(s) => s.is_empty(), + LazyLoad::Dst(s) => s.is_empty(), + } + } + + // TODO: refactor + fn apply_diff_and_convert(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> Diff { + self.update_version(); + let InternalDiff::RichtextRaw(richtext) = diff else { + unreachable!() + }; + + // tracing::info!("Self state = {:#?}", &self); + // PERF: compose delta + let mut ans: TextDiff = TextDiff::new(); + let mut style_delta: TextDiff = TextDiff::new(); + let mut style_starts: FxHashMap, Pos> = FxHashMap::default(); + let mut entity_index = 0; + let mut event_index = 0; + let mut new_style_deltas: Vec = Vec::new(); + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + entity_index += len; + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + // Deletions + let mut deleted_style_keys: FxHashSet = + FxHashSet::default(); + let DrainInfo { + start_event_index: start, + end_event_index: end, + affected_style_range, + } = self.state.get_mut().drain_by_entity_index( + entity_index, + *delete, + Some(&mut |c| match c { + RichtextStateChunk::Style { + style, + anchor_type: AnchorType::Start, + } => { + deleted_style_keys.insert(style.key.clone()); + } + RichtextStateChunk::Style { + style, + anchor_type: AnchorType::End, + } => { + deleted_style_keys.insert(style.key.clone()); + } + _ => {} + }), + ); + + if start > event_index { + ans.push_retain(start - event_index, Default::default()); + event_index = start; + } + + if let Some((entity_range, event_range)) = affected_style_range { + let mut delta: TextDiff = DeltaRopeBuilder::new() + .retain(event_range.start, Default::default()) + .build(); + let mut entity_len_sum = 0; + let expected_sum = entity_range.len(); + + for IterRangeItem { + event_len, + chunk, + styles, + entity_len, + .. + } in self.state.get_mut().iter_range(entity_range) + { + entity_len_sum += entity_len; + match chunk { + RichtextStateChunk::Text(_) => { + let mut style_meta: StyleMeta = styles.into(); + for key in deleted_style_keys.iter() { + if !style_meta.contains_key(key) { + style_meta.insert( + key.clone(), + StyleMetaItem { + lamport: 0, + peer: 0, + value: LoroValue::Null, + }, + ) + } + } + delta.push_retain( + event_len, + style_meta.to_option_map().unwrap_or_default().into(), + ); + } + RichtextStateChunk::Style { .. } => {} + } + } + + debug_assert_eq!(entity_len_sum, expected_sum); + delta.chop(); + style_delta.compose(&delta); + } + + ans.push_delete(end - start); + } + + if value.rle_len() > 0 { + // Insertions + match value { + RichtextStateChunk::Text(s) => { + let (pos, styles) = + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Text(s.clone()), + ); + // PERF: this can be optimized + let insert_styles = Into::::into(styles.clone()) + .to_option_map() + .unwrap_or_default(); + + if pos > event_index { + ans.push_retain(pos - event_index, Default::default()); + } + event_index = pos + s.event_len() as usize; + ans.push_insert( + StringSlice::from(s.bytes().clone()), + insert_styles.into(), + ); + } + RichtextStateChunk::Style { anchor_type, style } => { + let (new_event_index, _) = + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Style { + style: style.clone(), + anchor_type: *anchor_type, + }, + ); + + if new_event_index > event_index { + ans.push_retain( + new_event_index - event_index, + Default::default(), + ); + // inserting style anchor will not affect event_index's positions + event_index = new_event_index; + } + + match anchor_type { + AnchorType::Start => { + style_starts.insert( + style.clone(), + Pos { + entity_index, + event_index: new_event_index, + }, + ); + } + AnchorType::End => { + // get the pair of style anchor. now we can annotate the range + let Pos { + entity_index: start_entity_index, + event_index: start_event_index, + } = self.get_style_start(&mut style_starts, style); + let mut delta: TextDiff = DeltaRopeBuilder::new() + .retain(start_event_index, Default::default()) + .build(); + // we need to + 1 because we also need to annotate the end anchor + let event = + self.state.get_mut().annotate_style_range_with_event( + start_entity_index..entity_index + 1, + style.clone(), + ); + for (s, l) in event { + delta.push_retain( + l, + s.to_option_map().unwrap_or_default().into(), + ); + } + + delta.chop(); + new_style_deltas.push(delta); + } + } + } + } + + entity_index += value.rle_len(); + } + } + } + } + + for s in new_style_deltas { + style_delta.compose(&s); + } + // self.check_consistency_between_content_and_style_ranges(); + ans.compose(&style_delta); + Diff::Text(ans) + } + + fn apply_diff(&mut self, diff: InternalDiff, _ctx: DiffApplyContext) -> LoroResult<()> { + self.update_version(); + let InternalDiff::RichtextRaw(richtext) = diff else { + unreachable!() + }; + + if let LazyLoad::Src(loader) = &mut self.state { + if loader.try_apply_append_delta(&richtext) { + return Ok(()); + } + } + + // Fast path for plain-text deltas (no style anchors / style ranges). + // + // Rebuilding avoids repeated BTree queries and mutations when the delta is very "choppy" + // (many small edit spans), but it allocates and clones chunks, so it can be slower for + // small deltas. Use a cheap cost model to enable it only when it's likely beneficial. + let should_fast_apply = { + #[inline] + fn ilog2_ceil(x: usize) -> usize { + debug_assert!(x > 0); + (usize::BITS - (x - 1).leading_zeros()) as usize + } + + let state = self.state.get_mut(); + if state.has_styles() { + false + } else { + // `edit_actions` approximates how many BTree mutations the incremental path will do: + // each Replace with delete>0 becomes a drain, and each Replace with value>0 becomes an insert. + let mut edit_actions: usize = 0; + let mut is_plain_text_delta = true; + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { .. } => {} + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + edit_actions += 1; + } + if value.rle_len() > 0 { + if !matches!(value, RichtextStateChunk::Text(_)) { + is_plain_text_delta = false; + break; + } + edit_actions += 1; + } + } + } + } + + if !is_plain_text_delta || edit_actions == 0 { + false + } else { + let content_nodes = state.content_node_len().max(1); + let log_n = ilog2_ceil(content_nodes + 1).max(1); + let incremental_score = edit_actions.saturating_mul(log_n); + let rebuild_score = content_nodes.saturating_add(edit_actions); + + let old_len = richtext.old_len().max(1); + let avg_action_span = old_len / edit_actions; + // A very rough proxy for "choppiness": many edit actions with small average span. + // The thresholds are intentionally conservative to avoid rebuilding for small or + // localized deltas. + let is_choppy = edit_actions >= 256 && avg_action_span <= 32; + + is_choppy && incremental_score >= rebuild_score.saturating_mul(4) + } + } + }; + + if should_fast_apply { + let new_state = { + let state = self.state.get_mut(); + let mut chunks: Vec = Vec::new(); + + let mut src_iter = state.iter_chunk(); + let mut cur = src_iter.next(); + let mut cur_offset: usize = 0; + + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + let mut left = *len; + while left > 0 { + let chunk = cur.expect("Delta retain exceeds source length"); + let chunk_len = chunk.rle_len(); + if chunk_len == 0 { + cur = src_iter.next(); + cur_offset = 0; + continue; + } + let available = chunk_len - cur_offset; + let take = left.min(available); + if take == chunk_len && cur_offset == 0 { + chunks.push(chunk.clone()); + } else { + chunks.push(chunk.slice(cur_offset..cur_offset + take)); + } + + left -= take; + cur_offset += take; + if cur_offset == chunk_len { + cur = src_iter.next(); + cur_offset = 0; + } + } + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + let mut left = *delete; + while left > 0 { + let chunk = cur.expect("Delta delete exceeds source length"); + let chunk_len = chunk.rle_len(); + if chunk_len == 0 { + cur = src_iter.next(); + cur_offset = 0; + continue; + } + let available = chunk_len - cur_offset; + let take = left.min(available); + left -= take; + cur_offset += take; + if cur_offset == chunk_len { + cur = src_iter.next(); + cur_offset = 0; + } + } + + if value.rle_len() > 0 { + chunks.push(value.clone()); + } + } + } + } + + if let Some(chunk) = cur { + let chunk_len = chunk.rle_len(); + if cur_offset < chunk_len { + if cur_offset == 0 { + chunks.push(chunk.clone()); + } else { + chunks.push(chunk.slice(cur_offset..chunk_len)); + } + } + } + for chunk in src_iter { + chunks.push(chunk.clone()); + } + + InnerState::from_chunks(chunks.into_iter()) + }; + + *self.state.get_mut() = new_state; + return Ok(()); + } + + let mut style_starts: FxHashMap, usize> = FxHashMap::default(); + let mut entity_index = 0; + for span in richtext.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + entity_index += len; + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + // Deletions + self.state + .get_mut() + .drain_by_entity_index(entity_index, *delete, None); + } + if value.rle_len() > 0 { + // Insertions + match value { + RichtextStateChunk::Text(s) => { + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Text(s.clone()), + ); + } + RichtextStateChunk::Style { style, anchor_type } => { + self.state.get_mut().insert_elem_at_entity_index( + entity_index, + RichtextStateChunk::Style { + style: style.clone(), + anchor_type: *anchor_type, + }, + ); + + if *anchor_type == AnchorType::Start { + style_starts.insert(style.clone(), entity_index); + } else { + let start_pos = match style_starts.get(style) { + Some(x) => *x, + None => { + // This should be rare, so it should be fine to scan + let mut start_entity_index = 0; + for c in self.state.get_mut().iter_chunk() { + match c { + RichtextStateChunk::Style { + style: s, + anchor_type: AnchorType::Start, + } if style == s => { + break; + } + RichtextStateChunk::Text(t) => { + start_entity_index += + t.unicode_len() as usize; + } + RichtextStateChunk::Style { .. } => { + start_entity_index += 1; + } + } + } + start_entity_index + } + }; + // we need to + 1 because we also need to annotate the end anchor + self.state.get_mut().annotate_style_range( + start_pos..entity_index + 1, + style.clone(), + ); + } + } + } + entity_index += value.rle_len(); + } + } + } + } + + // self.check_consistency_between_content_and_style_ranges() + Ok(()) + } + + fn apply_local_op(&mut self, r_op: &RawOp, op: &Op) -> LoroResult { + self.update_version(); + match &op.content { + crate::op::InnerContent::List(l) => match l { + list_op::InnerListOp::Insert { slice: _, pos: _ } => { + unreachable!() + } + list_op::InnerListOp::InsertText { + slice, + unicode_len: _, + unicode_start: _, + pos, + } => { + self.state.get_mut().insert_at_entity_index( + *pos as usize, + slice.clone(), + r_op.id_full(), + ); + } + list_op::InnerListOp::Delete(del) => { + self.state.get_mut().drain_by_entity_index( + del.start() as usize, + rle::HasLength::atom_len(&del), + None, + ); + } + list_op::InnerListOp::StyleStart { + start, + end, + key, + value, + info, + } => { + // Behavior here is a little different from apply_diff. + // + // When apply_diff, we only do the mark when we have included both + // StyleStart and StyleEnd. + // + // When applying local op, we can do the mark when we have StyleStart. + // We can assume StyleStart and StyleEnd are always appear in a pair + // for apply_local_op. (Because for local behavior, when we mark, + // we always create a pair of style ops.) + self.state.get_mut().mark_with_entity_index( + *start as usize..*end as usize, + Arc::new(StyleOp { + lamport: r_op.lamport, + peer: r_op.id.peer, + cnt: r_op.id.counter, + key: key.clone(), + value: value.clone(), + info: *info, + }), + ); + } + list_op::InnerListOp::Set { .. } => {} + list_op::InnerListOp::StyleEnd => {} + list_op::InnerListOp::Move { .. } => unreachable!(), + }, + _ => unreachable!(), + } + + // self.check_consistency_between_content_and_style_ranges(); + Ok(Default::default()) + } + + fn to_diff(&mut self, _doc: &Weak) -> Diff { + let mut delta = TextDiff::new(); + for span in self.state.get_mut().iter() { + delta.push_insert( + span.text, + span.attributes.to_option_map().unwrap_or_default().into(), + ); + } + + Diff::Text(delta) + } + + // value is a list + fn get_value(&mut self) -> LoroValue { + match &self.state { + LazyLoad::Src(loader) => loader.to_plain_string().into(), + LazyLoad::Dst(_) => self.state.get_mut().to_string().into(), + } + } + + #[doc = r" Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + None + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + Vec::new() + } + + fn contains_child(&self, _id: &ContainerID) -> bool { + false + } + + fn fork(&self, config: &crate::configure::Configure) -> Self { + Self { + idx: self.idx, + config: config.text_style_config.clone(), + state: self.state.clone(), + version_id: 0, + } + } +} + +impl RichtextState { + #[inline(always)] + pub fn len_utf8(&mut self) -> usize { + // §perf-heaven T8.15: O(1) from the Src loader (net-guarded accumulator). + if let LazyLoad::Src(loader) = &self.state { + return loader.bytes_len; + } + self.state.get_mut().len_utf8() + } + + #[inline(always)] + pub fn len(&mut self, pos_type: PosType) -> usize { + // §perf-heaven T1: unicode/entity length from the Src loader (O(1)) so the + // `char_at` bounds check (`len(Unicode)`) does not force the `Src -> Dst` + // build ahead of the object-anchor validation fast path. + if let LazyLoad::Src(loader) = &self.state { + match pos_type { + PosType::Unicode => return loader.unicode_len, + PosType::Entity => return loader.entity_index, + PosType::Utf16 => return loader.utf16_len, + PosType::Event => { + return if cfg!(feature = "wasm") { loader.utf16_len } else { loader.unicode_len }; + }, + PosType::Bytes => return loader.bytes_len, + #[allow(unreachable_patterns)] + _ => {}, + } + } + self.state.get_mut().len(pos_type) + } + + #[inline(always)] + pub fn len_utf16(&mut self) -> usize { + // §perf-heaven T7.2: O(1) from the Src loader (net-guarded accumulator). + if let LazyLoad::Src(loader) = &self.state { + return loader.utf16_len; + } + self.state.get_mut().len_utf16() + } + + #[inline(always)] + pub fn len_entity(&self) -> usize { + match &self.state { + LazyLoad::Src(s) => s.entity_index, + LazyLoad::Dst(d) => d.len_entity(), + } + } + + pub fn len_event(&mut self) -> usize { + if cfg!(feature = "wasm") { + self.len_utf16() + } else { + self.len_unicode() + } + } + + #[inline] + #[allow(dead_code)] + pub(crate) fn has_styles(&mut self) -> bool { + self.state.get_mut().has_styles() + } + + pub(crate) fn has_style_key_in_entity_range( + &mut self, + range: Range, + key: &StyleKey, + ) -> bool { + self.state.get_mut().range_has_style_key(range, key) + } + + /// Check if the content and style ranges are consistent. + /// + /// Panic if inconsistent. + #[allow(unused)] + pub(crate) fn check_consistency_between_content_and_style_ranges(&mut self) { + if !cfg!(debug_assertions) { + return; + } + + self.state + .get_mut() + .check_consistency_between_content_and_style_ranges(); + } + + #[inline] + pub fn len_unicode(&mut self) -> usize { + // §perf-heaven T1: answer from the Src loader (O(1)) instead of forcing + // the `Src -> Dst` B-tree build. The projection calls this per boundary + // BEFORE `to_delta`; forcing the build here would defeat the whole + // cold-decode fast path. + if let LazyLoad::Src(loader) = &self.state { + return loader.unicode_len; + } + self.state.get_mut().len_unicode() + } + + #[inline] + pub(crate) fn get_entity_index_for_text_insert( + &mut self, + index: usize, + pos_type: PosType, + ) -> Result<(usize, Option), LoroError> { + self.state + .get_mut() + .get_entity_index_for_text_insert(index, pos_type) + } + + #[inline] + pub(crate) fn get_event_index_by_cursor(&mut self, cursor: Cursor) -> usize { + self.state + .get_mut() + .get_index_from_cursor(cursor, PosType::Event) + .unwrap() + } + + pub(crate) fn get_entity_range_and_styles_at_range( + &mut self, + range: Range, + pos_type: PosType, + ) -> (Range, Option<&Styles>) { + self.state + .get_mut() + .get_entity_range_and_text_styles_at_range(range, pos_type) + } + + #[inline] + pub(crate) fn get_styles_at_entity_index(&mut self, entity_index: usize) -> StyleMeta { + self.state + .get_mut() + .get_styles_at_entity_index_for_insert(entity_index) + } + + #[inline] + pub(crate) fn get_text_entity_ranges_in_event_index_range( + &mut self, + pos: usize, + len: usize, + ) -> LoroResult> { + self.state + .get_mut() + .get_text_entity_ranges(pos, len, PosType::Event) + } + + #[inline] + pub fn get_richtext_value(&mut self) -> LoroValue { + // §perf-heaven T1: on a freshly decoded snapshot the state is still + // `LazyLoad::Src` (a cheap loader of chunks + resolved style ranges). + // `get_mut()` here would force the whole O(doc) `Src -> Dst` B-tree build + // — the dominant cost of the cold projection body decode. Resolve the + // value directly from the loader instead, skipping the text tree. Debug + // (and `FLOWSTATE_RICHTEXT_VERIFY`) builds ALSO build the tree and assert + // the fast path is bit-identical, then return the built value — so the + // fuzz/corpus oracle validates the cut and a divergence trips loudly. + let fast = if richtext_src_fastpath_enabled() { + match &self.state { + LazyLoad::Src(loader) => Some(InnerState::richtext_value_from_src(&loader.elements, &loader.style_ranges)), + LazyLoad::Dst(_) => None, + } + } else { + None + }; + match fast { + Some(fast) if !richtext_verify_enabled() => fast, + Some(fast) => { + let slow = self.state.get_mut().get_richtext_value(); + assert!( + fast == slow, + "FLOWSTATE perf-heaven T1: richtext_value_from_src diverged from the built richtext state" + ); + slow + } + None => self.state.get_mut().get_richtext_value(), + } + } + + /// §act-eleven A11.10 (flowstate vendor patch): stream `(text, styles)` + /// spans to `f` without materializing the delta `Vec`. Mirrors + /// [`Self::get_richtext_value`]'s T1 discipline exactly: on a freshly + /// decoded snapshot (`LazyLoad::Src`) the spans come straight from the + /// loader without forcing the O(doc) B-tree build; verify builds take the + /// value-level equivalence assert first (same oracle — both value functions + /// are expressed on these walkers), then stream from the built state. + pub fn for_each_richtext_span(&mut self, f: &mut dyn FnMut(&str, &crate::delta::StyleMeta)) { + if richtext_src_fastpath_enabled() { + if let LazyLoad::Src(loader) = &self.state { + if !richtext_verify_enabled() { + InnerState::for_each_span_from_src(&loader.elements, &loader.style_ranges, f); + return; + } + let fast = + InnerState::richtext_value_from_src(&loader.elements, &loader.style_ranges); + let slow = self.state.get_mut().get_richtext_value(); + assert!( + fast == slow, + "FLOWSTATE act-eleven A11.10: for_each_span_from_src diverged from the built richtext state" + ); + // fall through to the built state below + } + } + self.state.get_mut().for_each_richtext_span(f); + } + + #[inline] + pub(crate) fn get_stable_position( + &mut self, + event_index: usize, + get_by_event_index: bool, + ) -> Option { + self.state.get_mut().get_stable_position_at_event_index( + event_index, + if get_by_event_index { + PosType::Event + } else { + PosType::Unicode + }, + ) + } + + pub(crate) fn entity_index_to_event_index(&mut self, entity_index: usize) -> usize { + self.state + .get_mut() + .entity_index_to_event_index(entity_index) + } + + pub(crate) fn index_to_event_index(&mut self, index: usize, pos_type: PosType) -> usize { + self.state.get_mut().index_to_event_index(index, pos_type) + } + + pub(crate) fn event_index_to_unicode_index(&mut self, event_index: usize) -> usize { + self.state + .get_mut() + .event_index_to_unicode_index(event_index) + } +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct RichtextStateLoader { + start_anchor_pos: FxHashMap, + elements: SmallVec<[RichtextStateChunk; 1]>, + style_ranges: Vec<(Arc, Range)>, + entity_index: usize, + /// §perf-heaven T1: running unicode length of the pushed text chunks, so + /// `RichtextState::len_unicode` can answer O(1) from the loader WITHOUT + /// forcing the `Src -> Dst` B-tree build. The projection's boundary resolver + /// calls `len_unicode` per boundary; if that forced the build, the whole + /// cold-decode fast path would be defeated before `to_delta` ever runs. + unicode_len: usize, + /// §perf-heaven T7.2: running UTF-16 length, mirroring `unicode_len`, so + /// `len_utf16` / `len(Utf16)` / `len(Event)` (and the wasm event index) can + /// also answer O(1) from the loader. Both accumulators are validated against + /// the built B-tree in `into_state` (a debug-assert net), which also + /// retroactively guards the T1 `unicode_len` field. + utf16_len: usize, + /// §perf-heaven T8.15: running UTF-8 BYTE length, so `len_utf8` / `len(Bytes)` + /// also answer O(1) from the loader instead of forcing the `Src -> Dst` build. + /// Validated against the built B-tree in `into_state`. + bytes_len: usize, +} + +impl From for InnerState { + fn from(value: RichtextStateLoader) -> Self { + value.into_state() + } +} + +impl RichtextStateLoader { + /// §perf-heaven T1: the char at a UNICODE offset, straight from the Src + /// chunks (no `Src -> Dst` build). Backs `char_at_pos` on a still-loaded + /// snapshot; guarded by the corpus Src-path equivalence net (T7.26). + fn char_at_unicode(&self, pos: usize) -> Option { + let mut remaining = pos; + for elem in &self.elements { + if let RichtextStateChunk::Text(chunk) = elem { + let chunk_len = chunk.unicode_len() as usize; + if remaining < chunk_len { + return chunk.as_str().chars().nth(remaining); + } + remaining -= chunk_len; + } + } + None + } + + /// §perf-heaven T8.16: the char at a UTF-16 offset, straight from the Src + /// chunks (mirrors `char_at_unicode`, walking by `utf16_len`). Returns `None` + /// for an out-of-range OR mid-surrogate offset (a position inside a surrogate + /// pair is not a char boundary), matching the built state's `char_at`. Backs + /// the wasm event-index / `Utf16` `char_at` so they don't force the `Dst` + /// build. Guarded by the same `utf16_len` net as `len_utf16` (`into_state`). + fn char_at_utf16(&self, pos: usize) -> Option { + let mut remaining = pos; + for elem in &self.elements { + if let RichtextStateChunk::Text(chunk) = elem { + let chunk_len = chunk.utf16_len() as usize; + if remaining < chunk_len { + let mut acc = 0usize; + for ch in chunk.as_str().chars() { + if acc == remaining { + return Some(ch); + } + acc += ch.len_utf16(); + if acc > remaining { + return None; // `pos` fell inside a surrogate pair + } + } + return None; + } + remaining -= chunk_len; + } + } + None + } + + pub fn push(&mut self, elem: RichtextStateChunk) { + if let RichtextStateChunk::Style { style, anchor_type } = &elem { + if *anchor_type == AnchorType::Start { + self.start_anchor_pos + .insert(ID::new(style.peer, style.cnt), self.entity_index); + } else { + let start_pos = self + .start_anchor_pos + .remove(&ID::new(style.peer, style.cnt)) + .expect("Style start not found"); + + // we need to + 1 because we also need to annotate the end anchor + self.style_ranges + .push((style.clone(), start_pos..self.entity_index + 1)); + } + } + + if let RichtextStateChunk::Text(text) = &elem { + self.unicode_len += text.unicode_len() as usize; + self.utf16_len += text.utf16_len() as usize; + self.bytes_len += text.bytes().len(); + } + self.entity_index += elem.rle_len(); + self.elements.push(elem); + } + + pub fn into_state(self) -> InnerState { + // §perf-heaven T7.2 net: capture the running accumulators BEFORE the + // elements are consumed, then assert they equal the built B-tree's real + // lengths. This is the oracle for the Src-safe `len_unicode`/`len_utf16` + // /`len_utf8` fast paths — if `push` ever mis-accumulates, the very act + // of building the state (any `Src -> Dst` promotion) trips here. + let expected_unicode = self.unicode_len; + let expected_utf16 = self.utf16_len; + let expected_bytes = self.bytes_len; + let mut state = InnerState::from_chunks(self.elements.into_iter()); + for (style, range) in self.style_ranges { + state.annotate_style_range(range, style); + } + + if cfg!(debug_assertions) { + debug_assert_eq!( + state.len_unicode(), + expected_unicode, + "T7 loader unicode_len accumulation diverged from the built B-tree" + ); + debug_assert_eq!( + state.len_utf16(), + expected_utf16, + "T7.2 loader utf16_len accumulation diverged from the built B-tree" + ); + debug_assert_eq!( + state.len_utf8(), + expected_bytes, + "T8.15 loader bytes_len accumulation diverged from the built B-tree" + ); + state.check(); + } + + state + } + + fn is_empty(&self) -> bool { + self.elements.is_empty() + } + + fn to_plain_string(&self) -> String { + let len = self + .elements + .iter() + .map(|elem| match elem { + RichtextStateChunk::Text(text) => text.bytes().len(), + RichtextStateChunk::Style { .. } => 0, + }) + .sum(); + let mut text = String::with_capacity(len); + for elem in &self.elements { + if let RichtextStateChunk::Text(chunk) = elem { + text.push_str(chunk.as_str()); + } + } + text + } + + fn try_apply_append_delta(&mut self, delta: &DeltaRope) -> bool { + // Style anchors/ranges need InnerState's range maintenance. + if !self.start_anchor_pos.is_empty() || !self.style_ranges.is_empty() { + return false; + } + + let old_len = self.entity_index; + let mut current_len = self.entity_index; + let mut entity_index = 0; + let mut appended = Vec::new(); + for span in delta.iter() { + match span { + loro_delta::DeltaItem::Retain { len, .. } => { + entity_index += len; + if entity_index > old_len { + return false; + } + } + loro_delta::DeltaItem::Replace { value, delete, .. } => { + if *delete > 0 { + return false; + } + + let insert_len = value.rle_len(); + if insert_len == 0 { + continue; + } + + if !matches!(value, RichtextStateChunk::Text(_)) { + return false; + } + + if entity_index != current_len { + return false; + } + + appended.push(value.clone()); + entity_index += insert_len; + current_len += insert_len; + } + } + } + + if appended.is_empty() { + return false; + } + + for elem in appended { + self.push(elem); + } + true + } +} + +mod snapshot { + use loro_common::{IdFull, InternalString, LoroValue, PeerID}; + use rustc_hash::FxHashMap; + use serde_columnar::columnar; + use std::sync::Arc; + + use crate::{ + container::richtext::{ + self, richtext_state::RichtextStateChunk, str_slice::StrSlice, StyleOp, + TextStyleInfoFlag, + }, + encoding::value_register::ValueRegister, + state::{ + decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, ContainerCreationContext, ContainerState, FastStateSnapshot, + }, + utils::lazy::LazyLoad, + }; + + use super::{RichtextState, RichtextStateLoader}; + + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedTextSpan { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + #[columnar(strategy = "DeltaRle")] + lamport_sub_counter: i32, + /// positive for text + /// 0 for mark start + /// -1 for mark end + #[columnar(strategy = "DeltaRle")] + len: i32, + } + + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedMark { + key_idx: usize, + value: LoroValue, + info: u8, + } + + #[columnar(ser, de)] + struct EncodedText { + #[columnar(class = "vec", iter = "EncodedTextSpan")] + spans: Vec, + keys: Vec, + marks: Vec, + } + + impl FastStateSnapshot for RichtextState { + /// Encodes the RichtextState into a compact binary format for fast snapshot storage and retrieval. + /// + /// The encoding format consists of: + /// 1. The full text content as a string, encoded using postcard serialization. + /// 2. A series of EncodedTextSpan structs representing text chunks and style markers: + /// - peer_idx: Index of the peer ID in a value register (delta-RLE encoded) + /// - counter: Operation counter (delta-RLE encoded) + /// - lamport_sub_counter: Lamport timestamp - counter (delta-RLE encoded) + /// - len: Length of text chunk or marker type (-1 for end, 0 for start, positive for text) + /// 3. A list of unique style keys as InternalString. + /// 4. A series of EncodedMark structs for style information: + /// - key_idx: Index of the style key in the keys list + /// - value: The style value + /// - info: Additional style information as a byte + fn encode_snapshot_fast(&mut self, mut w: W) { + let value = self.get_value().into_string().unwrap(); + postcard::to_io(&*value, &mut w).unwrap(); + let mut spans = Vec::new(); + let mut marks = Vec::new(); + + let mut peers: ValueRegister = ValueRegister::new(); + let iter: &mut dyn Iterator; + let mut a; + let mut b; + match &self.state { + LazyLoad::Src(s) => { + a = Some(s.elements.iter()); + iter = &mut *a.as_mut().unwrap(); + } + LazyLoad::Dst(s) => { + b = Some(s.iter_chunk()); + iter = &mut *b.as_mut().unwrap(); + } + } + + let mut keys: ValueRegister = ValueRegister::new(); + + for chunk in iter { + match chunk { + RichtextStateChunk::Text(t) => { + let id = t.id_full(); + assert!(t.unicode_len() > 0); + spans.push(EncodedTextSpan { + peer_idx: peers.register(&id.peer), + counter: id.counter, + lamport_sub_counter: id.lamport as i32 - id.counter, + len: t.unicode_len(), + }) + } + RichtextStateChunk::Style { style, anchor_type } => match anchor_type { + richtext::AnchorType::Start => { + let id = style.id_full(); + spans.push(EncodedTextSpan { + peer_idx: peers.register(&id.peer), + counter: id.counter, + lamport_sub_counter: id.lamport as i32 - id.counter, + len: 0, + }); + marks.push(EncodedMark { + key_idx: keys.register(&style.key), + value: style.value.clone(), + info: style.info.to_byte(), + }) + } + richtext::AnchorType::End => { + let id = style.id_full(); + spans.push(EncodedTextSpan { + peer_idx: peers.register(&id.peer), + counter: id.counter + 1, + lamport_sub_counter: id.lamport as i32 - id.counter, + len: -1, + }) + } + }, + } + } + + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let bytes = serde_columnar::to_vec(&EncodedText { + spans, + keys: keys.unwrap_vec(), + marks, + }) + .unwrap(); + w.write_all(&bytes).unwrap(); + } + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + let (value, bytes) = postcard::take_from_bytes(bytes).map_err(|_| { + loro_common::LoroError::DecodeError( + "Decode list value failed".to_string().into_boxed_str(), + ) + })?; + let s: String = value; + Ok((LoroValue::String(s.into()), bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (string, mut bytes): (loro_common::LoroValue, &[u8]), + ctx: ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let mut text = RichtextState::new(idx, ctx.configure.text_style_config.clone()); + let mut loader = RichtextStateLoader::default(); + let peers = decode_peer_table(&mut bytes, "Decode richtext state failed")?; + + let string = string.into_string().map_err(|_| { + state_decode_error("Decode richtext state failed: value is not a string") + })?; + let mut s = StrSlice::new_from_str(&string); + let iters = serde_columnar::from_bytes::(bytes).map_err(|err| { + state_decode_error(format!( + "Decode richtext state failed: invalid spans: {err}" + )) + })?; + let keys = iters.keys; + let span_iter = iters.spans.into_iter(); + let mut mark_iter = iters.marks.into_iter(); + let mut id_to_style = FxHashMap::default(); + for span in span_iter { + let EncodedTextSpan { + peer_idx, + counter, + lamport_sub_counter, + len, + } = span; + let peer = + decode_peer_from_table(&peers, peer_idx, "Decode richtext state failed")?; + let lamport = decode_lamport_from_delta( + counter, + lamport_sub_counter, + "Decode richtext state failed", + )?; + let id_full = IdFull::new(peer, counter, lamport); + let chunk = match len { + 0 => { + // Style Start + let EncodedMark { + key_idx, + value, + info, + } = mark_iter.next().ok_or_else(|| { + state_decode_error("Decode richtext state failed: missing style mark") + })?; + let key = keys.get(key_idx).ok_or_else(|| { + state_decode_error( + "Decode richtext state failed: style key index out of range", + ) + })?; + let style_op = Arc::new(StyleOp { + lamport, + peer: id_full.peer, + cnt: id_full.counter, + key: key.clone(), + value, + info: TextStyleInfoFlag::from_byte(info), + }); + id_to_style.insert(id_full.id(), style_op.clone()); + RichtextStateChunk::new_style(style_op, richtext::AnchorType::Start) + } + -1 => { + // Style End + let style = id_to_style.remove(&id_full.id().inc(-1)).ok_or_else(|| { + state_decode_error("Decode richtext state failed: unmatched style end") + })?; + RichtextStateChunk::new_style(style, richtext::AnchorType::End) + } + len => { + if len < -1 { + return Err(state_decode_error( + "Decode richtext state failed: invalid text span length", + )); + } + + // Text — §flowstate vendor patch (oom-leads #2): ONE + // O(len) walk that validates bounds AND splits, replacing + // the per-span `chars().count()` check over the ENTIRE + // remaining document string (which made snapshot decode + // O(spans × chars) — 84% of a 2.6M-char doc's cold open, + // a ~13x cold-projection regression when this caller was + // lost in the vendored-file git-checkout revert). The + // method survived in str_slice.rs; this caller did not. + let Some((new, rest)) = s.try_split_at_unicode_pos(len as usize) else { + return Err(state_decode_error( + "Decode richtext state failed: text span exceeds value length", + )); + }; + s = rest; + RichtextStateChunk::new_text(new.bytes().clone(), id_full) + } + }; + + loader.push(chunk); + } + if !s.as_str().is_empty() { + return Err(state_decode_error( + "Decode richtext state failed: text value not fully covered", + )); + } + if mark_iter.next().is_some() { + return Err(state_decode_error( + "Decode richtext state failed: unused style mark", + )); + } + if !id_to_style.is_empty() { + return Err(state_decode_error( + "Decode richtext state failed: unclosed style mark", + )); + } + text.state = LazyLoad::Src(loader); + // NOTE: We need to ensure the invariance that the version id is always increased when the richtext state is changed + // This is used to avoid the version_id to be the same as the previous zero version + text.version_id = 1; + Ok(text) + } + } + + #[cfg(test)] + mod tests { + use loro_common::{ContainerType, LoroValue}; + + use crate::container::idx::ContainerIdx; + + use super::*; + + #[test] + fn richtext_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Text); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + assert!(RichtextState::decode_snapshot_fast( + idx, + (LoroValue::String("a".into()), &[1]), + ctx, + ) + .is_err()); + + let mut text = RichtextState::new(idx, configure.text_style_config.clone()); + let mut bytes = Vec::new(); + text.encode_snapshot_fast(&mut bytes); + let (_, state_bytes) = RichtextState::decode_value(&bytes).unwrap(); + assert!(RichtextState::decode_snapshot_fast( + idx, + (LoroValue::String("a".into()), state_bytes), + ctx, + ) + .is_err()); + } + } +} diff --git a/vendor/loro-internal/src/state/tree_state.rs b/vendor/loro-internal/src/state/tree_state.rs new file mode 100644 index 00000000..e89964a2 --- /dev/null +++ b/vendor/loro-internal/src/state/tree_state.rs @@ -0,0 +1,1831 @@ +use either::Either; +use enum_as_inner::EnumAsInner; +use fractional_index::FractionalIndex; +use itertools::Itertools; +use loro_common::{ + ContainerID, IdFull, IdLp, LoroError, LoroResult, LoroTreeError, LoroValue, PeerID, TreeID, + DELETED_TREE_ROOT, ID, +}; +use rand::SeedableRng; +use rustc_hash::{FxHashMap, FxHashSet}; +use serde::Serialize; +use std::collections::VecDeque; +use std::fmt::Debug; +use std::ops::{Deref, DerefMut}; +use std::sync::Weak; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; +use crate::configure::Configure; +use crate::container::idx::ContainerIdx; +use crate::delta::{TreeDiff, TreeDiffItem, TreeExternalDiff}; +use crate::diff_calc::DiffMode; +use crate::event::InternalDiff; +use crate::op::Op; +use crate::{ + container::tree::tree_op::TreeOp, + delta::TreeInternalDiff, + event::{Diff, Index}, + op::RawOp, +}; +use crate::{DocState, LoroDocInner}; + +#[derive(Clone, Debug, EnumAsInner)] +pub enum TreeFractionalIndexConfigInner { + GenerateFractionalIndex { + jitter: u8, + rng: Box, + }, + MoveDisabled, +} + +/// The state of movable tree. +/// +/// using flat representation +#[derive(Debug, Clone)] +pub struct TreeState { + idx: ContainerIdx, + trees: FxHashMap, + children: TreeChildrenCache, + fractional_index_config: TreeFractionalIndexConfigInner, + peer_id: PeerID, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TreeStateNode { + pub parent: TreeParentId, + pub position: Option, + pub last_move_op: IdFull, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct NodePosition { + pub(crate) position: FractionalIndex, + // different nodes created by a peer may have the same position + // when we merge updates that cause cycles. + // for example [::fuzz::test::test_tree::same_peer_have_same_position()] + pub(crate) idlp: IdLp, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumAsInner, Serialize)] +pub enum TreeParentId { + Node(TreeID), + Root, + Deleted, + // We use `Unexist` as the old parent of a new node created + // so we can infer the retreat internal diff is `Uncreate` + Unexist, +} + +impl From> for TreeParentId { + fn from(id: Option) -> Self { + match id { + Some(id) => { + if TreeID::is_deleted_root(&id) { + TreeParentId::Deleted + } else { + TreeParentId::Node(id) + } + } + None => TreeParentId::Root, + } + } +} + +impl From for TreeParentId { + fn from(id: TreeID) -> Self { + if TreeID::is_deleted_root(&id) { + TreeParentId::Deleted + } else { + TreeParentId::Node(id) + } + } +} + +impl From<&TreeID> for TreeParentId { + fn from(id: &TreeID) -> Self { + if TreeID::is_deleted_root(id) { + TreeParentId::Deleted + } else { + TreeParentId::Node(*id) + } + } +} + +impl TreeParentId { + pub fn tree_id(&self) -> Option { + match self { + TreeParentId::Node(id) => Some(*id), + TreeParentId::Root => None, + TreeParentId::Deleted => Some(DELETED_TREE_ROOT), + TreeParentId::Unexist => unreachable!(), + } + } +} + +#[derive(Debug, Clone)] +enum NodeChildren { + Vec(Vec<(NodePosition, TreeID)>), + BTree(btree::ChildTree), +} + +impl Default for NodeChildren { + fn default() -> Self { + NodeChildren::Vec(vec![]) + } +} + +impl NodeChildren { + const MAX_SIZE_FOR_ARRAY: usize = 16; + fn get_index_by_child_id(&self, target: &TreeID) -> Option { + match self { + NodeChildren::Vec(v) => v.iter().position(|(_, id)| id == target), + NodeChildren::BTree(btree) => btree.id_to_index(target), + } + } + + fn get_last_insert_index_by_position( + &self, + node_position: &NodePosition, + ) -> Result { + match self { + NodeChildren::Vec(v) => v.binary_search_by_key(&node_position, |x| &x.0), + NodeChildren::BTree(btree) => btree.get_index_by_node_position(node_position), + } + } + + fn get_node_position_at(&self, pos: usize) -> Option<&NodePosition> { + match self { + NodeChildren::Vec(v) => v.get(pos).map(|x| &x.0), + NodeChildren::BTree(btree) => btree.get_elem_at(pos).map(|x| x.pos.as_ref()), + } + } + + fn get_elem_at(&self, pos: usize) -> Option<(&NodePosition, &TreeID)> { + match self { + NodeChildren::Vec(v) => v.get(pos).map(|x| (&x.0, &x.1)), + NodeChildren::BTree(btree) => btree.get_elem_at(pos).map(|x| (x.pos.as_ref(), &x.id)), + } + } + + fn generate_fi_at(&self, pos: usize, target: &TreeID) -> FractionalIndexGenResult { + let mut reset_ids = vec![]; + let mut left = None; + let mut next_right = None; + { + let mut right = None; + let children_num = self.len(); + if children_num == 0 { + return FractionalIndexGenResult::Ok(FractionalIndex::default()); + } + + if pos > 0 { + left = self.get_node_position_at(pos - 1); + } + if pos < children_num { + right = self.get_elem_at(pos); + } + + let left_fi = left.map(|x| &x.position); + // if left and right have the same fractional indexes, we need to scan further to + // find all the ids that need to be reset + if let Some(left_fi) = left_fi { + if Some(left_fi) == right.map(|x| &x.0.position) { + // TODO: the min length between left and right + reset_ids.push(*right.unwrap().1); + for i in (pos + 1)..children_num { + let this_position = + self.get_node_position_at(i).map(|x| &x.position).unwrap(); + if this_position == left_fi { + reset_ids.push(*self.get_elem_at(i).unwrap().1); + } else { + next_right = Some(this_position.clone()); + break; + } + } + } + } + + if reset_ids.is_empty() { + return FractionalIndexGenResult::Ok( + FractionalIndex::new(left.map(|x| &x.position), right.map(|x| &x.0.position)) + .unwrap(), + ); + } + } + let positions = FractionalIndex::generate_n_evenly( + left.map(|x| &x.position), + next_right.as_ref(), + reset_ids.len() + 1, + ) + .unwrap(); + FractionalIndexGenResult::Rearrange( + Some(*target) + .into_iter() + .chain(reset_ids) + .zip(positions) + .collect(), + ) + } + + fn get_id_at(&self, pos: usize) -> Option { + match self { + NodeChildren::Vec(v) => v.get(pos).map(|x| x.1), + NodeChildren::BTree(btree) => btree.get_elem_at(pos).map(|x| x.id), + } + } + + fn delete_child(&mut self, target: &TreeID) { + match self { + NodeChildren::Vec(v) => { + v.retain(|(_, id)| id != target); + } + NodeChildren::BTree(v) => { + v.delete_child(target); + } + } + } + + fn upgrade(&mut self) { + match self { + NodeChildren::Vec(v) => { + let mut btree = btree::ChildTree::new(); + for (pos, id) in v.drain(..) { + btree.insert_child(pos, id); + } + + *self = NodeChildren::BTree(btree); + } + NodeChildren::BTree(_) => unreachable!(), + } + } + + fn insert_child(&mut self, pos: NodePosition, id: TreeID) { + match self { + NodeChildren::Vec(v) => { + if v.len() >= Self::MAX_SIZE_FOR_ARRAY { + self.upgrade(); + return self.insert_child(pos, id); + } + + let r = v.binary_search_by(|(target, _)| target.cmp(&pos)); + match r { + Ok(_) => unreachable!(), + Err(i) => { + v.insert(i, (pos, id)); + } + } + } + NodeChildren::BTree(v) => { + v.insert_child(pos, id); + } + } + } + + fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { + match self { + NodeChildren::Vec(v) => { + if v.len() >= Self::MAX_SIZE_FOR_ARRAY { + self.upgrade(); + return self.push_child_in_order(pos, id); + } + + if let Some(last) = v.last() { + assert!(last.0 < pos); + } + v.push((pos, id)); + } + NodeChildren::BTree(v) => { + v.push_child_in_order(pos, id); + } + } + } + + fn len(&self) -> usize { + match self { + NodeChildren::Vec(v) => v.len(), + NodeChildren::BTree(v) => v.len(), + } + } + + fn has_child(&self, node_position: &NodePosition) -> bool { + match self { + NodeChildren::Vec(v) => v + .binary_search_by(|(target, _)| target.cmp(node_position)) + .is_ok(), + NodeChildren::BTree(v) => v.has_child(node_position), + } + } + + fn iter(&self) -> impl Iterator { + match self { + NodeChildren::Vec(v) => Either::Left(v.iter().map(|x| (&x.0, &x.1))), + NodeChildren::BTree(t) => Either::Right(t.iter()), + } + } +} + +#[derive(Clone, Default)] +struct TreeChildrenCache(FxHashMap); + +mod btree { + use std::{cmp::Ordering, ops::Range, sync::Arc}; + + use generic_btree::{ + rle::{CanRemove, HasLength, Mergeable, Sliceable, TryInsert}, + BTree, BTreeTrait, Cursor, FindResult, LeafIndex, LengthFinder, Query, UseLengthFinder, + }; + use loro_common::TreeID; + use rustc_hash::FxHashMap; + + use super::NodePosition; + + struct ChildTreeTrait; + #[derive(Debug, Clone)] + pub(super) struct ChildTree { + tree: BTree, + id_to_leaf_index: FxHashMap, + } + + impl ChildTree { + pub(super) fn new() -> Self { + Self { + tree: BTree::new(), + id_to_leaf_index: FxHashMap::default(), + } + } + + pub(super) fn insert_child(&mut self, pos: NodePosition, id: TreeID) { + let (c, _) = self.tree.insert::( + &pos, + Elem { + pos: Arc::new(pos.clone()), + id, + }, + ); + + self.id_to_leaf_index.insert(id, c.leaf); + } + + pub(super) fn push_child_in_order(&mut self, pos: NodePosition, id: TreeID) { + let c = self.tree.push(Elem { + pos: Arc::new(pos.clone()), + id, + }); + self.id_to_leaf_index.insert(id, c.leaf); + } + + pub(super) fn delete_child(&mut self, id: &TreeID) { + if let Some(leaf) = self.id_to_leaf_index.remove(id) { + self.tree.remove_leaf(Cursor { leaf, offset: 0 }); + } + } + + pub(super) fn has_child(&self, pos: &NodePosition) -> bool { + match self.tree.query::(pos) { + Some(r) => r.found, + None => false, + } + } + + pub(super) fn iter(&self) -> impl Iterator { + self.tree.iter().map(|x| (&*x.pos, &x.id)) + } + + pub(super) fn len(&self) -> usize { + self.tree.root_cache().len + } + + pub(super) fn get_elem_at(&self, pos: usize) -> Option<&Elem> { + let result = self.tree.query::(&pos)?; + if !result.found { + return None; + } + self.tree.get_elem(result.leaf()) + } + + pub(super) fn id_to_index(&self, id: &TreeID) -> Option { + let leaf_index = self.id_to_leaf_index.get(id)?; + let mut ans = 0; + self.tree.visit_previous_caches( + Cursor { + leaf: *leaf_index, + offset: 0, + }, + |prev| match prev { + generic_btree::PreviousCache::NodeCache(c) => { + ans += c.len; + } + generic_btree::PreviousCache::PrevSiblingElem(_) => { + ans += 1; + } + generic_btree::PreviousCache::ThisElemAndOffset { .. } => {} + }, + ); + + Some(ans) + } + + pub(super) fn get_index_by_node_position( + &self, + node_position: &NodePosition, + ) -> Result { + let Some(res) = self.tree.query::(node_position) else { + return Ok(0); + }; + let mut ans = 0; + self.tree + .visit_previous_caches(res.cursor, |prev| match prev { + generic_btree::PreviousCache::NodeCache(c) => { + ans += c.len; + } + generic_btree::PreviousCache::PrevSiblingElem(_) => { + ans += 1; + } + generic_btree::PreviousCache::ThisElemAndOffset { elem: _, offset } => { + ans += offset; + } + }); + if res.found { + Ok(ans) + } else { + Err(ans) + } + } + } + + #[derive(Clone, Debug)] + pub(super) struct Elem { + pub(super) pos: Arc, + pub(super) id: TreeID, + } + + impl Mergeable for Elem { + fn can_merge(&self, _rhs: &Self) -> bool { + false + } + + fn merge_right(&mut self, _rhs: &Self) { + unreachable!() + } + + fn merge_left(&mut self, _left: &Self) { + unreachable!() + } + } + + impl HasLength for Elem { + fn rle_len(&self) -> usize { + 1 + } + } + + impl Sliceable for Elem { + fn _slice(&self, range: std::ops::Range) -> Self { + assert!(range.len() == 1); + self.clone() + } + } + + impl CanRemove for Elem { + fn can_remove(&self) -> bool { + false + } + } + + impl TryInsert for Elem { + fn try_insert(&mut self, _pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + Err(elem) + } + } + + #[derive(Clone, Debug, Default, PartialEq, Eq)] + struct Cache { + range: Option>>, + len: usize, + } + + impl BTreeTrait for ChildTreeTrait { + type Elem = Elem; + type Cache = Cache; + type CacheDiff = (); + const USE_DIFF: bool = false; + + fn calc_cache_internal( + cache: &mut Self::Cache, + caches: &[generic_btree::Child], + ) -> Self::CacheDiff { + if caches.is_empty() { + *cache = Default::default(); + return; + } + + *cache = Cache { + range: Some( + caches[0].cache.range.as_ref().unwrap().start.clone() + ..caches + .last() + .unwrap() + .cache + .range + .as_ref() + .unwrap() + .end + .clone(), + ), + len: caches.iter().map(|x| x.cache.len).sum(), + }; + } + + fn apply_cache_diff(_cache: &mut Self::Cache, _diff: &Self::CacheDiff) { + unreachable!() + } + + fn merge_cache_diff(_diff1: &mut Self::CacheDiff, _diff2: &Self::CacheDiff) {} + + fn get_elem_cache(elem: &Self::Elem) -> Self::Cache { + Cache { + range: Some(elem.pos.clone()..elem.pos.clone()), + len: 1, + } + } + + fn new_cache_to_diff(_cache: &Self::Cache) -> Self::CacheDiff {} + + fn sub_cache(_cache_lhs: &Self::Cache, _cache_rhs: &Self::Cache) -> Self::CacheDiff {} + } + + struct KeyQuery; + + impl Query for KeyQuery { + type QueryArg = NodePosition; + + #[inline(always)] + fn init(_target: &Self::QueryArg) -> Self { + KeyQuery + } + + #[inline] + fn find_node( + &mut self, + target: &Self::QueryArg, + caches: &[generic_btree::Child], + ) -> FindResult { + let result = caches.binary_search_by(|x| { + let range = x.cache.range.as_ref().unwrap(); + if target < &range.start { + core::cmp::Ordering::Greater + } else if target > &range.end { + core::cmp::Ordering::Less + } else { + core::cmp::Ordering::Equal + } + }); + + match result { + Ok(i) => FindResult::new_found(i, 0), + Err(i) => FindResult::new_missing( + i.min(caches.len() - 1), + if i == caches.len() { 1 } else { 0 }, + ), + } + } + + #[inline(always)] + fn confirm_elem( + &mut self, + q: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + match q.cmp(&elem.pos) { + Ordering::Less => (0, false), + Ordering::Equal => (0, true), + Ordering::Greater => (1, false), + } + } + } + + impl UseLengthFinder for ChildTreeTrait { + fn get_len(cache: &::Cache) -> usize { + cache.len + } + } +} + +impl Debug for TreeChildrenCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "TreeChildrenCache {{")?; + for (parent, children) in self.0.iter() { + writeln!(f, " {:?}:{{", parent)?; + for (position, id) in children.iter() { + writeln!(f, " {:?} -> {:?}", position, id)?; + } + writeln!(f, " }}")?; + } + writeln!(f, "}}") + } +} + +impl Deref for TreeChildrenCache { + type Target = FxHashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TreeChildrenCache { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl NodePosition { + fn new(position: FractionalIndex, idlp: IdLp) -> Self { + Self { position, idlp } + } +} + +pub(crate) enum FiIfNotConfigured { + UseJitterZero, + #[allow(unused)] + Throw, + #[allow(unused)] + Zero, +} + +impl TreeState { + pub fn new(idx: ContainerIdx, peer_id: PeerID) -> Self { + Self { + idx, + trees: FxHashMap::default(), + children: Default::default(), + fractional_index_config: TreeFractionalIndexConfigInner::GenerateFractionalIndex { + jitter: 0, + rng: Box::new(rand::rngs::StdRng::seed_from_u64(0)), + }, + peer_id, + } + } + + pub fn mov( + &mut self, + target: TreeID, + parent: TreeParentId, + id: IdFull, + position: Option, + with_check: bool, + ) -> Result<(), LoroError> { + if with_check { + if let TreeParentId::Node(parent) = parent { + if !self.trees.contains_key(&parent) { + return Err(LoroTreeError::TreeNodeParentNotFound(parent).into()); + } + } + if self.is_ancestor_of(&target, &parent) { + return Err(LoroTreeError::CyclicMoveError.into()); + } + } + if let Some(old_parent) = self.trees.get(&target).map(|x| x.parent) { + // remove old position + self.try_delete_position_cache(&old_parent, &target); + } + + let entry = self.children.entry(parent).or_default(); + let node_position = NodePosition::new(position.clone().unwrap_or_default(), id.idlp()); + debug_assert!(!entry.has_child(&node_position)); + entry.insert_child(node_position, target); + + self.trees.insert( + target, + TreeStateNode { + parent, + position, + last_move_op: id, + }, + ); + + Ok(()) + } + + fn _init_push_tree_node_in_order( + &mut self, + target: TreeID, + parent: TreeParentId, + last_move_op: IdFull, + position: Option, + ) -> Result<(), LoroError> { + debug_assert!(!self.trees.contains_key(&target)); + let entry = self.children.entry(parent).or_default(); + let node_position = + NodePosition::new(position.clone().unwrap_or_default(), last_move_op.idlp()); + debug_assert!(!entry.has_child(&node_position)); + entry.push_child_in_order(node_position, target); + self.trees.insert( + target, + TreeStateNode { + parent, + position, + last_move_op, + }, + ); + + Ok(()) + } + + #[inline(never)] + fn is_ancestor_of(&self, maybe_ancestor: &TreeID, node_id: &TreeParentId) -> bool { + if !self.trees.contains_key(maybe_ancestor) { + return false; + } + if let TreeParentId::Node(id) = node_id { + if id == maybe_ancestor { + return true; + } + } + match node_id { + TreeParentId::Node(id) => { + let parent = &self.trees.get(id).unwrap().parent; + if parent == node_id { + panic!("is_ancestor_of loop") + } + self.is_ancestor_of(maybe_ancestor, parent) + } + TreeParentId::Deleted | TreeParentId::Root => false, + TreeParentId::Unexist => unreachable!(), + } + } + + /// Get the parent of the node, if the node does not exist, return None + pub fn parent(&self, target: &TreeID) -> Option { + self.trees.get(target).map(|x| x.parent) + } + + /// If the node is unexist, return None. If the node is deleted, return true. + pub(crate) fn is_node_deleted(&self, target: &TreeID) -> Option { + match self.trees.get(target).map(|x| x.parent)? { + TreeParentId::Deleted => Some(true), + TreeParentId::Root => Some(false), + TreeParentId::Node(p) => self.is_node_deleted(&p), + TreeParentId::Unexist => unreachable!(), + } + } + + pub(crate) fn is_node_unexist(&self, target: &TreeID) -> bool { + !self.trees.contains_key(target) + } + + // Get all flat tree nodes, excluding deleted nodes. + pub(crate) fn tree_nodes(&self) -> Vec { + self.get_all_tree_nodes_under(TreeParentId::Root) + } + + // Get all flat deleted nodes + #[allow(unused)] + pub(crate) fn deleted_tree_nodes(&self) -> Vec { + self.get_all_tree_nodes_under(TreeParentId::Deleted) + } + + // Get all flat tree nodes under the parent + pub(crate) fn get_all_tree_nodes_under(&self, root: TreeParentId) -> Vec { + let mut ans = vec![]; + let children = self.children.get(&root); + let mut q = children + .map(|x| VecDeque::from_iter(x.iter().enumerate().zip(std::iter::repeat(root)))) + .unwrap_or_default(); + + while let Some(((index, (position, &target)), parent)) = q.pop_front() { + ans.push(TreeNode { + id: target, + parent, + fractional_index: position.position.clone(), + index, + last_move_op: self.trees.get(&target).map(|x| x.last_move_op).unwrap(), + }); + if let Some(children) = self.children.get(&TreeParentId::Node(target)) { + q.extend( + children + .iter() + .enumerate() + .map(|(index, (position, this_target))| { + ((index, (position, this_target)), TreeParentId::Node(target)) + }), + ); + } + } + ans + } + + pub(crate) fn get_all_hierarchy_nodes_under( + &self, + root: TreeParentId, + ) -> Vec { + let mut ans = vec![]; + let Some(children) = self.children.get(&root) else { + return ans; + }; + for (index, (position, &target)) in children.iter().enumerate() { + ans.push(TreeNodeWithChildren { + id: target, + parent: root, + fractional_index: position.position.clone(), + index, + children: self.get_all_hierarchy_nodes_under(TreeParentId::Node(target)), + }) + } + ans + } + + fn bfs_all_alive_nodes_for_fast_snapshot(&self) -> Vec { + let mut ans = vec![]; + self._bfs_all_nodes(TreeParentId::Root, &mut ans); + ans + } + + fn bfs_all_deleted_nodes_for_fast_snapshot(&self) -> Vec { + let mut ans = vec![]; + self._bfs_all_nodes(TreeParentId::Deleted, &mut ans); + ans + } + + fn _bfs_all_nodes(&self, root: TreeParentId, ans: &mut Vec) { + let children = self.children.get(&root); + if let Some(children) = children { + for (index, (position, target)) in children.iter().enumerate() { + ans.push(TreeNode { + id: *target, + parent: root, + fractional_index: position.position.clone(), + index, + last_move_op: self.trees.get(target).map(|x| x.last_move_op).unwrap(), + }); + } + + for (_, id) in children.iter() { + self._bfs_all_nodes(TreeParentId::Node(*id), ans); + } + } + } + + pub fn nodes(&self) -> Vec { + self.trees.keys().copied().collect::>() + } + + #[cfg(feature = "test_utils")] + pub fn max_counter(&self) -> i32 { + self.trees + .keys() + .filter(|&k| !self.is_node_deleted(k).unwrap()) + .map(|k| k.counter) + .max() + .unwrap_or(0) + } + + pub fn get_children<'a>( + &'a self, + parent: &TreeParentId, + ) -> Option + 'a> { + self.children.get(parent).map(|x| x.iter().map(|x| *x.1)) + } + + pub fn children_num(&self, parent: &TreeParentId) -> Option { + self.children.get(parent).map(|x| x.len()) + } + + /// Determine whether the target is the child of the node + /// + /// O(1) + pub fn is_parent(&self, target: &TreeID, parent: &TreeParentId) -> bool { + self.trees.get(target).is_some_and(|x| x.parent == *parent) + } + + /// Delete the position cache of the node + pub(crate) fn try_delete_position_cache(&mut self, parent: &TreeParentId, target: &TreeID) { + if let Some(x) = self.children.get_mut(parent) { + x.delete_child(target); + } + } + + pub(crate) fn generate_position_at( + &mut self, + target: &TreeID, + parent: &TreeParentId, + index: usize, + cfg: FiIfNotConfigured, + ) -> FractionalIndexGenResult { + match &mut self.fractional_index_config { + TreeFractionalIndexConfigInner::GenerateFractionalIndex { jitter, rng } => { + if *jitter == 0 { + self.children + .entry(*parent) + .or_default() + .generate_fi_at(index, target) + } else { + self.children + .entry(*parent) + .or_default() + .generate_fi_at_jitter(index, target, rng.as_mut(), *jitter) + } + } + TreeFractionalIndexConfigInner::MoveDisabled => match cfg { + FiIfNotConfigured::Throw => FractionalIndexGenResult::NotConfigured, + FiIfNotConfigured::UseJitterZero => self + .children + .entry(*parent) + .or_default() + .generate_fi_at(index, target), + FiIfNotConfigured::Zero => FractionalIndexGenResult::Ok(FractionalIndex::default()), + }, + } + } + + pub(crate) fn is_fractional_index_enabled(&self) -> bool { + !matches!( + self.fractional_index_config, + TreeFractionalIndexConfigInner::MoveDisabled + ) + } + + pub(crate) fn enable_generate_fractional_index(&mut self, jitter: u8) { + if let TreeFractionalIndexConfigInner::GenerateFractionalIndex { + jitter: old_jitter, .. + } = &mut self.fractional_index_config + { + *old_jitter = jitter; + return; + } + self.fractional_index_config = TreeFractionalIndexConfigInner::GenerateFractionalIndex { + jitter, + rng: Box::new(rand::rngs::StdRng::seed_from_u64(self.peer_id)), + }; + } + + pub(crate) fn disable_generate_fractional_index(&mut self) { + self.fractional_index_config = TreeFractionalIndexConfigInner::MoveDisabled; + } + + pub(crate) fn get_position(&self, target: &TreeID) -> Option { + self.trees.get(target).and_then(|x| x.position.clone()) + } + + pub(crate) fn get_index_by_tree_id(&self, target: &TreeID) -> Option { + let parent = self.parent(target)?; + (!parent.is_deleted()) + .then(|| { + self.children + .get(&parent) + .and_then(|x| x.get_index_by_child_id(target)) + }) + .flatten() + } + + pub(crate) fn get_index_by_position( + &self, + parent: &TreeParentId, + node_position: &NodePosition, + ) -> Option { + self.children.get(parent).map(|c| { + match c.get_last_insert_index_by_position(node_position) { + Ok(i) => i, + Err(i) => i, + } + }) + } + + pub(crate) fn get_id_by_index(&self, parent: &TreeParentId, index: usize) -> Option { + (!parent.is_deleted()) + .then(|| self.children.get(parent).and_then(|x| x.get_id_at(index))) + .flatten() + } + + /// Check the consistency between `self.trees` and `self.children` + /// + /// It's used for debug and test + #[allow(unused)] + pub(crate) fn check_tree_integrity(&self) { + let mut parent_children_map: FxHashMap> = + FxHashMap::default(); + for (id, node) in self.trees.iter() { + let parent = node.parent; + parent_children_map.entry(parent).or_default().insert(*id); + } + + for (parent, children) in parent_children_map.iter() { + let cached_children = self.get_children(parent).expect("parent not found"); + let cached_children = cached_children.collect::>(); + if &cached_children != children { + panic!( + "tree integrity broken: children set of node {:?} is not consistent", + parent + ); + } + } + } + + pub fn is_empty(&self) -> bool { + match self.children.get(&TreeParentId::Root) { + Some(c) => c.len() == 0, + None => true, + } + } + + pub fn get_last_move_id(&self, id: &TreeID) -> Option { + self.trees.get(id).map(|x| x.last_move_op.id()) + } +} + +pub(crate) enum FractionalIndexGenResult { + Ok(FractionalIndex), + Rearrange(Vec<(TreeID, FractionalIndex)>), + NotConfigured, +} + +impl ContainerState for TreeState { + fn container_idx(&self) -> crate::container::idx::ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + self.nodes().is_empty() + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + fn apply_diff_and_convert( + &mut self, + diff: crate::event::InternalDiff, + ctx: DiffApplyContext, + ) -> Diff { + let need_check = !matches!(ctx.mode, DiffMode::Checkout | DiffMode::Linear); + let mut ans = vec![]; + if let InternalDiff::Tree(tree) = &diff { + // assert never cause cycle move + for diff in tree.diff.iter() { + let last_move_op = diff.last_effective_move_op_id; + let target = diff.target; + // create associated metadata container + match &diff.action { + TreeInternalDiff::Create { parent, position } => { + self.mov(target, *parent, last_move_op, Some(position.clone()), false) + .unwrap(); + + if let TreeParentId::Node(p) = parent { + // reuse diff cache, at this time,it's "move in deleted" + if self.is_node_deleted(p).unwrap() { + continue; + } + } + + let index = self.get_index_by_tree_id(&target).unwrap(); + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: *parent, + index, + position: position.clone(), + }, + }); + } + TreeInternalDiff::Move { parent, position } => { + let old_parent = self.trees.get(&target).unwrap().parent; + // If this is some, the node is still alive at the moment + let old_index = self.get_index_by_tree_id(&target); + let was_alive = !self.is_node_deleted(&target).unwrap(); + if need_check { + if self + .mov(target, *parent, last_move_op, Some(position.clone()), true) + .is_ok() + { + if self.is_node_deleted(&target).unwrap() { + if was_alive { + // delete event + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent, + old_index: old_index.unwrap(), + }, + }); + } + // Otherwise, it's a normal move inside deleted nodes, no event is needed + } else if was_alive { + // normal move + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent: *parent, + index: self.get_index_by_tree_id(&target).unwrap(), + position: position.clone(), + old_parent, + old_index: old_index.unwrap(), + }, + }); + } else { + // create event + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: *parent, + index: self.get_index_by_tree_id(&target).unwrap(), + position: position.clone(), + }, + }); + } + } + } else { + self.mov(target, *parent, last_move_op, Some(position.clone()), false) + .unwrap(); + + if let TreeParentId::Node(p) = parent { + // reuse diff cache, at this time,it's "move in deleted" + if self.is_node_deleted(p).unwrap() { + continue; + } + } + + let index = self.get_index_by_tree_id(&target).unwrap(); + match was_alive { + true => { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Move { + parent: *parent, + index, + position: position.clone(), + old_parent, + old_index: old_index.unwrap(), + }, + }); + } + false => { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: *parent, + index, + position: position.clone(), + }, + }); + } + } + }; + } + TreeInternalDiff::Delete { parent, position } => { + let mut send_event = true; + if self.is_node_deleted(&target).unwrap() { + send_event = false; + } + if send_event { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent: self.trees.get(&target).unwrap().parent, + old_index: self.get_index_by_tree_id(&target).unwrap(), + }, + }); + } + self.mov(target, *parent, last_move_op, position.clone(), false) + .unwrap(); + } + TreeInternalDiff::MoveInDelete { parent, position } => { + self.mov(target, *parent, last_move_op, position.clone(), false) + .unwrap(); + } + TreeInternalDiff::UnCreate => { + // maybe the node created and moved to the parent deleted + if !self.is_node_deleted(&target).unwrap() { + ans.push(TreeDiffItem { + target, + action: TreeExternalDiff::Delete { + old_parent: self.trees.get(&target).unwrap().parent, + old_index: self.get_index_by_tree_id(&target).unwrap(), + }, + }); + } + // delete it from state + let node = self.trees.remove(&target); + if let Some(node) = node { + if !node.parent.is_deleted() { + self.children + .get_mut(&node.parent) + .unwrap() + .delete_child(&target); + } + } + continue; + } + }; + } + } + + // self.check_tree_integrity(); + Diff::Tree(TreeDiff { diff: ans }) + } + + // How we apply the diff is coupled with the [DiffMode] we used to calculate the diff. + // So be careful when you modify this function. + fn apply_diff(&mut self, diff: InternalDiff, ctx: DiffApplyContext) -> LoroResult<()> { + if let InternalDiff::Tree(tree) = &diff { + let need_check = !matches!(ctx.mode, DiffMode::Checkout | DiffMode::Linear); + // assert never cause cycle move + for diff in tree.diff.iter() { + let last_move_op = diff.last_effective_move_op_id; + let target = diff.target; + // create associated metadata container + match &diff.action { + TreeInternalDiff::Create { parent, position } + | TreeInternalDiff::Move { + parent, position, .. + } => { + if need_check { + self.mov(target, *parent, last_move_op, Some(position.clone()), true) + .unwrap_or_default(); + } else { + self.mov(target, *parent, last_move_op, Some(position.clone()), false)?; + } + } + TreeInternalDiff::Delete { parent, position } => { + self.mov(target, *parent, last_move_op, position.clone(), false)?; + } + TreeInternalDiff::MoveInDelete { parent, position } => { + self.mov(target, *parent, last_move_op, position.clone(), false)?; + } + TreeInternalDiff::UnCreate => { + // delete it from state + let parent = self.trees.remove(&target); + if let Some(parent) = parent { + if !parent.parent.is_deleted() { + self.children + .get_mut(&parent.parent) + .unwrap() + .delete_child(&target); + } + } + continue; + } + }; + } + } + // self.check_tree_integrity(); + Ok(()) + } + + fn apply_local_op(&mut self, raw_op: &RawOp, _op: &Op) -> LoroResult { + let mut deleted_containers = vec![]; + match &raw_op.content { + crate::op::RawOpContent::Tree(tree) => match &**tree { + TreeOp::Create { + target, + parent, + position, + } + | TreeOp::Move { + target, + parent, + position, + } => { + let parent = TreeParentId::from(*parent); + self.mov( + *target, + parent, + raw_op.id_full(), + Some(position.clone()), + true, + )?; + } + TreeOp::Delete { target } => { + let parent = TreeParentId::Deleted; + deleted_containers.push(ContainerID::new_normal( + target.id(), + loro_common::ContainerType::Map, + )); + self.mov(*target, parent, raw_op.id_full(), None, true)?; + } + }, + _ => unreachable!(), + } + // self.check_tree_integrity(); + Ok(ApplyLocalOpReturn { deleted_containers }) + } + + fn to_diff(&mut self, _doc: &Weak) -> Diff { + let mut diffs = vec![]; + let Some(roots) = self.children.get(&TreeParentId::Root) else { + return Diff::Tree(TreeDiff { diff: vec![] }); + }; + + let mut q = VecDeque::from_iter(roots.iter()); + let mut index = 0; + let mut parent = TreeParentId::Root; + while let Some((position, node)) = q.pop_front() { + let node_parent = self.trees.get(node).unwrap().parent; + if node_parent != parent { + index = 0; + parent = node_parent; + } + let diff = TreeDiffItem { + target: *node, + action: TreeExternalDiff::Create { + parent: node_parent, + index, + position: position.position.clone(), + }, + }; + index += 1; + diffs.push(diff); + if let Some(children) = self.children.get(&TreeParentId::Node(*node)) { + // TODO: Refactor: you can include the index and parent in the q + // The code will be more robust and easy to understand + q.extend(children.iter()); + } + } + + Diff::Tree(TreeDiff { diff: diffs }) + } + + fn get_value(&mut self) -> LoroValue { + self.get_all_hierarchy_nodes_under(TreeParentId::Root) + .into_iter() + .map(|x| x.into_value()) + .collect::>() + .into() + } + + /// Get the index of the child container + fn get_child_index(&self, id: &ContainerID) -> Option { + let id = id.as_normal().unwrap(); + let tree_id = TreeID { + peer: *id.0, + counter: *id.1, + }; + if !self.trees.contains_key(&tree_id) || self.is_node_deleted(&tree_id).unwrap() { + None + } else { + Some(Index::Node(tree_id)) + } + } + + fn contains_child(&self, id: &ContainerID) -> bool { + let id = id.as_normal().unwrap(); + let tree_id = TreeID { + peer: *id.0, + counter: *id.1, + }; + self.trees.contains_key(&tree_id) && !self.is_node_deleted(&tree_id).unwrap() + } + + fn get_child_containers(&self) -> Vec { + self.trees + .keys() + .map(|n| n.associated_meta_container()) + .collect_vec() + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +// convert map container to LoroValue +#[allow(clippy::ptr_arg)] +pub(crate) fn get_meta_value(nodes: &mut Vec, state: &mut DocState) { + for node in nodes.iter_mut() { + let map = node.as_map_mut().unwrap().make_mut(); + let meta = map.get_mut("meta").unwrap(); + let id = meta.as_container().unwrap(); + *meta = state.get_container_deep_value(state.arena.register_container(id)); + let children = map.get_mut("children").unwrap().as_list_mut().unwrap(); + get_meta_value(children.make_mut(), state); + } +} + +#[derive(Debug, Clone)] +pub struct TreeNode { + pub id: TreeID, + pub parent: TreeParentId, + pub fractional_index: FractionalIndex, + pub index: usize, + pub last_move_op: IdFull, +} + +#[derive(Debug, Clone)] +pub struct TreeNodeWithChildren { + pub id: TreeID, + pub parent: TreeParentId, + pub fractional_index: FractionalIndex, + pub index: usize, + pub children: Vec, +} + +impl TreeNodeWithChildren { + fn into_value(self) -> LoroValue { + let mut t = FxHashMap::default(); + t.insert("id".to_string(), self.id.to_string().into()); + let p = self + .parent + .tree_id() + .map(|p| p.to_string().into()) + .unwrap_or(LoroValue::Null); + t.insert("parent".to_string(), p); + t.insert( + "meta".to_string(), + self.id.associated_meta_container().into(), + ); + t.insert("index".to_string(), (self.index as i64).into()); + t.insert( + "fractional_index".to_string(), + self.fractional_index.to_string().into(), + ); + t.insert( + "children".to_string(), + self.children + .into_iter() + .map(|x| x.into_value()) + .collect::>() + .into(), + ); + t.into() + } +} + +mod jitter { + use super::{FractionalIndexGenResult, NodeChildren}; + use fractional_index::FractionalIndex; + use loro_common::TreeID; + use rand::Rng; + + impl NodeChildren { + pub(super) fn generate_fi_at_jitter( + &self, + pos: usize, + target: &TreeID, + rng: &mut impl Rng, + jitter: u8, + ) -> FractionalIndexGenResult { + let mut reset_ids = vec![]; + let mut left = None; + let mut next_right = None; + { + let mut right = None; + let children_num = self.len(); + if children_num == 0 { + return FractionalIndexGenResult::Ok(FractionalIndex::jitter_default( + rng, jitter, + )); + } + + if pos > 0 { + left = self.get_node_position_at(pos - 1); + } + if pos < children_num { + right = self.get_elem_at(pos); + } + + let left_fi = left.map(|x| &x.position); + // if left and right have the same fractional indexes, we need to scan further to + // find all the ids that need to be reset + if let Some(left_fi) = left_fi { + if Some(left_fi) == right.map(|x| &x.0.position) { + // TODO: the min length between left and right + reset_ids.push(*right.unwrap().1); + for i in (pos + 1)..children_num { + let this_position = &self.get_node_position_at(i).unwrap().position; + if this_position == left_fi { + reset_ids.push(*self.get_elem_at(i).unwrap().1); + } else { + next_right = Some(this_position.clone()); + break; + } + } + } + } + + if reset_ids.is_empty() { + return FractionalIndexGenResult::Ok( + FractionalIndex::new_jitter( + left.map(|x| &x.position), + right.map(|x| &x.0.position), + rng, + jitter, + ) + .unwrap(), + ); + } + } + let positions = FractionalIndex::generate_n_evenly_jitter( + left.map(|x| &x.position), + next_right.as_ref(), + reset_ids.len() + 1, + rng, + jitter, + ) + .unwrap(); + FractionalIndexGenResult::Rearrange( + Some(*target) + .into_iter() + .chain(reset_ids) + .zip(positions) + .collect(), + ) + } + } +} + +mod snapshot { + use std::{borrow::Cow, collections::BTreeSet}; + + use fractional_index::FractionalIndex; + use loro_common::{IdFull, PeerID, TreeID}; + use rustc_hash::FxHashMap; + + use serde_columnar::columnar; + + use crate::{ + encoding::{arena::PositionArena, value_register::ValueRegister}, + state::{ + decode_counter, decode_lamport_from_delta, decode_peer_from_table, decode_peer_table, + state_decode_error, FastStateSnapshot, + }, + }; + + use super::{TreeNode, TreeParentId, TreeState}; + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedTreeNodeId { + #[columnar(strategy = "DeltaRle")] + peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + counter: i32, + } + + #[columnar(vec, ser, de, iterable)] + #[derive(Debug, Clone)] + struct EncodedTreeNode { + /// If this field is 0, it means none, its parent is root + /// If this field is 1, its parent is the deleted root + #[columnar(strategy = "DeltaRle")] + parent_idx_plus_two: usize, + #[columnar(strategy = "DeltaRle")] + last_set_peer_idx: usize, + #[columnar(strategy = "DeltaRle")] + last_set_counter: i32, + #[columnar(strategy = "DeltaRle")] + last_set_lamport_sub_counter: i32, + fractional_index_idx: usize, + } + + #[columnar(ser, de)] + struct EncodedTree<'a> { + #[columnar(class = "vec", iter = "EncodedTreeNodeId")] + node_ids: Vec, + #[columnar(class = "vec", iter = "EncodedTreeNode")] + nodes: Vec, + #[columnar(borrow)] + fractional_indexes: Cow<'a, [u8]>, + #[columnar(borrow)] + reserved_has_effect_bool_rle: Cow<'a, [u8]>, + } + + fn encode( + state: &TreeState, + input: Vec, + deleted_nodes: Vec, + ) -> (ValueRegister, EncodedTree<'_>) { + let mut peers: ValueRegister = ValueRegister::new(); + let mut position_set = BTreeSet::default(); + let mut nodes = Vec::with_capacity(input.len()); + let mut node_ids = Vec::with_capacity(input.len()); + let mut position_register = ValueRegister::new(); + let mut id_to_idx = FxHashMap::default(); + for node in input.iter().chain(deleted_nodes.iter()) { + position_set.insert(node.fractional_index.clone()); + let idx = node_ids.len(); + node_ids.push(EncodedTreeNodeId { + peer_idx: peers.register(&node.id.peer), + counter: node.id.counter, + }); + id_to_idx.insert(node.id, idx); + } + + for p in position_set { + position_register.register(&p); + } + + for node in input { + let n = state.trees.get(&node.id).unwrap(); + let last_set_id = n.last_move_op; + nodes.push(EncodedTreeNode { + parent_idx_plus_two: match node.parent { + TreeParentId::Deleted => 1, + TreeParentId::Root => 0, + TreeParentId::Node(id) => id_to_idx.get(&id).unwrap() + 2, + TreeParentId::Unexist => unreachable!(), + }, + last_set_peer_idx: peers.register(&last_set_id.peer), + last_set_counter: last_set_id.counter, + last_set_lamport_sub_counter: last_set_id.lamport as i32 - last_set_id.counter, + fractional_index_idx: position_register.register(&node.fractional_index), + }) + } + + for node in deleted_nodes { + let n = state.trees.get(&node.id).unwrap(); + let last_set_id = n.last_move_op; + nodes.push(EncodedTreeNode { + parent_idx_plus_two: match node.parent { + TreeParentId::Deleted => 1, + TreeParentId::Root => 0, + TreeParentId::Node(id) => id_to_idx.get(&id).unwrap() + 2, + TreeParentId::Unexist => unreachable!(), + }, + last_set_peer_idx: peers.register(&last_set_id.peer), + last_set_counter: last_set_id.counter, + last_set_lamport_sub_counter: last_set_id.lamport as i32 - last_set_id.counter, + fractional_index_idx: position_register.register(&node.fractional_index), + }) + } + + let position_vec = position_register.unwrap_vec(); + let positions = PositionArena::from_positions(position_vec.iter().map(|p| p.as_bytes())); + ( + peers, + EncodedTree { + node_ids, + nodes, + fractional_indexes: positions.encode().into(), + reserved_has_effect_bool_rle: vec![].into(), + }, + ) + } + + impl FastStateSnapshot for TreeState { + /// Encodes the TreeState into a compact binary format for efficient serialization. + /// + /// The encoding schema: + /// 1. Encodes all nodes using a breadth-first search traversal, ensuring a consistent order. + /// 2. Uses a ValueRegister to deduplicate and index PeerIDs and TreeIDs, reducing redundancy. + /// - PeerIDs are stored once and referenced by index. + /// - TreeIDs are decomposed into peer index and counter for compact representation. + /// 3. Encodes fractional indexes using a PositionArena for space efficiency + /// 4. Utilizes delta encoding and run-length encoding for further size reduction: + /// - Delta encoding stores differences between consecutive values. + /// - Run-length encoding compresses sequences of repeated values. + /// 5. Stores parent relationships using indices, with special values for root and deleted nodes. + /// 6. Encodes last move operation details (peer_idx, counter[Delta], lamport clock[Delta]) for each node. + fn encode_snapshot_fast(&mut self, mut w: W) { + let all_alive_nodes = self.bfs_all_alive_nodes_for_fast_snapshot(); + let all_deleted_nodes = self.bfs_all_deleted_nodes_for_fast_snapshot(); + let (peers, encoded) = encode(self, all_alive_nodes, all_deleted_nodes); + let peers = peers.unwrap_vec(); + leb128::write::unsigned(&mut w, peers.len() as u64).unwrap(); + for peer in peers { + w.write_all(&peer.to_le_bytes()).unwrap(); + } + + let vec = serde_columnar::to_vec(&encoded).unwrap(); + w.write_all(&vec).unwrap(); + } + + fn decode_value(_: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + unreachable!() + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + (_, mut bytes): (loro_common::LoroValue, &[u8]), + ctx: crate::state::ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + let peers = decode_peer_table(&mut bytes, "Decode tree state failed")?; + + let mut tree = TreeState::new(idx, ctx.peer); + let encoded: EncodedTree = serde_columnar::from_bytes(bytes).map_err(|err| { + state_decode_error(format!("Decode tree state failed: invalid metadata: {err}")) + })?; + let fractional_indexes = + PositionArena::decode(&encoded.fractional_indexes)?.try_parse_to_positions()?; + if encoded.node_ids.len() != encoded.nodes.len() { + return Err(state_decode_error( + "Decode tree state failed: node id/node length mismatch", + )); + } + let node_ids = encoded + .node_ids + .iter() + .map(|x| { + Ok(TreeID::new( + decode_peer_from_table(&peers, x.peer_idx, "Decode tree state failed")?, + decode_counter(x.counter, "Decode tree state failed")?, + )) + }) + .collect::>>()?; + for (node_id, node) in node_ids.iter().zip(encoded.nodes) { + // PERF: we don't need to mov the deleted node, instead we can cache them + // If the parent is TreeParentId::Deleted, then all the nodes afterwards are deleted + tree._init_push_tree_node_in_order( + *node_id, + match node.parent_idx_plus_two { + 0 => TreeParentId::Root, + 1 => TreeParentId::Deleted, + n => { + let id = *node_ids.get(n - 2).ok_or_else(|| { + state_decode_error( + "Decode tree state failed: parent index out of range", + ) + })?; + TreeParentId::from(Some(id)) + } + }, + IdFull::new( + decode_peer_from_table( + &peers, + node.last_set_peer_idx, + "Decode tree state failed", + )?, + node.last_set_counter, + decode_lamport_from_delta( + node.last_set_counter, + node.last_set_lamport_sub_counter, + "Decode tree state failed", + )?, + ), + Some(FractionalIndex::from_bytes( + fractional_indexes + .get(node.fractional_index_idx) + .ok_or_else(|| { + state_decode_error( + "Decode tree state failed: fractional index out of range", + ) + })? + .clone(), + )), + ) + .map_err(|err| { + state_decode_error(format!("Decode tree state failed: invalid node: {err}")) + })?; + } + + Ok(tree) + } + } + + #[cfg(test)] + mod tests { + use loro_common::{ContainerType, LoroValue}; + + use crate::{container::idx::ContainerIdx, state::ContainerCreationContext}; + + use super::*; + + #[test] + fn tree_fast_snapshot_rejects_corrupt_state_metadata() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Tree); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + assert!(TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &[1]), ctx).is_err()); + } + + #[test] + fn tree_fast_snapshot_rejects_negative_node_counter() { + let idx = ContainerIdx::from_index_and_type(0, ContainerType::Tree); + let configure = Default::default(); + let ctx = ContainerCreationContext { + configure: &configure, + peer: 0, + }; + + let position = fractional_index::FractionalIndex::default(); + let positions = PositionArena::from_positions([position.as_bytes()]); + let encoded = EncodedTree { + node_ids: vec![EncodedTreeNodeId { + peer_idx: 0, + counter: -1, + }], + nodes: vec![EncodedTreeNode { + parent_idx_plus_two: 0, + last_set_peer_idx: 0, + last_set_counter: 0, + last_set_lamport_sub_counter: 0, + fractional_index_idx: 0, + }], + fractional_indexes: positions.encode().into(), + reserved_has_effect_bool_rle: vec![].into(), + }; + + let mut bytes = Vec::new(); + leb128::write::unsigned(&mut bytes, 1).unwrap(); + bytes.extend_from_slice(&1_u64.to_le_bytes()); + bytes.extend_from_slice(&serde_columnar::to_vec(&encoded).unwrap()); + + assert!(TreeState::decode_snapshot_fast(idx, (LoroValue::Null, &bytes), ctx).is_err()); + } + } +} diff --git a/vendor/loro-internal/src/state/unknown_state.rs b/vendor/loro-internal/src/state/unknown_state.rs new file mode 100644 index 00000000..d3b11139 --- /dev/null +++ b/vendor/loro-internal/src/state/unknown_state.rs @@ -0,0 +1,176 @@ +use std::sync::Weak; + +use loro_common::{ContainerID, LoroResult, LoroValue}; + +use crate::{ + configure::Configure, + container::idx::ContainerIdx, + event::{Diff, Index, InternalDiff}, + op::{Op, RawOp}, + LoroDocInner, +}; + +use super::{ApplyLocalOpReturn, ContainerState, DiffApplyContext}; + +#[derive(Debug, Clone)] +pub struct UnknownState { + idx: ContainerIdx, +} + +impl UnknownState { + pub fn new(idx: ContainerIdx) -> Self { + Self { idx } + } +} + +impl ContainerState for UnknownState { + fn container_idx(&self) -> ContainerIdx { + self.idx + } + + fn is_state_empty(&self) -> bool { + false + } + + fn apply_diff_and_convert(&mut self, _diff: InternalDiff, _ctx: DiffApplyContext) -> Diff { + unreachable!() + } + + fn apply_diff(&mut self, _diff: InternalDiff, _ctx: DiffApplyContext) -> LoroResult<()> { + unreachable!() + } + + fn apply_local_op(&mut self, _raw_op: &RawOp, _op: &Op) -> LoroResult { + unreachable!() + } + + #[doc = r" Convert a state to a diff, such that an empty state will be transformed into the same as this state when it's applied."] + fn to_diff(&mut self, _doc: &Weak) -> Diff { + Diff::Unknown + } + + fn get_value(&mut self) -> LoroValue { + unreachable!() + } + + #[doc = r" Get the index of the child container"] + #[allow(unused)] + fn get_child_index(&self, id: &ContainerID) -> Option { + None + } + + fn contains_child(&self, _id: &ContainerID) -> bool { + false + } + + #[allow(unused)] + fn get_child_containers(&self) -> Vec { + vec![] + } + + fn fork(&self, _config: &Configure) -> Self { + self.clone() + } +} + +mod snapshot { + use loro_common::LoroValue; + + use crate::state::FastStateSnapshot; + + use super::UnknownState; + + impl FastStateSnapshot for UnknownState { + fn encode_snapshot_fast(&mut self, _w: W) {} + + fn decode_value(bytes: &[u8]) -> loro_common::LoroResult<(loro_common::LoroValue, &[u8])> { + Ok((LoroValue::Null, bytes)) + } + + fn decode_snapshot_fast( + idx: crate::container::idx::ContainerIdx, + _v: (loro_common::LoroValue, &[u8]), + _ctx: crate::state::ContainerCreationContext, + ) -> loro_common::LoroResult + where + Self: Sized, + { + Ok(UnknownState::new(idx)) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Weak; + + use loro_common::{ContainerID, ContainerType}; + + use crate::{ + configure::Configure, + container::idx::ContainerIdx, + event::Diff, + state::{ContainerCreationContext, ContainerState, FastStateSnapshot}, + InternalString, + }; + + use super::UnknownState; + + fn unknown_idx() -> ContainerIdx { + ContainerIdx::from_index_and_type(7, ContainerType::Unknown(3)) + } + + #[test] + fn unknown_state_reports_identity_without_material_value() { + let mut state = UnknownState::new(unknown_idx()); + assert_eq!(state.container_idx(), unknown_idx()); + assert!(!state.is_state_empty()); + + assert!(matches!(state.to_diff(&Weak::new()), Diff::Unknown)); + assert_eq!(state.get_child_containers(), Vec::::new()); + assert!(!state.contains_child(&ContainerID::Root { + name: InternalString::from("child"), + container_type: ContainerType::Text, + })); + assert_eq!( + state.get_child_index(&ContainerID::Root { + name: InternalString::from("child"), + container_type: ContainerType::Text, + }), + None + ); + } + + #[test] + fn unknown_state_fork_keeps_container_identity() { + let state = UnknownState::new(unknown_idx()); + let forked = state.fork(&Configure::default()); + + assert_eq!(forked.container_idx(), unknown_idx()); + assert!(!forked.is_state_empty()); + } + + #[test] + fn unknown_snapshot_roundtrip_preserves_bytes_and_index() { + let mut state = UnknownState::new(unknown_idx()); + let mut encoded = Vec::new(); + state.encode_snapshot_fast(&mut encoded); + assert!(encoded.is_empty()); + + let input = b"opaque-unknown-payload"; + let (value, rest) = UnknownState::decode_value(input).unwrap(); + assert_eq!(value, loro_common::LoroValue::Null); + assert_eq!(rest, input); + + let decoded = UnknownState::decode_snapshot_fast( + unknown_idx(), + (value, rest), + ContainerCreationContext { + configure: &Configure::default(), + peer: 0, + }, + ) + .unwrap(); + assert_eq!(decoded.container_idx(), unknown_idx()); + } +} diff --git a/vendor/loro-internal/src/subscription.rs b/vendor/loro-internal/src/subscription.rs new file mode 100644 index 00000000..71138334 --- /dev/null +++ b/vendor/loro-internal/src/subscription.rs @@ -0,0 +1,222 @@ +use super::{ + arena::SharedArena, + event::{DiffEvent, DocDiff}, +}; +use crate::{ + container::idx::ContainerIdx, utils::subscription::SubscriberSet, ContainerDiff, LoroDoc, + Subscription, +}; +use loro_common::{ContainerID, ID}; +use rustc_hash::FxHashMap; +use smallvec::SmallVec; +use std::{collections::VecDeque, sync::Arc}; + +use crate::sync::Mutex; +/// The callback of the local update. +pub type LocalUpdateCallback = Box) -> bool + Send + Sync + 'static>; +/// The callback of the peer id change. The second argument is the next counter for the peer. +pub type PeerIdUpdateCallback = Box bool + Send + Sync + 'static>; +#[allow(clippy::unused_unit)] +pub type Subscriber = Arc Fn(DiffEvent<'a>) + Send + Sync>; + +impl LoroDoc { + /// Subscribe to the changes of the peer id. + pub fn subscribe_peer_id_change(&self, callback: PeerIdUpdateCallback) -> Subscription { + let (s, enable) = self.peer_id_change_subs.inner().insert((), callback); + enable(); + s + } +} + +struct ObserverInner { + subscriber_set: SubscriberSet, Subscriber>, + queue: Arc>>, +} + +impl Default for ObserverInner { + fn default() -> Self { + Self { + subscriber_set: SubscriberSet::new(), + queue: Arc::new(Mutex::new(VecDeque::new())), + } + } +} + +pub struct Observer { + inner: ObserverInner, + arena: SharedArena, +} + +impl std::fmt::Debug for Observer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Observer").finish() + } +} + +impl Observer { + pub fn new(arena: SharedArena) -> Self { + Self { + arena, + inner: ObserverInner::default(), + } + } + + pub fn subscribe(&self, id: &ContainerID, callback: Subscriber) -> Subscription { + let idx = self.arena.register_container(id); + let inner = &self.inner; + let (sub, enable) = inner.subscriber_set.insert(Some(idx), callback); + enable(); + sub + } + + pub fn subscribe_root(&self, callback: Subscriber) -> Subscription { + let inner = &self.inner; + let (sub, enable) = inner.subscriber_set.insert(None, callback); + enable(); + sub + } + + pub(crate) fn emit(&self, doc_diff: DocDiff) { + let success = self.emit_inner(doc_diff); + if success { + let mut e = self.inner.queue.lock().pop_front(); + while let Some(event) = e { + self.emit_inner(event); + e = self.inner.queue.lock().pop_front(); + } + } + } + + // When emitting changes, we need to make sure that the observer is not locked. + fn emit_inner(&self, doc_diff: DocDiff) -> bool { + let inner = &self.inner; + let mut container_events_map: FxHashMap> = + Default::default(); + for container_diff in doc_diff.diff.iter() { + self.arena + .with_ancestors(container_diff.idx, |ancestor, _| { + if inner.subscriber_set.may_include(&Some(ancestor)) { + container_events_map + .entry(ancestor) + .or_default() + .push(container_diff); + } + }); + } + + { + // Check whether we are calling events recursively. + // If so, push the event to the queue + if inner.subscriber_set.is_recursive_calling(&None) + || container_events_map + .keys() + .any(|x| inner.subscriber_set.is_recursive_calling(&Some(*x))) + { + drop(container_events_map); + inner.queue.lock().push_back(doc_diff); + return false; + } + } + + for (container_idx, container_diffs) in container_events_map { + inner + .subscriber_set + .retain(&Some(container_idx), &mut |callback| { + (callback)(DiffEvent { + current_target: Some(self.arena.get_container_id(container_idx).unwrap()), + events: &container_diffs, + event_meta: &doc_diff, + }); + true + }) + .unwrap(); + } + + let events: Vec<_> = doc_diff.diff.iter().collect(); + inner + .subscriber_set + .retain(&None, &mut |callback| { + (callback)(DiffEvent { + current_target: None, + events: &events, + event_meta: &doc_diff, + }); + true + }) + .unwrap(); + + true + } +} + +#[cfg(test)] +mod test { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use crate::{cursor::PosType, handler::HandlerTrait, LoroDoc}; + + #[test] + fn test_recursive_events() { + let loro = Arc::new(LoroDoc::new()); + let loro_cp = loro.clone(); + let count = Arc::new(AtomicUsize::new(0)); + let count_cp = Arc::clone(&count); + let _g = loro_cp.subscribe_root(Arc::new(move |_| { + count_cp.fetch_add(1, Ordering::SeqCst); + let mut txn = loro.txn().unwrap(); + let text = loro.get_text("id"); + if text.get_value().as_string().unwrap().len() > 10 { + return; + } + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + })); + + let loro = loro_cp; + let mut txn = loro.txn().unwrap(); + let text = loro.get_text("id"); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let count = count.load(Ordering::SeqCst); + assert!(count > 2, "{}", count); + } + + #[test] + fn unsubscribe() { + let loro = Arc::new(LoroDoc::new()); + let count = Arc::new(AtomicUsize::new(0)); + let count_cp = Arc::clone(&count); + let sub = loro.subscribe_root(Arc::new(move |_| { + count_cp.fetch_add(1, Ordering::SeqCst); + })); + + let text = loro.get_text("id"); + + assert_eq!(count.load(Ordering::SeqCst), 0); + { + let mut txn = loro.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + assert_eq!(count.load(Ordering::SeqCst), 1); + { + let mut txn = loro.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + assert_eq!(count.load(Ordering::SeqCst), 2); + sub.unsubscribe(); + { + let mut txn = loro.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + } + assert_eq!(count.load(Ordering::SeqCst), 2); + } +} diff --git a/vendor/loro-internal/src/sync.rs b/vendor/loro-internal/src/sync.rs new file mode 100644 index 00000000..3cc5f884 --- /dev/null +++ b/vendor/loro-internal/src/sync.rs @@ -0,0 +1,169 @@ +#[cfg(loom)] +pub use loom::thread; +#[cfg(not(loom))] +pub use std::thread; + +#[cfg(loom)] +mod raw { + pub use loom::sync::{ + LockResult, Mutex as RawMutex, MutexGuard, RwLock as RawRwLock, RwLockReadGuard, + RwLockWriteGuard, + }; +} +#[cfg(not(loom))] +mod raw { + pub use std::sync::{ + LockResult, Mutex as RawMutex, MutexGuard, RwLock as RawRwLock, RwLockReadGuard, + RwLockWriteGuard, + }; +} + +pub use raw::{MutexGuard, RwLockReadGuard, RwLockWriteGuard}; + +#[cfg(loom)] +pub use loom::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicU8, AtomicUsize}; +#[cfg(not(loom))] +pub use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, AtomicU8, AtomicUsize}; + +#[cfg(loom)] +pub(crate) use my_thread_local::ThreadLocal; +#[cfg(not(loom))] +pub(crate) use thread_local::ThreadLocal; + +fn expect_not_poisoned(result: raw::LockResult, lock_kind: &str) -> T { + result.unwrap_or_else(|_| panic!("poisoned {lock_kind}")) +} + +#[derive(Debug)] +pub struct Mutex { + inner: raw::RawMutex, +} + +impl Mutex { + pub fn new(value: T) -> Self { + Self { + inner: raw::RawMutex::new(value), + } + } +} + +impl Mutex { + pub fn lock(&self) -> MutexGuard<'_, T> { + self.lock_with_kind("mutex") + } + + pub(crate) fn lock_with_kind(&self, lock_kind: &str) -> MutexGuard<'_, T> { + expect_not_poisoned(self.inner.lock(), lock_kind) + } + + pub(crate) fn is_locked(&self) -> bool { + self.inner.try_lock().is_err() + } +} + +impl Default for Mutex { + fn default() -> Self { + Self::new(T::default()) + } +} + +#[derive(Debug)] +pub struct RwLock { + inner: raw::RawRwLock, +} + +impl RwLock { + pub fn new(value: T) -> Self { + Self { + inner: raw::RawRwLock::new(value), + } + } + + pub fn into_inner(self) -> T { + expect_not_poisoned(self.inner.into_inner(), "rwlock") + } +} + +impl RwLock { + pub fn read(&self) -> RwLockReadGuard<'_, T> { + expect_not_poisoned(self.inner.read(), "rwlock") + } + + pub fn write(&self) -> RwLockWriteGuard<'_, T> { + expect_not_poisoned(self.inner.write(), "rwlock") + } +} + +impl Default for RwLock { + fn default() -> Self { + Self::new(T::default()) + } +} + +#[cfg(loom)] +mod my_thread_local { + use std::sync::Arc; + + use super::thread; + use super::Mutex; + use rustc_hash::FxHashMap; + + #[derive(Debug)] + pub(crate) struct ThreadLocal { + content: Arc>>>, + } + + impl ThreadLocal { + pub fn new() -> Self { + Self { + content: Arc::new(Mutex::new(FxHashMap::default())), + } + } + + pub fn get_or_default(&self) -> Arc { + let mut content = self.content.lock(); + let v = content + .entry(thread::current().id()) + .or_insert_with(|| Arc::new(T::default())); + v.clone() + } + } + + impl Clone for ThreadLocal { + fn clone(&self) -> Self { + Self { + content: self.content.clone(), + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic(expected = "poisoned mutex")] + fn mutex_lock_panics_after_poison() { + let lock = Mutex::new(7); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = lock.lock(); + panic!("poison mutex"); + })); + + drop(lock.lock()); + } + + #[test] + #[should_panic(expected = "poisoned rwlock")] + fn rwlock_read_panics_after_poison() { + let lock = RwLock::new(7); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut guard = lock.write(); + *guard = 9; + panic!("poison rwlock"); + })); + + drop(lock.read()); + } +} diff --git a/vendor/loro-internal/src/tests/import_atomicity.rs b/vendor/loro-internal/src/tests/import_atomicity.rs new file mode 100644 index 00000000..0e1d8f65 --- /dev/null +++ b/vendor/loro-internal/src/tests/import_atomicity.rs @@ -0,0 +1,304 @@ +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use crate::{ + cursor::PosType, + encoding::json_schema::json::{JsonOpContent, JsonSchema, ListOp}, + loro::ExportMode, + state::fail_next_import_state_apply_for_test, + version::{Frontiers, VersionVector}, + LoroDoc, LoroError, TreeParentId, +}; + +fn pending_len(doc: &LoroDoc) -> usize { + doc.oplog().lock().pending_changes_len() +} + +fn corrupt_snapshot_state_bytes(snapshot: &mut [u8]) { + let body_start = 22; + let oplog_len = + u32::from_le_bytes(snapshot[body_start..body_start + 4].try_into().unwrap()) as usize; + let state_len_pos = body_start + 4 + oplog_len; + let state_len = u32::from_le_bytes( + snapshot[state_len_pos..state_len_pos + 4] + .try_into() + .unwrap(), + ) as usize; + assert!(state_len > 0); + let state_start = state_len_pos + 4; + snapshot[state_start] ^= 0xff; + + let checksum = xxhash_rust::xxh32::xxh32(&snapshot[20..], u32::from_le_bytes(*b"LORO")); + snapshot[16..20].copy_from_slice(&checksum.to_le_bytes()); +} + +fn make_json_list_update_with_four_ops(peer: u64) -> (LoroDoc, JsonSchema) { + let doc = LoroDoc::new(); + doc.set_peer_id(peer).unwrap(); + let map = doc.get_map("map"); + let list = doc.get_list("list"); + let text = doc.get_text("text"); + + let mut txn = doc.txn().unwrap(); + map.insert_with_txn(&mut txn, "prefix", "map-value".into()) + .unwrap(); + list.insert_with_txn(&mut txn, 0, "seed".into()).unwrap(); + text.insert_with_txn(&mut txn, 0, "text-value", PosType::Unicode) + .unwrap(); + list.insert_with_txn(&mut txn, 1, "tail".into()).unwrap(); + txn.commit().unwrap(); + + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false); + assert_eq!(json.changes.len(), 1); + assert_eq!(json.changes[0].ops.len(), 4); + (doc, json) +} + +fn move_last_list_insert_far_out_of_bounds(json: &mut JsonSchema) { + let last_change = json.changes.last_mut().unwrap(); + let last_op = last_change.ops.last_mut().unwrap(); + match &mut last_op.content { + JsonOpContent::List(ListOp::Insert { pos, .. }) => { + *pos = 1_000; + } + other => panic!("expected list insert op, got {other:?}"), + } +} + +fn make_multi_peer_frontier_doc() -> LoroDoc { + let base = LoroDoc::new_auto_commit(); + base.set_peer_id(1).unwrap(); + base.get_map("map").insert("base", 0).unwrap(); + base.get_text("text").insert_unicode(0, "base").unwrap(); + let tree = base.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + tree.get_meta(root).unwrap().insert("base", 0).unwrap(); + + let base_updates = base.export(ExportMode::all_updates()).unwrap(); + + let peer2 = base.fork(); + peer2.set_peer_id(2).unwrap(); + peer2.get_map("map").insert("p2", 2).unwrap(); + peer2.get_text("text").insert_unicode(0, "p2").unwrap(); + peer2.commit_then_renew(); + let peer2_updates = peer2.export(ExportMode::updates(&base.oplog_vv())).unwrap(); + + let peer3 = base.fork(); + peer3.set_peer_id(3).unwrap(); + peer3.get_map("map").insert("p3", 3).unwrap(); + let peer3_tree = peer3.get_tree("tree"); + let node = peer3_tree.create(TreeParentId::Root).unwrap(); + peer3_tree.get_meta(node).unwrap().insert("p3", 3).unwrap(); + peer3.commit_then_renew(); + let peer3_updates = peer3.export(ExportMode::updates(&base.oplog_vv())).unwrap(); + + let target = LoroDoc::new(); + target.import(&base_updates).unwrap(); + target.import(&peer2_updates).unwrap(); + target.import(&peer3_updates).unwrap(); + target +} + +fn assert_doc_unchanged( + doc: &LoroDoc, + vv: &VersionVector, + frontiers: &Frontiers, + state: &crate::LoroValue, +) { + assert_eq!(&doc.oplog_vv(), vv); + assert_eq!(&doc.oplog_frontiers(), frontiers); + assert_eq!(&doc.get_deep_value(), state); +} + +#[test] +fn failed_dependency_import_rolls_back_single_pending_change() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(1).unwrap(); + let map = src.get_map("map"); + map.insert("seed", "base").unwrap(); + let update_base = src + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = src.oplog_vv(); + + map.insert("later", "value").unwrap(); + let update_later = src.export(ExportMode::updates(&version_base)).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&update_later).unwrap(); + assert_eq!(pending_len(&dst), 1); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + + fail_next_import_state_apply_for_test(); + let err = dst.import(&update_base).unwrap_err(); + assert!( + err.to_string().contains("state apply failpoint"), + "unexpected error: {err:?}" + ); + assert_eq!(pending_len(&dst), 1); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + + dst.import(&update_base).unwrap(); + assert_eq!(pending_len(&dst), 0); + assert_eq!(dst.oplog_vv(), src.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), src.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); +} + +#[test] +fn failed_dependency_import_rolls_back_multiple_pending_changes() { + let base = LoroDoc::new_auto_commit(); + base.set_peer_id(1).unwrap(); + let map = base.get_map("map"); + map.insert("seed", "base").unwrap(); + let update_base = base + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = base.oplog_vv(); + + let peer2 = LoroDoc::new_auto_commit(); + peer2.set_peer_id(2).unwrap(); + peer2.import(&update_base).unwrap(); + peer2.get_map("map").insert("p2", "B").unwrap(); + let update_peer2 = peer2.export(ExportMode::updates(&version_base)).unwrap(); + + let peer3 = LoroDoc::new_auto_commit(); + peer3.set_peer_id(3).unwrap(); + peer3.import(&update_base).unwrap(); + peer3.get_map("map").insert("p3", "C").unwrap(); + let update_peer3 = peer3.export(ExportMode::updates(&version_base)).unwrap(); + + let expected = LoroDoc::new(); + expected.import(&update_base).unwrap(); + expected.import(&update_peer2).unwrap(); + expected.import(&update_peer3).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&update_peer2).unwrap(); + dst.import(&update_peer3).unwrap(); + assert_eq!(pending_len(&dst), 2); + + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import: Frontiers = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + + fail_next_import_state_apply_for_test(); + let err = dst.import(&update_base).unwrap_err(); + assert!( + err.to_string().contains("state apply failpoint"), + "unexpected error: {err:?}" + ); + assert_eq!(pending_len(&dst), 2); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + + dst.import(&update_base).unwrap(); + assert_eq!(pending_len(&dst), 0); + assert_eq!(dst.oplog_vv(), expected.oplog_vv()); + assert_eq!(dst.oplog_frontiers(), expected.oplog_frontiers()); + assert_eq!(dst.get_deep_value(), expected.get_deep_value()); +} + +#[test] +fn failed_import_keeps_multi_peer_frontiers_intact() { + let target = make_multi_peer_frontier_doc(); + let vv_before_import = target.oplog_vv(); + assert!(vv_before_import.iter().count() >= 3); + let frontiers_before_import = target.oplog_frontiers(); + let state_before_import = target.get_deep_value(); + + let (_, mut bad_json) = make_json_list_update_with_four_ops(4); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + let bad_json = serde_json::to_string(&bad_json).unwrap(); + + let err = target.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(target.oplog_vv(), vv_before_import); + assert_eq!(target.oplog_frontiers(), frontiers_before_import); + assert_eq!(target.get_deep_value(), state_before_import); +} + +#[test] +fn malformed_json_import_returns_error_without_mutating_doc() { + let doc = make_multi_peer_frontier_doc(); + let vv_before_import = doc.oplog_vv(); + let frontiers_before_import = doc.oplog_frontiers(); + let state_before_import = doc.get_deep_value(); + + let err = doc + .import_json_updates("[3,{ \"(' k\" :\n\n42222 }]") + .unwrap_err(); + assert_eq!(err, LoroError::InvalidJsonSchema); + assert_doc_unchanged( + &doc, + &vv_before_import, + &frontiers_before_import, + &state_before_import, + ); +} + +#[test] +fn failed_import_does_not_emit_events() { + let doc = LoroDoc::new(); + let hit = Arc::new(AtomicUsize::new(0)); + let hit_cloned = hit.clone(); + let _sub = doc.subscribe_root(Arc::new(move |_event| { + hit_cloned.fetch_add(1, Ordering::SeqCst); + })); + + let (_, mut bad_json) = make_json_list_update_with_four_ops(7); + move_last_list_insert_far_out_of_bounds(&mut bad_json); + let bad_json = serde_json::to_string(&bad_json).unwrap(); + let err = doc.import_json_updates(&bad_json).unwrap_err(); + assert!( + err.to_string().contains("list diff"), + "expected state list bounds validation, got {err:?}" + ); + assert_eq!(hit.load(Ordering::SeqCst), 0); + assert!(doc.drop_pending_events().is_empty()); + + let (_, good_json) = make_json_list_update_with_four_ops(8); + let good_json = serde_json::to_string(&good_json).unwrap(); + doc.import_json_updates(&good_json).unwrap(); + assert!(hit.load(Ordering::SeqCst) > 0); +} + +#[test] +fn corrupt_snapshot_import_rolls_back_empty_doc() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(9).unwrap(); + src.get_text("text").insert_unicode(0, "snapshot").unwrap(); + src.get_list("list").push("value").unwrap(); + let snapshot = src.export(ExportMode::Snapshot).unwrap(); + let mut corrupt_snapshot = snapshot.clone(); + corrupt_snapshot_state_bytes(&mut corrupt_snapshot); + + let dst = LoroDoc::new(); + let vv_before_import = dst.oplog_vv(); + let frontiers_before_import = dst.oplog_frontiers(); + let state_before_import = dst.get_deep_value(); + let err = dst.import(&corrupt_snapshot).unwrap_err(); + assert!( + err.to_string().contains("decode_snapshot") + || err.to_string().contains("Decode") + || err.to_string().contains("snapshot"), + "unexpected error: {err:?}" + ); + assert_eq!(dst.oplog_vv(), vv_before_import); + assert_eq!(dst.oplog_frontiers(), frontiers_before_import); + assert_eq!(dst.get_deep_value(), state_before_import); + assert!(dst.oplog().lock().is_empty()); + + dst.import(&snapshot).unwrap(); + assert_eq!(dst.get_deep_value(), src.get_deep_value()); +} diff --git a/vendor/loro-internal/src/tests/mod.rs b/vendor/loro-internal/src/tests/mod.rs new file mode 100644 index 00000000..83923c87 --- /dev/null +++ b/vendor/loro-internal/src/tests/mod.rs @@ -0,0 +1,36 @@ +#[ctor::ctor] +fn init() { + dev_utils::setup_test_log(); +} + +mod import_atomicity; +mod op_count; + +use crate::{op::ListSlice, LoroValue}; + +pub const PROPTEST_FACTOR_10: usize = 1; +pub const PROPTEST_FACTOR_1: usize = 0; + +#[test] +fn size_of() { + use crate::change::Change; + use crate::{ + container::{map::MapSet, ContainerID}, + id::ID, + op::{Op, RawOpContent}, + span::IdSpan, + InternalString, + }; + + println!("Change {}", std::mem::size_of::()); + println!("Op {}", std::mem::size_of::()); + println!("InsertContent {}", std::mem::size_of::()); + println!("MapSet {}", std::mem::size_of::()); + println!("ListSlice {}", std::mem::size_of::()); + println!("LoroValue {}", std::mem::size_of::()); + println!("ID {}", std::mem::size_of::()); + println!("Vec {}", std::mem::size_of::>()); + println!("IdSpan {}", std::mem::size_of::()); + println!("ContainerID {}", std::mem::size_of::()); + println!("InternalString {}", std::mem::size_of::()); +} diff --git a/vendor/loro-internal/src/tests/op_count.rs b/vendor/loro-internal/src/tests/op_count.rs new file mode 100644 index 00000000..1980d300 --- /dev/null +++ b/vendor/loro-internal/src/tests/op_count.rs @@ -0,0 +1,107 @@ +use crate::{ + loro::ExportMode, state::fail_next_import_state_apply_for_test, version::VersionVector, LoroDoc, +}; +use std::sync::atomic::Ordering::{Acquire, Release}; + +fn assert_visible_op_count_cache_matches_exact(doc: &LoroDoc) { + let exact = doc.oplog().lock().visible_op_count_exact(); + assert_eq!(doc.visible_op_count.load(Acquire), exact); + + let _state = doc.app_state().lock(); + assert_eq!(doc.len_ops(), exact); +} + +#[test] +fn len_ops_is_available_while_state_is_locked_with_local_ops() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + doc.get_map("map").insert("k", 1).unwrap(); + + let _state = doc.app_state().lock(); + assert_eq!(doc.len_ops(), 1); +} + +#[test] +fn len_ops_tracks_snapshot_import_while_state_is_locked() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(2).unwrap(); + src.get_map("map").insert("k", 1).unwrap(); + src.commit_then_renew(); + let expected = src.len_ops(); + let snapshot = src.export(ExportMode::Snapshot).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&snapshot).unwrap(); + + let _state = dst.app_state().lock(); + assert_eq!(dst.len_ops(), expected); +} + +#[test] +fn len_ops_prefers_exact_oplog_value_when_lock_order_allows() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(3).unwrap(); + doc.get_map("map").insert("k", 1).unwrap(); + + doc.visible_op_count.store(0, Release); + assert_eq!(doc.len_ops(), 1); +} + +#[test] +fn visible_op_count_cache_tracks_pending_activation() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(4).unwrap(); + let map = src.get_map("map"); + map.insert("base", "base").unwrap(); + let update_base = src + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = src.oplog_vv(); + + map.insert("later", "later").unwrap(); + let update_later = src.export(ExportMode::updates(&version_base)).unwrap(); + + let dst = LoroDoc::new(); + let status = dst.import(&update_later).unwrap(); + assert!(status.pending.is_some()); + assert_visible_op_count_cache_matches_exact(&dst); + assert_eq!(dst.len_ops(), 0); + + let status = dst.import(&update_base).unwrap(); + assert!(status.pending.is_none()); + assert_eq!(dst.len_ops(), src.len_ops()); + assert_visible_op_count_cache_matches_exact(&dst); +} + +#[test] +fn visible_op_count_cache_rolls_back_after_failed_pending_activation() { + let src = LoroDoc::new_auto_commit(); + src.set_peer_id(5).unwrap(); + let map = src.get_map("map"); + map.insert("base", "base").unwrap(); + let update_base = src + .export(ExportMode::updates(&VersionVector::default())) + .unwrap(); + let version_base = src.oplog_vv(); + + map.insert("later", "later").unwrap(); + let update_later = src.export(ExportMode::updates(&version_base)).unwrap(); + + let dst = LoroDoc::new(); + dst.import(&update_later).unwrap(); + assert_eq!(dst.len_ops(), 0); + assert_visible_op_count_cache_matches_exact(&dst); + + fail_next_import_state_apply_for_test(); + let err = dst.import(&update_base).unwrap_err(); + assert!( + err.to_string().contains("state apply failpoint"), + "unexpected error: {err:?}" + ); + assert_eq!(dst.len_ops(), 0); + assert_visible_op_count_cache_matches_exact(&dst); + + dst.import(&update_base).unwrap(); + assert_eq!(dst.len_ops(), src.len_ops()); + assert_visible_op_count_cache_matches_exact(&dst); +} diff --git a/vendor/loro-internal/src/txn.rs b/vendor/loro-internal/src/txn.rs new file mode 100644 index 00000000..3d8ee4ad --- /dev/null +++ b/vendor/loro-internal/src/txn.rs @@ -0,0 +1,1435 @@ +use std::{ + borrow::Cow, + mem::take, + sync::{Arc, Weak}, +}; + +use enum_as_inner::EnumAsInner; +use generic_btree::rle::{HasLength as RleHasLength, Mergeable as GBSliceable}; +use loro_common::{ContainerType, IdLp, IdSpan, LoroResult}; +use loro_delta::{array_vec::ArrayVec, DeltaRopeBuilder}; +use rle::{HasLength, Mergable, RleVec, Sliceable}; +use rustc_hash::FxHashMap; +use smallvec::{smallvec, SmallVec}; + +use crate::{ + change::{Change, Lamport, Timestamp}, + container::{ + idx::ContainerIdx, + list::list_op::{DeleteSpan, InnerListOp}, + richtext::Style, + IntoContainerId, + }, + delta::{ResolvedMapDelta, ResolvedMapValue, StyleMeta, StyleMetaItem, TreeDiff, TreeDiffItem}, + encoding::export_fast_updates_in_range, + event::{Diff, ListDeltaMeta, TextDiff}, + handler::{Handler, ValueOrHandler}, + id::{Counter, PeerID, ID}, + lock::{LoroMutex, LoroMutexGuard}, + loro::CommitOptions, + op::{Op, RawOp, RawOpContent}, + pre_commit::{ChangeModifier, PreCommitCallbackPayload}, + span::HasIdSpan, + version::Frontiers, + ChangeMeta, InternalString, LoroDoc, LoroDocInner, LoroError, LoroValue, +}; + +use super::{ + arena::SharedArena, + event::{InternalContainerDiff, InternalDocDiff}, + handler::{ListHandler, MapHandler, TextHandler, TreeHandler}, + oplog::OpLog, + state::DocState, +}; + +impl crate::LoroDoc { + /// Create a new transaction. + /// Every ops created inside one transaction will be packed into a single + /// [Change]. + /// + /// There can only be one active transaction at a time for a [LoroDoc]. + #[inline(always)] + pub fn txn(&self) -> Result { + self.txn_with_origin("") + } + + /// Create a new transaction with specified origin. + /// + /// The origin will be propagated to the events. + /// There can only be one active transaction at a time for a [LoroDoc]. + pub fn txn_with_origin(&self, origin: &str) -> Result { + if !self.can_edit() { + return Err(LoroError::TransactionError( + String::from("LoroDoc is in readonly detached mode. To make it writable in detached mode, call `set_detached_editing(true)`.").into_boxed_str(), + )); + } + + let mut txn = Transaction::new_with_origin(self.inner.clone(), origin.into())?; + + let obs = self.observer.clone(); + let local_update_subs_weak = self.local_update_subs.downgrade(); + txn.set_on_commit(Box::new(move |state, oplog, id_span| { + let mut state = state.lock(); + let events = state.take_events(); + drop(state); + for event in events { + obs.emit(event); + } + + if id_span.atom_len() == 0 { + return; + } + + if let Some(local_update_subs) = local_update_subs_weak.upgrade() { + if !local_update_subs.inner().is_empty() { + let bytes = { export_fast_updates_in_range(&oplog.lock(), &[id_span]) }; + local_update_subs.emit(&(), bytes); + } + } + })); + + Ok(txn) + } + + pub fn start_auto_commit(&self) { + self.auto_commit + .store(true, std::sync::atomic::Ordering::Release); + let mut self_txn = self.txn.lock(); + if self_txn.is_some() || !self.can_edit() { + return; + } + + let txn = self + .txn() + .expect("auto-commit should be able to create a transaction"); + self_txn.replace(txn); + } + + #[inline] + pub fn renew_txn_if_auto_commit(&self, options: Option) { + if self.auto_commit.load(std::sync::atomic::Ordering::Acquire) && self.can_edit() { + let mut self_txn = self.txn.lock(); + if self_txn.is_some() { + return; + } + + let mut txn = self + .txn() + .expect("auto-commit should be able to renew a transaction"); + if let Some(options) = options { + txn.set_options(options); + } + self_txn.replace(txn); + } + } + + #[inline] + pub(crate) fn _renew_txn_if_auto_commit_with_guard( + &self, + options: Option, + mut guard: LoroMutexGuard>, + ) { + if self.auto_commit.load(std::sync::atomic::Ordering::Acquire) && self.can_edit() { + if guard.is_some() { + return; + } + + let mut txn = self + .txn() + .expect("auto-commit should be able to renew a transaction"); + if let Some(options) = options { + txn.set_options(options); + } + guard.replace(txn); + } + } +} + +pub(crate) type OnCommitFn = + Box>, &Arc>, IdSpan) + Sync + Send>; + +pub struct Transaction { + peer: PeerID, + origin: InternalString, + start_counter: Counter, + next_counter: Counter, + start_lamport: Lamport, + next_lamport: Lamport, + doc: Weak, + frontiers: Frontiers, + local_ops: RleVec<[Op; 1]>, // TODO: use a more efficient data structure + event_hints: FxHashMap>, + pub(super) arena: SharedArena, + finished: bool, + on_commit: Option, + timestamp: Option, + msg: Option>, + latest_timestamp: Timestamp, + pub(super) is_peer_first_appearance: bool, +} + +impl std::fmt::Debug for Transaction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Transaction") + .field("peer", &self.peer) + .field("origin", &self.origin) + .field("start_counter", &self.start_counter) + .field("next_counter", &self.next_counter) + .field("start_lamport", &self.start_lamport) + .field("next_lamport", &self.next_lamport) + .field("frontiers", &self.frontiers) + .field("local_ops", &self.local_ops) + .field("event_hints", &self.event_hints) + .field("arena", &self.arena) + .field("finished", &self.finished) + .field("on_commit", &self.on_commit.is_some()) + .field("timestamp", &self.timestamp) + .finish() + } +} + +/// We can infer local events directly from the local behavior. This enum is used to +/// record them, so that we can avoid recalculate them when we commit the transaction. +/// +/// For example, when we insert a text in wasm, users use the utf16 index to send the +/// command. However, internally loro will convert it to unicode index. But the users +/// still need events that are in utf16 index. To avoid the round trip, we record the +/// events here. +#[derive(Debug, Clone, EnumAsInner)] +pub(super) enum EventHint { + Mark { + start: u32, + end: u32, + style: Style, + }, + InsertText { + /// pos is a Unicode index. If wasm, it's a UTF-16 index. + pos: u32, + event_len: u32, + unicode_len: u32, + styles: StyleMeta, + }, + /// pos is a Unicode index. If wasm, it's a UTF-16 index. + DeleteText { + span: DeleteSpan, + unicode_len: usize, + }, + InsertList { + len: u32, + pos: usize, + }, + SetList { + index: usize, + value: LoroValue, + }, + Move { + value: LoroValue, + from: u32, + to: u32, + }, + DeleteList(DeleteSpan), + Map { + key: InternalString, + value: Option, + }, + // use vec because we could bring back some node that has children + Tree(SmallVec<[TreeDiffItem; 1]>), + MarkEnd, + #[cfg(feature = "counter")] + Counter(f64), +} + +impl generic_btree::rle::HasLength for EventHint { + fn rle_len(&self) -> usize { + match self { + EventHint::Mark { .. } => 1, + EventHint::InsertText { + unicode_len: len, .. + } => *len as usize, + EventHint::DeleteText { unicode_len, .. } => *unicode_len, + EventHint::InsertList { len, .. } => *len as usize, + EventHint::DeleteList(d) => d.len(), + EventHint::Map { .. } => 1, + EventHint::Tree(_) => 1, + EventHint::MarkEnd => 1, + EventHint::Move { .. } => 1, + EventHint::SetList { .. } => 1, + #[cfg(feature = "counter")] + EventHint::Counter(_) => 1, + } + } +} + +impl generic_btree::rle::Mergeable for EventHint { + fn can_merge(&self, rhs: &Self) -> bool { + match (self, rhs) { + ( + EventHint::InsertText { + pos, + unicode_len: _, + event_len, + styles, + }, + EventHint::InsertText { + pos: r_pos, + styles: r_styles, + .. + }, + ) => *pos + *event_len == *r_pos && styles == r_styles, + (EventHint::InsertList { pos, len }, EventHint::InsertList { pos: pos_right, .. }) => { + pos + *len as usize == *pos_right + } + // We don't merge delete text because it's hard to infer the correct pos to split: + // `range` param is in unicode range, but the delete text event is in UTF-16 range. + // Without the original text, it's impossible to convert the range. + (EventHint::DeleteText { span, .. }, EventHint::DeleteText { span: r, .. }) => { + span.is_mergable(r, &()) + } + (EventHint::DeleteList(l), EventHint::DeleteList(r)) => l.is_mergable(r, &()), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + match (self, rhs) { + ( + EventHint::InsertText { + event_len, + unicode_len: len, + .. + }, + EventHint::InsertText { + event_len: r_event_len, + unicode_len: r_len, + .. + }, + ) => { + *len += *r_len; + *event_len += *r_event_len; + } + ( + EventHint::InsertList { len, pos: _ }, + EventHint::InsertList { len: r_len, pos: _ }, + ) => *len += *r_len, + (EventHint::DeleteList(l), EventHint::DeleteList(r)) => l.merge(r, &()), + ( + EventHint::DeleteText { span, unicode_len }, + EventHint::DeleteText { + span: r_span, + unicode_len: r_len, + }, + ) => { + *unicode_len += *r_len; + span.merge(r_span, &()); + } + _ => unreachable!(), + } + } + + fn merge_left(&mut self, _: &Self) { + unreachable!() + } +} + +impl Transaction { + #[inline] + pub fn new(doc: Arc) -> LoroResult { + Self::new_with_origin(doc.clone(), "".into()) + } + + pub fn new_with_origin(doc: Arc, origin: InternalString) -> LoroResult { + let oplog_lock = doc.oplog.lock(); + let mut state_lock = doc.state.lock(); + if state_lock.is_in_txn() { + return Err(LoroError::DuplicatedTransactionError); + } + + state_lock.start_txn(origin, crate::event::EventTriggerKind::Local); + let arena = state_lock.arena.clone(); + let frontiers = state_lock.frontiers.clone(); + let peer = state_lock.peer.load(std::sync::atomic::Ordering::Relaxed); + let next_counter = oplog_lock.next_id(peer).counter; + let next_lamport = oplog_lock.dag.frontiers_to_next_lamport(&frontiers); + let latest_timestamp = oplog_lock.get_greatest_timestamp(&frontiers); + if let Err(err) = + oplog_lock.check_change_greater_than_last_peer_id(peer, next_counter, &frontiers) + { + state_lock.abort_txn(); + return Err(err); + } + drop(state_lock); + drop(oplog_lock); + Ok(Self { + peer, + doc: Arc::downgrade(&doc), + arena, + frontiers, + timestamp: None, + next_counter, + next_lamport, + origin: Default::default(), + start_counter: next_counter, + start_lamport: next_lamport, + event_hints: Default::default(), + local_ops: RleVec::new(), + finished: false, + on_commit: None, + msg: None, + latest_timestamp, + is_peer_first_appearance: false, + }) + } + + pub fn set_origin(&mut self, origin: InternalString) { + self.origin = origin; + } + + pub fn set_timestamp(&mut self, time: Timestamp) { + self.timestamp = Some(time); + } + + pub fn set_msg(&mut self, msg: Option>) { + self.msg = msg; + } + + pub fn local_ops(&self) -> &RleVec<[Op; 1]> { + &self.local_ops + } + + pub fn peer(&self) -> &PeerID { + &self.peer + } + + pub fn timestamp(&self) -> &Option { + &self.timestamp + } + + pub fn frontiers(&self) -> &Frontiers { + &self.frontiers + } + pub fn msg(&self) -> &Option> { + &self.msg + } + + pub fn lamport(&self) -> &Lamport { + &self.start_lamport + } + + pub(crate) fn set_on_commit(&mut self, f: OnCommitFn) { + self.on_commit = Some(f); + } + + pub(crate) fn take_on_commit(&mut self) -> Option { + self.on_commit.take() + } + + pub fn commit(mut self) -> Result, LoroError> { + self._commit() + } + + #[tracing::instrument(level = "debug", skip(self))] + fn _commit(&mut self) -> Result, LoroError> { + if self.finished { + return Ok(None); + } + + let Some(doc) = self.doc.upgrade() else { + return Ok(None); + }; + self.finished = true; + if self.local_ops.is_empty() { + let mut state = doc.state.lock(); + state.abort_txn(); + return Ok(Some(self.take_options())); + } + + let ops = std::mem::take(&mut self.local_ops); + let deps = take(&mut self.frontiers); + let change = Change { + lamport: self.start_lamport, + ops, + deps, + id: ID::new(self.peer, self.start_counter), + timestamp: self.latest_timestamp.max( + self.timestamp + .unwrap_or_else(|| doc.oplog.lock().get_timestamp_for_next_txn()), + ), + commit_msg: take(&mut self.msg), + }; + + let change_meta = ChangeMeta::from_change(&change); + { + // add change to uncommit field of oplog + let mut oplog = doc.oplog.lock(); + oplog.set_uncommitted_change(change); + } + + let modifier = ChangeModifier::default(); + doc.pre_commit_subs.emit( + &(), + PreCommitCallbackPayload { + change_meta, + origin: self.origin.to_string(), + modifier: modifier.clone(), + }, + ); + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + + let Some(mut change) = oplog.uncommitted_change.take() else { + state.abort_txn(); + drop(state); + drop(oplog); + return Err(LoroError::internal( + "missing uncommitted change while committing transaction", + )); + }; + modifier.modify_change(&mut change); + let diff = if state.is_recording() { + Some(change_to_diff( + &change, + doc.clone(), + std::mem::take(&mut self.event_hints), + )) + } else { + None + }; + + let last_id = change.id_last(); + if let Err(err) = oplog.import_local_change(change) { + state.abort_txn(); + drop(state); + drop(oplog); + return Err(err); + } + + state.commit_txn( + Frontiers::from_id(last_id), + diff.map(|arr| InternalDocDiff { + by: crate::event::EventTriggerKind::Local, + origin: self.origin.clone(), + diff: Cow::Owned( + arr.into_iter() + .map(|x| InternalContainerDiff { + idx: x.idx, + bring_back: false, + diff: (x.diff.into()), + diff_mode: crate::diff_calc::DiffMode::Linear, + }) + .collect(), + ), + new_version: Cow::Borrowed(oplog.frontiers()), + }), + ); + drop(state); + drop(oplog); + if let Some(on_commit) = self.on_commit.take() { + assert!(!doc.txn.is_locked()); + on_commit(&doc.state.clone(), &doc.oplog.clone(), self.id_span()); + } + Ok(None) + } + + fn take_options(&self) -> CommitOptions { + let mut options = CommitOptions::new(); + if !self.origin.is_empty() { + options = options.origin(self.origin.as_str()); + } + if let Some(msg) = self.msg.as_ref() { + options = options.commit_msg(msg); + } + if let Some(timestamp) = self.timestamp { + options = options.timestamp(timestamp); + } + options + } + + pub(super) fn apply_local_op( + &mut self, + container: ContainerIdx, + content: RawOpContent, + event: EventHint, + // check whether context and txn are referring to the same state context + doc: &LoroDoc, + ) -> LoroResult<()> { + // TODO: need to check if the doc is the same + let this_doc = self.doc.upgrade().unwrap(); + if Arc::as_ptr(&this_doc.state) != Arc::as_ptr(&doc.state) { + return Err(LoroError::UnmatchedContext { + expected: this_doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + found: doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + }); + } + + let len = content.content_len(); + assert!(len > 0); + let raw_op = RawOp { + id: ID { + peer: self.peer, + counter: self.next_counter, + }, + lamport: self.next_lamport, + container, + content, + }; + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + if state.is_deleted(container) { + return Err(LoroError::ContainerDeleted { + container: Box::new(state.arena.idx_to_id(container).unwrap()), + }); + } + + let op = self.arena.convert_raw_op(&raw_op); + state.apply_local_op(&raw_op, &op)?; + { + if !self.is_peer_first_appearance && !oplog.dag.latest_vv_contains_peer(self.peer) { + self.is_peer_first_appearance = true; + } + // update version info + let dep_id = Frontiers::from_id(ID::new(self.peer, self.next_counter - 1)); + let start_id = ID::new(self.peer, self.next_counter); + self.next_counter += len as Counter; + oplog.dag.update_version_on_new_local_op( + if self.local_ops.is_empty() { + &self.frontiers + } else { + &dep_id + }, + start_id, + self.next_lamport, + len, + ); + oplog.refresh_visible_op_count(); + self.next_lamport += len as Lamport; + // set frontiers to the last op id + let last_id = start_id.inc(len as Counter - 1); + state.frontiers = Frontiers::from_id(last_id); + }; + drop(state); + drop(oplog); + debug_assert_eq!( + event.rle_len(), + op.atom_len(), + "event:{:#?} \nop:{:#?}", + &event, + &op + ); + + let container_hints = self.event_hints.entry(container).or_default(); + + match container_hints.last_mut() { + Some(last) if last.can_merge(&event) => { + last.merge_right(&event); + } + _ => { + container_hints.push(event); + } + } + self.local_ops.push(op); + Ok(()) + } + + /// §perf (flowstate vendor patch, act three D.2): apply MANY local ops to + /// one container under ONE oplog/state lock acquisition, with ONE DAG + /// version update covering the whole consecutive counter run (the DAG + /// RLE-merges consecutive local spans, so one call with the total length + /// is equivalent to N per-op calls). A whole-document delete in a + /// heavily-styled document fans out into one op per text fragment between + /// style anchors (~128k on the reference doc); the per-op ceremony + /// (locking, DAG update, frontier rebuild) dominated the apply loop at + /// ~5 µs/op. STATE mutation stays strictly per-op — each op applies + /// exactly as a remote replay of the same ops would — so convergence + /// semantics are untouched. + pub(super) fn apply_local_ops_batch( + &mut self, + container: ContainerIdx, + ops: Vec<(RawOpContent, EventHint)>, + doc: &LoroDoc, + ) -> LoroResult<()> { + if ops.is_empty() { + return Ok(()); + } + let this_doc = self.doc.upgrade().unwrap(); + if Arc::as_ptr(&this_doc.state) != Arc::as_ptr(&doc.state) { + return Err(LoroError::UnmatchedContext { + expected: this_doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + found: doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + }); + } + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + if state.is_deleted(container) { + return Err(LoroError::ContainerDeleted { + container: Box::new(state.arena.idx_to_id(container).unwrap()), + }); + } + if !self.is_peer_first_appearance && !oplog.dag.latest_vv_contains_peer(self.peer) { + self.is_peer_first_appearance = true; + } + + let start_id = ID::new(self.peer, self.next_counter); + let start_lamport = self.next_lamport; + let deps = if self.local_ops.is_empty() { + self.frontiers.clone() + } else { + Frontiers::from_id(ID::new(self.peer, self.next_counter - 1)) + }; + let mut applied_len: usize = 0; + let mut result = Ok(()); + for (content, event) in ops { + let len = content.content_len(); + assert!(len > 0); + let raw_op = RawOp { + id: ID { + peer: self.peer, + counter: self.next_counter, + }, + lamport: self.next_lamport, + container, + content, + }; + let op = self.arena.convert_raw_op(&raw_op); + if let Err(error) = state.apply_local_op(&raw_op, &op) { + result = Err(error); + break; + } + self.next_counter += len as Counter; + self.next_lamport += len as Lamport; + applied_len += len; + debug_assert_eq!(event.rle_len(), op.atom_len()); + let container_hints = self.event_hints.entry(container).or_default(); + match container_hints.last_mut() { + Some(last) if last.can_merge(&event) => { + last.merge_right(&event); + } + _ => { + container_hints.push(event); + } + } + self.local_ops.push(op); + } + // ONE version update for the whole applied prefix (all of it on + // success; the consistent prefix on a mid-batch error). + if applied_len > 0 { + oplog + .dag + .update_version_on_new_local_op(&deps, start_id, start_lamport, applied_len); + oplog.refresh_visible_op_count(); + state.frontiers = Frontiers::from_id(start_id.inc(applied_len as Counter - 1)); + } + drop(state); + drop(oplog); + result + } + + /// §perf (flowstate vendor patch, act three D.1): batched local TEXT + /// INSERTS for a multi-span delta. Entity/style resolution per span stays + /// live-interleaved (each insert shifts later positions), only the + /// transaction ceremony is hoisted — op emission is byte-identical to the + /// per-span `insert_with_txn_and_attr` path. Returns the override-style + /// marks (sequence, start, end, key, value) the caller must apply AFTER + /// the inserts, exactly as the per-span contract does. Event positions + /// are the delta-walk indices (post-prior-inserts), non-wasm builds only. + #[allow(clippy::type_complexity)] + pub(super) fn apply_local_text_insert_batch( + &mut self, + container: ContainerIdx, + inserts: Vec<(usize, usize, String, Option>)>, + doc: &LoroDoc, + ) -> LoroResult> { + use crate::container::list::list_op::ListOp; + use crate::op::ListSlice; + use std::ops::Deref as _; + if inserts.is_empty() { + return Ok(Vec::new()); + } + let this_doc = self.doc.upgrade().unwrap(); + if Arc::as_ptr(&this_doc.state) != Arc::as_ptr(&doc.state) { + return Err(LoroError::UnmatchedContext { + expected: this_doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + found: doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + }); + } + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + if state.is_deleted(container) { + return Err(LoroError::ContainerDeleted { + container: Box::new(state.arena.idx_to_id(container).unwrap()), + }); + } + if !self.is_peer_first_appearance && !oplog.dag.latest_vv_contains_peer(self.peer) { + self.is_peer_first_appearance = true; + } + + let start_id = ID::new(self.peer, self.next_counter); + let start_lamport = self.next_lamport; + let deps = if self.local_ops.is_empty() { + self.frontiers.clone() + } else { + Frontiers::from_id(ID::new(self.peer, self.next_counter - 1)) + }; + let mut applied_len: usize = 0; + let mut override_marks: Vec<(usize, usize, usize, InternalString, LoroValue)> = Vec::new(); + let mut result = Ok(()); + for (seq, event_pos, text, attr) in inserts { + if text.is_empty() { + continue; + } + let probe = state.with_state_mut(container, |container_state| { + let richtext = container_state.as_richtext_state_mut().unwrap(); + if event_pos > richtext.len(crate::cursor::PosType::Event) { + return Err(LoroError::OutOfBound { + pos: event_pos, + len: richtext.len(crate::cursor::PosType::Event), + info: format!("Position: {}:{}", file!(), line!()).into_boxed_str(), + }); + } + let (entity_index, _cursor) = richtext + .get_entity_index_for_text_insert(event_pos, crate::cursor::PosType::Event)?; + let styles = richtext.get_styles_at_entity_index(entity_index); + Ok((entity_index, styles)) + }); + let (entity_index, styles) = match probe { + Ok(pair) => pair, + Err(error) => { + result = Err(error); + break; + } + }; + + if let Some(attr) = attr.as_ref() { + let map: FxHashMap<_, _> = styles.iter().map(|x| (x.0.clone(), x.1.data)).collect(); + for (key, style) in map.iter() { + match attr.get(key.deref()) { + Some(v) if v == style => {} + new_style_value => { + let new_style_value = new_style_value.cloned().unwrap_or(LoroValue::Null); + override_marks.push(( + seq, + event_pos, + event_pos + text.chars().count(), + key.clone(), + new_style_value, + )); + } + } + } + for (key, style) in attr.iter() { + let key: InternalString = key.as_str().into(); + if !map.contains_key(&key) { + override_marks.push(( + seq, + event_pos, + event_pos + text.chars().count(), + key, + style.clone(), + )); + } + } + } + + let unicode_len = text.chars().count(); + let event_len = unicode_len; // non-wasm: event index == unicode index + let raw_op = RawOp { + id: ID { + peer: self.peer, + counter: self.next_counter, + }, + lamport: self.next_lamport, + container, + content: RawOpContent::List(ListOp::Insert { + slice: ListSlice::RawStr { + str: std::borrow::Cow::Borrowed(text.as_str()), + unicode_len, + }, + pos: entity_index, + }), + }; + let event = EventHint::InsertText { + pos: event_pos as u32, + styles, + unicode_len: unicode_len as u32, + event_len: event_len as u32, + }; + let len = raw_op.content.content_len(); + let op = self.arena.convert_raw_op(&raw_op); + if let Err(error) = state.apply_local_op(&raw_op, &op) { + result = Err(error); + break; + } + self.next_counter += len as Counter; + self.next_lamport += len as Lamport; + applied_len += len; + debug_assert_eq!(event.rle_len(), op.atom_len()); + let container_hints = self.event_hints.entry(container).or_default(); + match container_hints.last_mut() { + Some(last) if last.can_merge(&event) => { + last.merge_right(&event); + } + _ => { + container_hints.push(event); + } + } + self.local_ops.push(op); + } + if applied_len > 0 { + oplog + .dag + .update_version_on_new_local_op(&deps, start_id, start_lamport, applied_len); + oplog.refresh_visible_op_count(); + state.frontiers = Frontiers::from_id(start_id.inc(applied_len as Counter - 1)); + } + drop(state); + drop(oplog); + result.map(|()| override_marks) + } + + /// §perf (flowstate vendor patch, act three D.1): batched local TEXT MARK + /// application — the mark-heavy import path (one styled document build = + /// tens of thousands of marks) paid the full per-op transaction ceremony + /// TWICE per mark (StyleStart + StyleEnd). Entity ranges are computed from + /// the LIVE state inside the loop (each mark's anchors shift later entity + /// positions, so precomputation would be wrong); only the surrounding + /// ceremony is batched. `LoroValue::Null` (unmark) is NOT supported here. + #[allow(clippy::type_complexity)] + pub(super) fn apply_local_text_mark_batch( + &mut self, + container: ContainerIdx, + marks: Vec<( + usize, + usize, + InternalString, + LoroValue, + crate::container::richtext::TextStyleInfoFlag, + )>, + pos_type: crate::cursor::PosType, + doc: &LoroDoc, + ) -> LoroResult<()> { + use crate::container::list::list_op::ListOp; + if marks.is_empty() { + return Ok(()); + } + let this_doc = self.doc.upgrade().unwrap(); + if Arc::as_ptr(&this_doc.state) != Arc::as_ptr(&doc.state) { + return Err(LoroError::UnmatchedContext { + expected: this_doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + found: doc + .state + .lock() + .peer + .load(std::sync::atomic::Ordering::Relaxed), + }); + } + + let mut oplog = doc.oplog.lock(); + let mut state = doc.state.lock(); + if state.is_deleted(container) { + return Err(LoroError::ContainerDeleted { + container: Box::new(state.arena.idx_to_id(container).unwrap()), + }); + } + if !self.is_peer_first_appearance && !oplog.dag.latest_vv_contains_peer(self.peer) { + self.is_peer_first_appearance = true; + } + + let start_id = ID::new(self.peer, self.next_counter); + let start_lamport = self.next_lamport; + let deps = if self.local_ops.is_empty() { + self.frontiers.clone() + } else { + Frontiers::from_id(ID::new(self.peer, self.next_counter - 1)) + }; + let mut applied_len: usize = 0; + let mut result = Ok(()); + 'marks: for (start, end, key, value, flag) in marks { + let is_delete = matches!(value, LoroValue::Null); + let (event_start, event_end, entity_range, skip, missing_style_key) = + state.with_state_mut(container, |container_state| { + let richtext = container_state.as_richtext_state_mut().unwrap(); + let event_start = richtext.index_to_event_index(start, pos_type); + let event_end = richtext.index_to_event_index(end, pos_type); + let (entity_range, styles) = + richtext.get_entity_range_and_styles_at_range(start..end, pos_type); + let skip = styles + .map(|styles| styles.has_key_value(&key, &value)) + .unwrap_or(false); + // §perf-heaven T8.3: `has_target_style` is ONLY consumed by the + // `is_delete` branch below, but the range scan + key clone ran + // for every mark — including all the non-delete import marks. + // The `&&` short-circuits, so the O(range) scan and the clone + // now happen only when this is actually a delete. + let missing_style_key = is_delete + && !richtext.has_style_key_in_entity_range( + entity_range.clone(), + &crate::container::richtext::StyleKey::Key(key.clone()), + ); + (event_start, event_end, entity_range, skip, missing_style_key) + }); + if skip || missing_style_key { + continue; + } + let pair: [(RawOpContent, crate::txn::EventHint); 2] = [ + ( + RawOpContent::List(ListOp::StyleStart { + start: entity_range.start as u32, + end: entity_range.end as u32, + key: key.clone(), + value: value.clone(), + info: flag, + }), + EventHint::Mark { + start: event_start as u32, + end: event_end as u32, + style: Style { key, data: value }, + }, + ), + (RawOpContent::List(ListOp::StyleEnd), EventHint::MarkEnd), + ]; + for (content, event) in pair { + let len = content.content_len(); + assert!(len > 0); + let raw_op = RawOp { + id: ID { + peer: self.peer, + counter: self.next_counter, + }, + lamport: self.next_lamport, + container, + content, + }; + let op = self.arena.convert_raw_op(&raw_op); + if let Err(error) = state.apply_local_op(&raw_op, &op) { + result = Err(error); + break 'marks; + } + self.next_counter += len as Counter; + self.next_lamport += len as Lamport; + applied_len += len; + debug_assert_eq!(event.rle_len(), op.atom_len()); + let container_hints = self.event_hints.entry(container).or_default(); + match container_hints.last_mut() { + Some(last) if last.can_merge(&event) => { + last.merge_right(&event); + } + _ => { + container_hints.push(event); + } + } + self.local_ops.push(op); + } + } + if applied_len > 0 { + oplog + .dag + .update_version_on_new_local_op(&deps, start_id, start_lamport, applied_len); + oplog.refresh_visible_op_count(); + state.frontiers = Frontiers::from_id(start_id.inc(applied_len as Counter - 1)); + } + drop(state); + drop(oplog); + result + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_text(&self, id: I) -> TextHandler { + let id = id.into_container_id(&self.arena, ContainerType::Text); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_text() + .unwrap() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_list(&self, id: I) -> ListHandler { + let id = id.into_container_id(&self.arena, ContainerType::List); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_list() + .unwrap() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_map(&self, id: I) -> MapHandler { + let id = id.into_container_id(&self.arena, ContainerType::Map); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_map() + .unwrap() + } + + /// id can be a str, ContainerID, or ContainerIdRaw. + /// if it's str it will use Root container, which will not be None + pub fn get_tree(&self, id: I) -> TreeHandler { + let id = id.into_container_id(&self.arena, ContainerType::Tree); + Handler::new_attached(id, LoroDoc::from_inner(self.doc.upgrade().unwrap())) + .into_tree() + .unwrap() + } + pub fn next_id(&self) -> ID { + ID { + peer: self.peer, + counter: self.next_counter, + } + } + + #[inline] + pub fn id_span(&self) -> IdSpan { + IdSpan::new(self.peer, self.start_counter, self.next_counter) + } + + pub fn next_idlp(&self) -> IdLp { + IdLp { + peer: self.peer, + lamport: self.next_lamport, + } + } + + pub fn is_empty(&self) -> bool { + self.local_ops.is_empty() + } + + pub(crate) fn len(&self) -> usize { + (self.next_counter - self.start_counter) as usize + } + + pub(crate) fn set_options(&mut self, options: CommitOptions) { + self.origin = options.origin.unwrap_or_default(); + self.msg = options.commit_msg; + self.timestamp = options.timestamp; + } + + pub(crate) fn set_default_options(&mut self, default_options: crate::loro::CommitOptions) { + if self.origin.is_empty() { + self.origin = default_options.origin.unwrap_or_default(); + } + if self.msg.is_none() { + self.msg = default_options.commit_msg; + } + if self.timestamp.is_none() { + self.timestamp = default_options.timestamp; + } + } +} + +impl Drop for Transaction { + #[tracing::instrument(level = "debug", skip(self))] + fn drop(&mut self) { + if !self.finished { + // TODO: should we abort here or commit here? + // what if commit fails? + self._commit().unwrap(); + } + } +} + +#[derive(Debug, Clone)] +pub(crate) struct TxnContainerDiff { + pub(crate) idx: ContainerIdx, + pub(crate) diff: Diff, +} + +// PERF: could be compacter +fn change_to_diff( + change: &Change, + doc: Arc, + event_hints: FxHashMap>, +) -> Vec { + let mut ans: Vec = Vec::with_capacity(change.ops.len()); + let peer = change.id.peer; + let mut lamport = change.lamport; + + // Group ops by container first to match our new structure + let mut ops_by_container: FxHashMap> = FxHashMap::default(); + for op in change.ops.iter() { + ops_by_container + .entry(op.container) + .or_default() + .push(op.clone()); + } + + // Process each container's hints and ops together + for (container_idx, hints) in event_hints { + let Some(container_ops) = ops_by_container.get_mut(&container_idx) else { + continue; + }; + + let mut op_index = 0; + let mut hint_iter = hints.into_iter(); + let mut current_hint = hint_iter.next(); + + while op_index < container_ops.len() { + let Some(hint) = current_hint.take() else { + unreachable!("Missing hint for op"); + }; + + // Collect ops that belong to this hint + let mut ops_for_hint: SmallVec<[Op; 1]> = smallvec![container_ops[op_index].clone()]; + let mut total_len = container_ops[op_index].atom_len(); + + // If hint spans multiple ops, collect them + while total_len < hint.rle_len() { + op_index += 1; + let next_op_len = container_ops[op_index].atom_len(); + let op = if next_op_len + total_len > hint.rle_len() { + let new_len = hint.rle_len() - total_len; + let left = container_ops[op_index].slice(0, new_len); + let right = container_ops[op_index].slice(new_len, next_op_len); + container_ops[op_index] = right; + op_index -= 1; + left + } else { + container_ops[op_index].clone() + }; + + total_len += op.atom_len(); + ops_for_hint.push(op); + } + + op_index += 1; + assert_eq!(total_len, hint.rle_len(), "Op/hint length mismatch"); + + // Move to next hint + current_hint = hint_iter.next(); + + // Generate diff based on hint type + match hint { + EventHint::Mark { start, end, style } => { + let mut meta = StyleMeta::default(); + meta.insert( + style.key.clone(), + StyleMetaItem { + lamport, + peer: change.id.peer, + value: style.data, + }, + ); + let diff = DeltaRopeBuilder::new() + .retain(start as usize, Default::default()) + .retain( + (end - start) as usize, + meta.to_option_map().unwrap_or_default().into(), + ) + .build(); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Text(diff), + }); + } + EventHint::InsertText { styles, pos, .. } => { + let mut delta: TextDiff = DeltaRopeBuilder::new() + .retain(pos as usize, Default::default()) + .build(); + for op in ops_for_hint.iter() { + let InnerListOp::InsertText { slice, .. } = op.content.as_list().unwrap() + else { + unreachable!() + }; + + delta.push_insert( + slice.clone().into(), + styles.to_option_map().unwrap_or_default().into(), + ); + } + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Text(delta), + }) + } + EventHint::DeleteText { + span, + unicode_len: _, + // we don't need to iter over ops here, because we already + // know what the events should be + } => ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Text( + DeltaRopeBuilder::new() + .retain(span.start() as usize, Default::default()) + .delete(span.len()) + .build(), + ), + }), + EventHint::InsertList { pos, .. } => { + // We should use pos from event hint because index in op may + // be using op index for the MovableList + let mut index = pos; + for op in ops_for_hint.iter() { + let (range, _) = op.content.as_list().unwrap().as_insert().unwrap(); + let values = doc + .arena + .get_values(range.to_range()) + .into_iter() + .map(|v| ValueOrHandler::from_value(v, &doc)); + let len = values.len(); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List( + DeltaRopeBuilder::new() + .retain(index, Default::default()) + .insert_many(values, Default::default()) + .build(), + ), + }); + index += len; + } + } + EventHint::DeleteList(s) => { + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List( + DeltaRopeBuilder::new() + .retain(s.start() as usize, Default::default()) + .delete(s.len()) + .build(), + ), + }); + } + EventHint::Map { key, value } => { + // Mirror `MapState::apply_diff_and_convert`: a marker write surfaces to + // subscribers as the deterministic Container, not the raw binary value. + let parent_id = doc.arena.idx_to_id(container_idx); + let value = value.map(|v| match parent_id.as_ref() { + Some(pid) => { + loro_common::translate_mergeable_marker_value(pid, key.as_ref(), v) + } + None => v, + }); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Map(ResolvedMapDelta::new().with_entry( + key, + ResolvedMapValue { + value: value.map(|v| ValueOrHandler::from_value(v, &doc)), + idlp: IdLp::new(peer, lamport), + }, + )), + }); + } + EventHint::Tree(tree_diff) => { + let mut diff = TreeDiff::default(); + diff.diff.extend(tree_diff); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Tree(diff), + }); + } + EventHint::Move { from, to, value } => { + let mut a = DeltaRopeBuilder::new() + .retain(from as usize, Default::default()) + .delete(1) + .build(); + a.compose( + &DeltaRopeBuilder::new() + .retain(to as usize, Default::default()) + .insert( + ArrayVec::from([ValueOrHandler::from_value(value, &doc)]), + ListDeltaMeta { from_move: true }, + ) + .build(), + ); + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List(a), + }); + } + EventHint::SetList { index, value } => { + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::List( + DeltaRopeBuilder::new() + .retain(index, Default::default()) + .delete(1) + .insert( + ArrayVec::from([ValueOrHandler::from_value(value, &doc)]), + Default::default(), + ) + .build(), + ), + }); + } + EventHint::MarkEnd => { + // do nothing + } + #[cfg(feature = "counter")] + EventHint::Counter(diff) => { + ans.push(TxnContainerDiff { + idx: container_idx, + diff: Diff::Counter(diff), + }); + } + } + + // Update lamport for this hint's operations + lamport += ops_for_hint + .iter() + .map(|x| x.content_len() as Lamport) + .sum::(); + } + } + + ans +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{cursor::PosType, version::Frontiers}; + + #[test] + fn txn_creation_rolls_back_in_txn_after_peer_conflict() { + let doc = LoroDoc::new(); + doc.set_detached_editing(true); + doc.set_peer_id(7).unwrap(); + + let text = doc.get_text("text"); + let mut txn = doc.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "a", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + + doc.checkout(&Frontiers::default()).unwrap(); + doc.set_peer_id(7).unwrap(); + + let err = doc + .txn() + .expect_err("stale detached frontiers should reject reusing the same peer"); + assert!(matches!( + err, + LoroError::ConcurrentOpsWithSamePeerID { peer: 7, .. } + )); + assert!(!doc.app_state().lock().is_in_txn()); + } +} diff --git a/vendor/loro-internal/src/undo.rs b/vendor/loro-internal/src/undo.rs new file mode 100644 index 00000000..2654253b --- /dev/null +++ b/vendor/loro-internal/src/undo.rs @@ -0,0 +1,1337 @@ +use std::{cell::RefCell, collections::VecDeque, sync::Arc}; + +use crate::sync::{AtomicU64, Mutex}; +use either::Either; +use loro_common::{ + ContainerID, Counter, CounterSpan, HasIdSpan, IdSpan, LoroError, LoroResult, LoroValue, PeerID, +}; +use parking_lot::lock_api::ReentrantMutex; +use rustc_hash::{FxHashMap, FxHashSet}; +use tracing::{debug_span, info_span, instrument}; + +use crate::{ + change::{get_sys_timestamp, Timestamp}, + cursor::{AbsolutePosition, Cursor}, + delta::TreeExternalDiff, + event::{Diff, EventTriggerKind}, + version::Frontiers, + ContainerDiff, DiffEvent, DocDiff, LoroDoc, Subscription, +}; + +/// A batch of diffs. +/// +/// You can use `loroDoc.apply_diff(diff)` to apply the diff to the document. +#[derive(Debug, Clone, Default)] +pub struct DiffBatch { + pub cid_to_events: FxHashMap, + pub order: Vec, +} + +impl DiffBatch { + pub fn new(diff: Vec) -> Self { + let mut map: FxHashMap = Default::default(); + let mut order: Vec = Vec::with_capacity(diff.len()); + for d in diff.into_iter() { + for item in d.diff.into_iter() { + let old = map.insert(item.id.clone(), item.diff); + assert!(old.is_none(), "Duplicate container ID in diff events"); + order.push(item.id.clone()); + } + } + + Self { + cid_to_events: map, + order, + } + } + + pub fn compose(&mut self, other: &Self) { + if other.cid_to_events.is_empty() { + return; + } + + for (id, diff) in other.iter() { + if let Some(this_diff) = self.cid_to_events.get_mut(id) { + this_diff.compose_ref(diff); + } else { + self.cid_to_events.insert(id.clone(), diff.clone()); + self.order.push(id.clone()); + } + } + } + + pub fn transform(&mut self, other: &Self, left_priority: bool) { + if other.cid_to_events.is_empty() || self.cid_to_events.is_empty() { + return; + } + + for (idx, diff) in self.cid_to_events.iter_mut() { + if let Some(b_diff) = other.cid_to_events.get(idx) { + diff.transform(b_diff, left_priority); + } + } + } + + pub fn clear(&mut self) { + self.cid_to_events.clear(); + self.order.clear(); + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.order + .iter() + .map(|cid| (cid, self.cid_to_events.get(cid).unwrap())) + } + + #[allow(clippy::should_implement_trait)] + pub fn into_iter(self) -> impl Iterator { + let mut cid_to_events = self.cid_to_events; + self.order.into_iter().map(move |cid| { + let d = cid_to_events.remove(&cid).unwrap(); + (cid, d) + }) + } +} + +fn transform_cursor( + cursor_with_pos: &mut CursorWithPos, + remote_diff: &DiffBatch, + doc: &LoroDoc, + container_remap: &FxHashMap, +) { + let mut container_changed = false; + let mut cid = &cursor_with_pos.cursor.container; + while let Some(new_cid) = container_remap.get(cid) { + cid = new_cid; + container_changed = true; + } + + if cursor_with_pos.cursor.id.is_none() { + // We don't need to transform a cursor that always points to the leftmost or rightmost position + if container_changed { + cursor_with_pos.cursor.container = cid.clone(); + } + return; + } + + if let Some(diff) = remote_diff.cid_to_events.get(cid) { + let new_pos = diff.transform_cursor(cursor_with_pos.pos.pos, false); + cursor_with_pos.pos.pos = new_pos; + }; + + let new_pos = cursor_with_pos.pos.pos; + match doc.get_handler(cid.clone()).unwrap() { + crate::handler::Handler::Text(h) => { + let Some(new_cursor) = h.get_cursor_internal(new_pos, cursor_with_pos.pos.side, false) + else { + return; + }; + + cursor_with_pos.cursor = new_cursor; + } + crate::handler::Handler::List(h) => { + let Some(new_cursor) = h.get_cursor(new_pos, cursor_with_pos.pos.side) else { + return; + }; + + cursor_with_pos.cursor = new_cursor; + } + crate::handler::Handler::MovableList(h) => { + let Some(new_cursor) = h.get_cursor(new_pos, cursor_with_pos.pos.side) else { + return; + }; + + cursor_with_pos.cursor = new_cursor; + } + crate::handler::Handler::Map(_) => {} + crate::handler::Handler::Tree(_) => {} + crate::handler::Handler::Unknown(_) => {} + #[cfg(feature = "counter")] + crate::handler::Handler::Counter(_) => {} + } +} + +/// UndoManager is responsible for managing undo/redo from the current peer's perspective. +/// +/// Undo/local is local: it cannot be used to undone the changes made by other peers. +/// If you want to undo changes made by other peers, you may need to use the time travel feature. +/// +/// PeerID cannot be changed during the lifetime of the UndoManager +pub struct UndoManager { + peer: Arc, + container_remap: Arc>>, + inner: Arc>>, + _peer_id_change_sub: Subscription, + _undo_sub: Subscription, + doc: LoroDoc, +} + +impl std::fmt::Debug for UndoManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UndoManager") + .field("peer", &self.peer) + .field("container_remap", &self.container_remap) + .field("inner", &self.inner) + .finish() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UndoOrRedo { + Undo, + Redo, +} + +impl UndoOrRedo { + fn opposite(&self) -> UndoOrRedo { + match self { + Self::Undo => Self::Redo, + Self::Redo => Self::Undo, + } + } +} + +/// When a undo/redo item is pushed, the undo manager will call the on_push callback to get the meta data of the undo item. +/// The returned cursors will be recorded for a new pushed undo item. +pub type OnPush = Box< + dyn for<'a> Fn(UndoOrRedo, CounterSpan, Option>) -> UndoItemMeta + Send + Sync, +>; +pub type OnPop = Box; + +struct UndoManagerInner { + next_counter: Option, + undo_stack: Stack, + redo_stack: Stack, + processing_undo: bool, + last_undo_time: i64, + merge_interval_in_ms: i64, + max_stack_size: usize, + exclude_origin_prefixes: Vec>, + // §flowstate vendor patch #14 (act-ten A10.2): origins whose local commits + // are IGNORED by the undo stacks entirely — not recorded AND not composed + // as remote-like pending diffs. Sound only for commits that can never touch + // content an undo item references (flowstate's "meta" metadata/revision + // commits, which write disjoint meta containers). The plain excluded-origin + // path marks the stacks dirty (`compose_remote_event`), which flips + // `peek_top_span`'s `clean` flag and forces the external fast path to bail + // after every autosave. + inert_origin_prefixes: Vec>, + last_popped_selection: Option>, + on_push: Option, + on_pop: Option, + group: Option, + // §perf (flowstate vendor patch, act-three B.1): in-flight external + // fast-path step state — the popped item (for abort restore) and the + // saved last-popped-selection relay value (for the finish push). + external_popped: Option<(UndoOrRedo, StackItem)>, + external_next_push_selection: Option>, +} + +impl std::fmt::Debug for UndoManagerInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UndoManagerInner") + .field("latest_counter", &self.next_counter) + .field("undo_stack", &self.undo_stack) + .field("redo_stack", &self.redo_stack) + .field("processing_undo", &self.processing_undo) + .field("last_undo_time", &self.last_undo_time) + .field("merge_interval", &self.merge_interval_in_ms) + .field("max_stack_size", &self.max_stack_size) + .field("exclude_origin_prefixes", &self.exclude_origin_prefixes) + .field("group", &self.group) + .finish() + } +} + +#[derive(Debug, Clone, Default)] +struct UndoGroup { + start_counter: Counter, + affected_cids: FxHashSet, +} + +impl UndoGroup { + pub fn new(start_counter: Counter) -> Self { + Self { + start_counter, + affected_cids: Default::default(), + } + } +} + +#[derive(Debug)] +struct Stack { + stack: VecDeque<(VecDeque, Arc>)>, + size: usize, +} + +#[derive(Debug, Clone)] +struct StackItem { + span: CounterSpan, + meta: UndoItemMeta, +} + +/// The metadata of an undo item. +/// +/// The cursors inside the metadata will be transformed by remote operations as well. +/// So that when the item is popped, users can restore the cursors position correctly. +#[derive(Debug, Default, Clone)] +pub struct UndoItemMeta { + pub value: LoroValue, + pub cursors: Vec, +} + +#[derive(Debug, Clone)] +pub struct CursorWithPos { + pub cursor: Cursor, + pub pos: AbsolutePosition, +} + +impl UndoItemMeta { + pub fn new() -> Self { + Self { + value: LoroValue::Null, + cursors: Default::default(), + } + } + + /// It's assumed that the cursor is just acquired before the ops that + /// need to be undo/redo. + /// + /// We need to rely on the validity of the original pos value + pub fn add_cursor(&mut self, cursor: &Cursor) { + self.cursors.push(CursorWithPos { + cursor: cursor.clone(), + pos: AbsolutePosition { + pos: cursor.origin_pos, + side: cursor.side, + }, + }); + } + + pub fn set_value(&mut self, value: LoroValue) { + self.value = value; + } +} + +impl Stack { + pub fn new() -> Self { + let mut stack = VecDeque::new(); + stack.push_back((VecDeque::new(), Arc::new(Mutex::new(Default::default())))); + Stack { stack, size: 0 } + } + + /// Peek the top-most StackItem's metadata without modifying the stack. + /// + /// Returns None if the stack is empty. + fn peek_top_meta(&self) -> Option { + if self.is_empty() { + return None; + } + + for (items, _) in self.stack.iter().rev() { + if let Some(item) = items.back() { + return Some(item.meta.clone()); + } + } + + None + } + + pub fn pop(&mut self) -> Option<(StackItem, Arc>)> { + while self.stack.back().unwrap().0.is_empty() && self.stack.len() > 1 { + let (_, diff) = self.stack.pop_back().unwrap(); + let diff = diff.lock(); + if !diff.cid_to_events.is_empty() { + self.stack.back_mut().unwrap().1.lock().compose(&diff); + } + } + + if self.stack.len() == 1 && self.stack.back().unwrap().0.is_empty() { + // If the stack is empty, we need to clear the remote diff + self.stack.back_mut().unwrap().1.lock().clear(); + return None; + } + + self.size -= 1; + let last = self.stack.back_mut().unwrap(); + last.0.pop_back().map(|x| (x, last.1.clone())) + // If this row in stack is empty, we don't pop it right away + // Because we still need the remote diff to be available. + // Cursor position transformation relies on the remote diff in the same row. + } + + pub fn push(&mut self, span: CounterSpan, meta: UndoItemMeta) { + self.push_with_merge(span, meta, false, None) + } + + pub fn push_with_merge( + &mut self, + span: CounterSpan, + meta: UndoItemMeta, + can_merge: bool, + group: Option<&UndoGroup>, + ) { + let last = self.stack.back_mut().unwrap(); + let last_remote_diff = last.1.lock(); + + // Check if the remote diff is disjoint with the current undo group + let is_disjoint_group = group.is_some_and(|g| { + g.affected_cids.iter().all(|cid| { + last_remote_diff + .cid_to_events + .get(cid) + .is_none_or(|diff| diff.is_empty()) + }) + }); + + // Can't merge if remote diffs exist and it's not disjoint with the current undo group + let should_create_new_entry = + !last_remote_diff.cid_to_events.is_empty() && !is_disjoint_group; + + if should_create_new_entry { + // Create a new entry in the stack + drop(last_remote_diff); + let mut v = VecDeque::new(); + v.push_back(StackItem { span, meta }); + self.stack + .push_back((v, Arc::new(Mutex::new(DiffBatch::default())))); + self.size += 1; + return; + } + + // Try to merge with the previous entry if allowed + if can_merge { + if let Some(last_span) = last.0.back_mut() { + if last_span.span.end == span.start { + // Merge spans by extending the end of the last span + last_span.span.end = span.end; + return; + } + } + } + + // Add as a new item to the existing entry + self.size += 1; + last.0.push_back(StackItem { span, meta }); + } + + pub fn compose_remote_event(&mut self, diff: &[&ContainerDiff]) { + if self.is_empty() { + return; + } + + let remote_diff = &mut self.stack.back_mut().unwrap().1; + let mut remote_diff = remote_diff.lock(); + for e in diff { + if let Some(d) = remote_diff.cid_to_events.get_mut(&e.id) { + d.compose_ref(&e.diff); + } else { + remote_diff + .cid_to_events + .insert(e.id.clone(), e.diff.clone()); + remote_diff.order.push(e.id.clone()); + } + } + } + + pub fn transform_based_on_this_delta(&mut self, diff: &DiffBatch) { + if self.is_empty() { + return; + } + let remote_diff = &mut self.stack.back_mut().unwrap().1; + remote_diff.lock().transform(diff, false); + } + + pub fn clear(&mut self) { + self.stack = VecDeque::new(); + self.stack.push_back((VecDeque::new(), Default::default())); + self.size = 0; + } + + pub fn is_empty(&self) -> bool { + self.size == 0 + } + + pub fn len(&self) -> usize { + self.size + } + + fn discard_empty_front_rows(&mut self) { + // Undo pop can leave an empty front row that only carries remote diffs. + // There is no older stack item for that diff to transform during trimming. + while self + .stack + .front() + .is_some_and(|(items, _)| items.is_empty()) + { + self.stack.pop_front(); + } + } + + fn ensure_trailing_empty_row(&mut self) { + if self.stack.is_empty() { + self.stack + .push_back((VecDeque::new(), Arc::new(Mutex::new(Default::default())))); + } + } + + fn pop_front(&mut self) { + if self.is_empty() { + return; + } + + self.discard_empty_front_rows(); + self.size -= 1; + let first = self.stack.front_mut().unwrap(); + let f = first.0.pop_front(); + assert!(f.is_some()); + if first.0.is_empty() { + self.stack.pop_front(); + } + + self.ensure_trailing_empty_row(); + } + + fn set_top_meta(&mut self, meta: UndoItemMeta) { + let Some(top) = self.stack.back_mut() else { + return; + }; + let Some(last) = top.0.back_mut() else { + return; + }; + last.meta = meta; + } + + /// §perf (flowstate vendor patch, act-three B.1): non-destructive top-item + /// probe for the external fast-path undo. Returns the top item's span plus + /// whether the stack tracks NO pending remote/excluded-origin diff anywhere + /// — the conservative "nothing interleaved since this item was pushed" + /// signal the external fast path requires before bypassing `perform`. + fn peek_top_span_and_clean(&self) -> Option<(CounterSpan, bool)> { + for (items, _) in self.stack.iter().rev() { + if let Some(item) = items.back() { + let clean = self + .stack + .iter() + .all(|(_, diff)| diff.lock().cid_to_events.is_empty()); + return Some((item.span, clean)); + } + } + None + } +} + +impl Default for Stack { + fn default() -> Self { + Stack::new() + } +} + +impl UndoManagerInner { + fn new(last_counter: Counter) -> Self { + Self { + next_counter: Some(last_counter), + undo_stack: Default::default(), + redo_stack: Default::default(), + processing_undo: false, + merge_interval_in_ms: 0, + last_undo_time: 0, + max_stack_size: usize::MAX, + exclude_origin_prefixes: vec![], + inert_origin_prefixes: vec![], + last_popped_selection: None, + on_pop: None, + on_push: None, + group: None, + external_popped: None, + external_next_push_selection: None, + } + } + + /// Returns true if a given container diff is disjoint with the current group. + /// They are disjoint if they have no overlap in changed container ids. + fn is_disjoint_with_group(&self, diff: &[&ContainerDiff]) -> bool { + let Some(group) = &self.group else { + return false; + }; + + diff.iter().all(|d| !group.affected_cids.contains(&d.id)) + } + + fn record_checkpoint(this: &RefCell, latest_counter: Counter, event: Option) { + let previous_counter = this.borrow().next_counter; + + if Some(latest_counter) == this.borrow().next_counter { + return; + } + + if this.borrow().next_counter.is_none() { + this.borrow_mut().next_counter = Some(latest_counter); + return; + } + + if let Some(group) = &mut this.borrow_mut().group { + event.iter().for_each(|e| { + e.events.iter().for_each(|e| { + group.affected_cids.insert(e.id.clone()); + }) + }); + } + + let now = get_sys_timestamp() as Timestamp; + let span = CounterSpan::new(this.borrow().next_counter.unwrap(), latest_counter); + let meta = this + .borrow() + .on_push + .as_ref() + .map(|x| x(UndoOrRedo::Undo, span, event)) + .unwrap_or_default(); + + let mut this = this.borrow_mut(); + let this: &mut Self = &mut this; + // Wether the change is within the accepted merge interval + let in_merge_interval = now - this.last_undo_time < this.merge_interval_in_ms; + + // If group is active, but there is nothing in the group, don't merge + // If the group is active and it's not the first push in the group, merge + let group_should_merge = this.group.is_some() + && match ( + previous_counter, + this.group.as_ref().map(|g| g.start_counter), + ) { + (Some(previous), Some(active)) => previous != active, + _ => true, + }; + + let should_merge = !this.undo_stack.is_empty() && (in_merge_interval || group_should_merge); + + if should_merge { + this.undo_stack + .push_with_merge(span, meta, true, this.group.as_ref()); + } else { + this.last_undo_time = now; + this.undo_stack.push(span, meta); + } + + this.next_counter = Some(latest_counter); + this.redo_stack.clear(); + while this.undo_stack.len() > this.max_stack_size { + this.undo_stack.pop_front(); + } + } +} + +fn get_counter_end(doc: &LoroDoc, peer: PeerID) -> Counter { + doc.oplog().lock().vv().get(&peer).cloned().unwrap_or(0) +} + +impl UndoManager { + pub fn new(doc: &LoroDoc) -> Self { + let peer = Arc::new(AtomicU64::new(doc.peer_id())); + let peer_clone = peer.clone(); + let peer_clone2 = peer.clone(); + let inner = Arc::new(ReentrantMutex::new(RefCell::new(UndoManagerInner::new( + get_counter_end(doc, doc.peer_id()), + )))); + let inner_clone = inner.clone(); + let inner_clone2 = inner.clone(); + let remap_containers = Arc::new(Mutex::new(FxHashMap::default())); + let remap_containers_clone = remap_containers.clone(); + let undo_sub = doc.subscribe_root(Arc::new(move |event| match event.event_meta.by { + EventTriggerKind::Local => { + // TODO: PERF undo can be significantly faster if we can get + // the DiffBatch for undo here + let lock = inner_clone.lock(); + if lock.borrow().processing_undo { + // §flowstate vendor patch #22: during an EXTERNAL fast step + // (external_step_begin..finish), the application's inverse + // commit lands here muted — but the native path transforms + // the popped-from stack's buffered remote diffs through the + // undo commit's diff (undo_internal's + // callback). Mirror that exactly, else a later slow-path pop + // of a deeper item transforms through stale coordinates. + let external_kind = lock + .borrow() + .external_popped + .as_ref() + .map(|(kind, _)| *kind); + if let Some(kind) = external_kind { + let mut batch = DiffBatch::default(); + for e in event.events { + batch.cid_to_events.insert(e.id.clone(), e.diff.clone()); + batch.order.push(e.id.clone()); + } + let mut inner = lock.borrow_mut(); + let stack = match kind { + UndoOrRedo::Undo => &mut inner.undo_stack, + UndoOrRedo::Redo => &mut inner.redo_stack, + }; + stack.transform_based_on_this_delta(&batch); + } + return; + } + if let Some(id) = event + .event_meta + .to + .iter() + .find(|x| x.peer == peer_clone.load(std::sync::atomic::Ordering::Relaxed)) + { + let should_ignore = lock + .borrow() + .inert_origin_prefixes + .iter() + .any(|x| event.event_meta.origin.starts_with(&**x)); + if should_ignore { + // §flowstate vendor patch #14: inert origin — advance the + // counter bookkeeping so the next real op's span starts + // after these ops, but leave both stacks untouched (no + // record, no remote-compose ⇒ `clean` stays true). + let mut inner = lock.borrow_mut(); + inner.next_counter = Some(id.counter + 1); + return; + } + let should_exclude = lock + .borrow() + .exclude_origin_prefixes + .iter() + .any(|x| event.event_meta.origin.starts_with(&**x)); + if should_exclude { + // If the event is from the excluded origin, we don't record it + // in the undo stack. But we need to record its effect like it's + // a remote event. + let mut inner = lock.borrow_mut(); + inner.undo_stack.compose_remote_event(event.events); + inner.redo_stack.compose_remote_event(event.events); + inner.next_counter = Some(id.counter + 1); + } else { + UndoManagerInner::record_checkpoint(&lock, id.counter + 1, Some(event)); + } + } + } + EventTriggerKind::Import => { + let lock = inner_clone.lock(); + let mut inner = lock.borrow_mut(); + + for e in event.events { + if let Diff::Tree(tree) = &e.diff { + for item in &tree.diff { + let target = item.target; + if let TreeExternalDiff::Create { .. } = &item.action { + // If the concurrent event is a create event, it may bring the deleted tree node back, + // so we need to remove it from the remap of the container. + remap_containers_clone + .lock() + .remove(&target.associated_meta_container()); + } + } + } + } + + let is_import_disjoint = inner.is_disjoint_with_group(event.events); + + inner.undo_stack.compose_remote_event(event.events); + inner.redo_stack.compose_remote_event(event.events); + + // If the import is not disjoint, we end the active group + // all subsequent changes will be new undo items + if !is_import_disjoint { + inner.group = None; + } + } + EventTriggerKind::Checkout => { + let lock = inner_clone.lock(); + let mut inner = lock.borrow_mut(); + inner.undo_stack.clear(); + inner.redo_stack.clear(); + inner.next_counter = None; + } + })); + + let sub = doc.subscribe_peer_id_change(Box::new(move |id| { + let lock = inner_clone2.lock(); + let mut inner = lock.borrow_mut(); + inner.undo_stack.clear(); + inner.redo_stack.clear(); + inner.next_counter = Some(id.counter); + peer_clone2.store(id.peer, std::sync::atomic::Ordering::Relaxed); + true + })); + + UndoManager { + peer, + container_remap: remap_containers, + inner, + _peer_id_change_sub: sub, + _undo_sub: undo_sub, + doc: doc.clone(), + } + } + + pub fn group_start(&self) -> LoroResult<()> { + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + + if inner.group.is_some() { + return Err(LoroError::UndoGroupAlreadyStarted); + } + + inner.group = + Some(UndoGroup::new(inner.next_counter.ok_or_else(|| { + LoroError::Unknown("UndoManager is not ready".into()) + })?)); + + Ok(()) + } + + pub fn group_end(&self) { + self.inner.lock().borrow_mut().group = None; + } + + pub fn peer(&self) -> PeerID { + self.peer.load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn set_merge_interval(&self, interval: i64) { + self.inner.lock().borrow_mut().merge_interval_in_ms = interval; + } + + pub fn set_max_undo_steps(&self, size: usize) { + self.inner.lock().borrow_mut().max_stack_size = size; + } + + /// §flowstate vendor patch #14: register an origin prefix whose local + /// commits the undo stacks IGNORE entirely (no record, no remote-compose). + /// Use only for commits that provably never touch content any undo item + /// references — otherwise stored cursors would go stale un-rebased. + pub fn add_inert_origin_prefix(&self, prefix: &str) { + self.inner + .lock() + .borrow_mut() + .inert_origin_prefixes + .push(prefix.into()); + } + + pub fn add_exclude_origin_prefix(&self, prefix: &str) { + self.inner + .lock() + .borrow_mut() + .exclude_origin_prefixes + .push(prefix.into()); + } + + pub fn record_new_checkpoint(&self) -> LoroResult<()> { + // Use implicit-style barrier to preserve next-commit options across + // an empty commit before undo/redo processing. + self.doc.with_barrier(|| {}); + let counter = get_counter_end(&self.doc, self.peer()); + UndoManagerInner::record_checkpoint(&self.inner.lock(), counter, None); + Ok(()) + } + + #[instrument(skip_all)] + pub fn undo(&self) -> LoroResult { + self.perform( + |x| &mut x.undo_stack, + |x| &mut x.redo_stack, + UndoOrRedo::Undo, + ) + } + + #[instrument(skip_all)] + pub fn redo(&self) -> LoroResult { + self.perform( + |x| &mut x.redo_stack, + |x| &mut x.undo_stack, + UndoOrRedo::Redo, + ) + } + + fn perform( + &self, + get_stack: impl Fn(&mut UndoManagerInner) -> &mut Stack, + get_opposite: impl Fn(&mut UndoManagerInner) -> &mut Stack, + kind: UndoOrRedo, + ) -> LoroResult { + let doc = &self.doc.clone(); + // When in the undo/redo loop, the new undo/redo stack item should restore the selection + // to the state it was in before the item that was popped two steps ago from the stack. + // + // ┌────────────┐ + // │Selection 1 │ + // └─────┬──────┘ + // │ Some + // ▼ ops + // ┌────────────┐ + // │Selection 2 │ + // └─────┬──────┘ + // │ Some + // ▼ ops + // ┌────────────┐ + // │Selection 3 │◁ ─ ─ ─ ─ ─ ─ ─ Restore ─ ─ ─ + // └─────┬──────┘ │ + // │ + // │ │ + // │ ┌ ─ ─ ─ ─ ─ ─ ─ + // Enter the │ Undo ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─▶ Push Redo │ + // undo/redo ─ ─ ─ ▶ ▼ └ ─ ─ ─ ─ ─ ─ ─ + // loop ┌────────────┐ │ + // │Selection 2 │◁─ ─ ─ Restore ─ + // └─────┬──────┘ │ │ + // │ + // │ │ │ + // │ ┌ ─ ─ ─ ─ ─ ─ ─ + // │ Undo ─ ─ ─ ─ ▶ Push Redo │ │ + // ▼ └ ─ ─ ─ ─ ─ ─ ─ + // ┌────────────┐ │ │ + // │Selection 1 │ + // └─────┬──────┘ │ │ + // │ Redo ◀ ─ ─ ─ ─ ─ ─ ─ ─ + // ▼ │ + // ┌────────────┐ + // ┌ Restore ─ ▷│Selection 2 │ │ + // └─────┬──────┘ + // │ │ │ + // ┌ ─ ─ ─ ─ ─ ─ ─ │ + // Push Undo │◀─ ─ ─ ─ ─ ─ ─ │ Redo ◀ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + // └ ─ ─ ─ ─ ─ ─ ─ ▼ + // │ ┌────────────┐ + // │Selection 3 │ + // │ └─────┬──────┘ + // ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ▶ │ Undo + // ▼ + // ┌────────────┐ + // │Selection 2 │ + // └────────────┘ + // + // Because users may change the selections during the undo/redo loop, it's + // more stable to keep the selection stored in the last stack item + // rather than using the current selection directly. + self.record_new_checkpoint()?; + let end_counter = get_counter_end(doc, self.peer()); + let mut top = { + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + inner.processing_undo = true; + get_stack(&mut inner).pop() + }; + + let mut executed = false; + while let Some((mut span, remote_diff)) = top { + let mut next_push_selection = None; + { + let inner = self.inner.clone(); + // We need to clone this because otherwise will be applied to the same remote diff + let remote_change_clone = remote_diff.lock().clone(); + let commit = doc.undo_internal( + IdSpan { + peer: self.peer(), + counter: span.span, + }, + &mut self.container_remap.lock(), + Some(&remote_change_clone), + &mut |diff| { + info_span!("transform remote diff").in_scope(|| { + let inner = inner.lock(); + // + get_stack(&mut inner.borrow_mut()).transform_based_on_this_delta(diff); + }); + }, + )?; + drop(commit); + let inner = self.inner.lock(); + let mut is_some = false; + + if let Some(on_pop) = inner.borrow().on_pop.as_ref() { + is_some = true; + for cursor in span.meta.cursors.iter_mut() { + // We need to transform cursor here. + // Note that right now is already done, + // remote_diff is also transformed by it now (that's what we need). + transform_cursor( + cursor, + &remote_diff.lock(), + doc, + &self.container_remap.lock(), + ); + } + + on_pop(kind, span.span, span.meta.clone()); + } + if is_some { + let take = inner.borrow_mut().last_popped_selection.take(); + next_push_selection = take; + inner.borrow_mut().last_popped_selection = Some(span.meta.cursors); + } + } + let new_counter = get_counter_end(doc, self.peer()); + if end_counter != new_counter { + let inner = self.inner.lock(); + let mut meta = inner + .borrow() + .on_push + .as_ref() + .map(|x| { + x( + kind.opposite(), + CounterSpan::new(end_counter, new_counter), + None, + ) + }) + .unwrap_or_default(); + + if matches!(kind, UndoOrRedo::Undo) + && get_opposite(&mut inner.borrow_mut()).is_empty() + { + // If it's the first undo, we use the cursors from the users + } else if let Some(inner) = next_push_selection.take() { + // Otherwise, we use the cursors from the undo/redo loop + meta.cursors = inner; + } + + get_opposite(&mut inner.borrow_mut()) + .push(CounterSpan::new(end_counter, new_counter), meta); + inner.borrow_mut().next_counter = Some(new_counter); + executed = true; + break; + } else { + // continue to pop the undo item as this undo is a no-op + top = get_stack(&mut self.inner.lock().borrow_mut()).pop(); + continue; + } + } + + self.inner.lock().borrow_mut().processing_undo = false; + Ok(executed) + } + + pub fn can_undo(&self) -> bool { + !self.inner.lock().borrow().undo_stack.is_empty() + } + + pub fn can_redo(&self) -> bool { + !self.inner.lock().borrow().redo_stack.is_empty() + } + + pub fn undo_count(&self) -> usize { + self.inner.lock().borrow().undo_stack.len() + } + + pub fn redo_count(&self) -> usize { + self.inner.lock().borrow().redo_stack.len() + } + + /// Get the metadata of the top undo stack item, if any. + pub fn top_undo_meta(&self) -> Option { + self.inner.lock().borrow().undo_stack.peek_top_meta() + } + + /// Get the metadata of the top redo stack item, if any. + pub fn top_redo_meta(&self) -> Option { + self.inner.lock().borrow().redo_stack.peek_top_meta() + } + + /// Get the value associated with the top undo stack item, if any. + pub fn top_undo_value(&self) -> Option { + self.top_undo_meta().map(|m| m.value) + } + + /// Get the value associated with the top redo stack item, if any. + pub fn top_redo_value(&self) -> Option { + self.top_redo_meta().map(|m| m.value) + } + + /// §perf (flowstate vendor patch, act-three B.1): probe the top item of + /// the undo or redo stack without popping. Returns `(span, clean)` where + /// `clean` is true iff the stack tracks no pending remote/excluded-origin + /// diff — i.e. nothing has interleaved that would require transforming + /// the item before inverting it. Applications use this to validate a + /// recorded inverse before calling + /// [`UndoManager::external_step_applied`]. + pub fn peek_top_span(&self, kind: UndoOrRedo) -> Option<(CounterSpan, bool)> { + let lock = self.inner.lock(); + let inner = lock.borrow(); + match kind { + UndoOrRedo::Undo => inner.undo_stack.peek_top_span_and_clean(), + UndoOrRedo::Redo => inner.redo_stack.peek_top_span_and_clean(), + } + } + + /// §perf (flowstate vendor patch, act-three B.1): external fast-path + /// undo/redo — the application applies a RECORDED INVERSE of the top + /// stack item as an ordinary forward transaction instead of paying + /// `undo_internal`'s checkout-based diff computation, and this API keeps + /// the undo/redo stacks bookkeeping-identical to a native [`Self::undo`]. + /// + /// Protocol: + /// 1. `external_step_begin(kind, expected_span)` — validates the top item + /// of `kind`'s stack against `expected_span`, pops it, fires `on_pop` + /// (selection restore), and sets `processing_undo` so the inverse + /// commit is invisible to the recording subscriber — exactly like the + /// transaction inside `perform`. Returns `false` (stack unchanged) if + /// the top item is missing or mismatched: the caller must fall back to + /// the slow path. + /// 2. The application commits the inverse transaction. + /// 3. `external_step_finish(kind, inverse_span)` — pushes the + /// opposite-stack item covering the inverse commit with `on_push` + /// metadata and clears `processing_undo`. On failure of step 2 call + /// `external_step_abort(kind)` instead, which restores the popped item + /// and clears the flag. + /// + /// Preconditions the caller MUST guarantee (via [`Self::peek_top_span`] + /// plus its own lineage check): every local edit is already committed and + /// recorded; the committed inverse EXACTLY inverts the popped item; no + /// remote/excluded diff is pending for the item (`clean == true`). + pub fn external_step_begin(&self, kind: UndoOrRedo, expected_span: CounterSpan) -> bool { + let doc = &self.doc.clone(); + let popped = { + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + inner.processing_undo = true; + let stack = match kind { + UndoOrRedo::Undo => &mut inner.undo_stack, + UndoOrRedo::Redo => &mut inner.redo_stack, + }; + stack.pop() + }; + let Some((mut item, remote_diff)) = popped else { + self.inner.lock().borrow_mut().processing_undo = false; + return false; + }; + if item.span != expected_span { + // Not the item the caller validated against — restore and bail so + // the caller can fall back to the checkout-based slow path. + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + let stack = match kind { + UndoOrRedo::Undo => &mut inner.undo_stack, + UndoOrRedo::Redo => &mut inner.redo_stack, + }; + stack.push(item.span, item.meta); + inner.processing_undo = false; + return false; + } + + // Mirror `perform`'s on_pop dance (cursor transform + the + // last-popped-selection relay that keeps undo/redo loops restoring + // the selection from two steps ago). + let lock = self.inner.lock(); + let mut is_some = false; + if let Some(on_pop) = lock.borrow().on_pop.as_ref() { + is_some = true; + for cursor in item.meta.cursors.iter_mut() { + transform_cursor( + cursor, + &remote_diff.lock(), + doc, + &self.container_remap.lock(), + ); + } + on_pop(kind, item.span, item.meta.clone()); + } + if is_some { + let take = lock.borrow_mut().last_popped_selection.take(); + lock.borrow_mut().external_next_push_selection = take; + lock.borrow_mut().last_popped_selection = Some(item.meta.cursors.clone()); + } + // Stash the full popped item so an abort can restore it. + lock.borrow_mut().external_popped = Some((kind, item)); + true + } + + /// Completes an external fast-path step begun by + /// [`Self::external_step_begin`]; see its protocol docs. `inverse_span` + /// is the local-peer counter range of the inverse transaction the + /// application committed between the two calls. + pub fn external_step_finish(&self, kind: UndoOrRedo, inverse_span: CounterSpan) { + // Mirror `perform`'s opposite-stack push, with the app-committed + // inverse span standing in for the undo transaction's counter range. + let lock = self.inner.lock(); + let next_push_selection = lock.borrow_mut().external_next_push_selection.take(); + lock.borrow_mut().external_popped = None; + let mut meta = lock + .borrow() + .on_push + .as_ref() + .map(|x| x(kind.opposite(), inverse_span, None)) + .unwrap_or_default(); + let opposite_is_empty = { + let inner = lock.borrow(); + match kind { + UndoOrRedo::Undo => inner.redo_stack.is_empty(), + UndoOrRedo::Redo => inner.undo_stack.is_empty(), + } + }; + if matches!(kind, UndoOrRedo::Undo) && opposite_is_empty { + // First undo keeps the cursors captured by on_push. + } else if let Some(cursors) = next_push_selection { + meta.cursors = cursors; + } + let mut inner = lock.borrow_mut(); + match kind { + UndoOrRedo::Undo => inner.redo_stack.push(inverse_span, meta), + UndoOrRedo::Redo => inner.undo_stack.push(inverse_span, meta), + } + inner.next_counter = Some(inverse_span.end); + inner.processing_undo = false; + } + + /// Aborts an external fast-path step begun by + /// [`Self::external_step_begin`] (the application failed to commit the + /// inverse): restores the popped item and the selection relay, resyncs + /// `next_counter` past any compensation commits the application made + /// while `processing_undo` muted the recording subscriber, then clears + /// `processing_undo`. Any partial inverse ops must have been reverted by + /// the caller before this call. + pub fn external_step_abort(&self, kind: UndoOrRedo) { + let counter_end = get_counter_end(&self.doc, self.peer()); + let lock = self.inner.lock(); + let mut inner = lock.borrow_mut(); + inner.next_counter = Some(counter_end); + if let Some((popped_kind, item)) = inner.external_popped.take() { + debug_assert!(matches!( + (popped_kind, kind), + (UndoOrRedo::Undo, UndoOrRedo::Undo) | (UndoOrRedo::Redo, UndoOrRedo::Redo) + )); + // Rewind the last-popped-selection relay that begin() advanced. + let previous = inner.external_next_push_selection.take(); + inner.last_popped_selection = previous; + let stack = match kind { + UndoOrRedo::Undo => &mut inner.undo_stack, + UndoOrRedo::Redo => &mut inner.redo_stack, + }; + stack.push(item.span, item.meta); + } + inner.external_next_push_selection = None; + inner.processing_undo = false; + } + + pub fn set_on_push(&self, on_push: Option) { + self.inner.lock().borrow_mut().on_push = on_push; + } + + pub fn set_on_pop(&self, on_pop: Option) { + self.inner.lock().borrow_mut().on_pop = on_pop; + } + + pub fn clear(&self) { + self.inner.lock().borrow_mut().undo_stack.clear(); + self.inner.lock().borrow_mut().redo_stack.clear(); + } + + /// Clear only the redo stack, preserving the undo stack. + pub fn clear_redo(&self) { + self.inner.lock().borrow_mut().redo_stack.clear(); + } + + /// Clear only the undo stack, preserving the redo stack. + pub fn clear_undo(&self) { + self.inner.lock().borrow_mut().undo_stack.clear(); + } + + pub fn set_top_undo_meta(&self, meta: UndoItemMeta) { + self.inner.lock().borrow_mut().undo_stack.set_top_meta(meta); + } + + pub fn set_top_redo_meta(&self, meta: UndoItemMeta) { + self.inner.lock().borrow_mut().redo_stack.set_top_meta(meta); + } +} + +/// Undo the given spans of operations. +/// +/// # Parameters +/// +/// - `spans`: A vector of tuples where each tuple contains an `IdSpan` and its associated `Frontiers`. +/// - `IdSpan`: Represents a span of operations identified by an ID. +/// - `Frontiers`: Represents the deps of the given id_span +/// - `latest_frontiers`: The latest frontiers of the document +/// - `calc_diff`: A closure that takes two `Frontiers` and calculates the difference between them, returning a `DiffBatch`. +/// +/// # Returns +/// +/// - `DiffBatch`: Applying this batch on the `latest_frontiers` will undo the ops in the given spans. +pub(crate) fn undo( + spans: Vec<(IdSpan, Frontiers)>, + last_frontiers_or_last_bi: Either<&Frontiers, &DiffBatch>, + mut calc_diff: impl FnMut(&Frontiers, &Frontiers) -> DiffBatch, + on_last_event_a: &mut dyn FnMut(&DiffBatch), +) -> DiffBatch { + // The process of performing undo is: + // + // 0. Split the span into a series of continuous spans. There is no external dep within each continuous span. + // + // For each continuous span_i: + // + // 1. a. Calculate the event of checkout from id_span.last to id_span.deps, call it Ai. It undo the ops in the current span. + // b. Calculate A'i = Ai + T(Ci-1, Ai) if i > 0, otherwise A'i = Ai. + // NOTE: A'i can undo the ops in the current span and the previous spans, if it's applied on the id_span.last version. + // 2. Calculate the event of checkout from id_span.last to [the next span's last id] or [the latest version], call it Bi. + // 3. Transform event A'i based on Bi, call it Ci + // 4. If span_i is the last span, apply Ci to the current state. + + // ------------------------------------------------------- + // 0. Split the span into a series of continuous spans + // ------------------------------------------------------- + + let mut last_ci: Option = None; + for i in 0..spans.len() { + debug_span!("Undo", ?i, "Undo span {:?}", &spans[i]).in_scope(|| { + let (this_id_span, this_deps) = &spans[i]; + // --------------------------------------- + // 1.a Calc event A_i + // --------------------------------------- + let mut event_a_i = debug_span!("1. Calc event A_i").in_scope(|| { + // Checkout to the last id of the id_span + calc_diff(&this_id_span.id_last().into(), this_deps) + }); + + // println!("event_a_i: {:?}", event_a_i); + + // --------------------------------------- + // 2. Calc event B_i + // --------------------------------------- + let stack_diff_batch; + let event_b_i = 'block: { + let next = if i + 1 < spans.len() { + spans[i + 1].0.id_last().into() + } else { + match last_frontiers_or_last_bi { + Either::Left(last_frontiers) => last_frontiers.clone(), + Either::Right(right) => break 'block right, + } + }; + stack_diff_batch = Some(calc_diff(&this_id_span.id_last().into(), &next)); + stack_diff_batch.as_ref().unwrap() + }; + + // println!("event_b_i: {:?}", event_b_i); + + // event_a_prime can undo the ops in the current span and the previous spans + let mut event_a_prime = if let Some(mut last_ci) = last_ci.take() { + // ------------------------------------------------------------------------------ + // 1.b Transform and apply Ci-1 based on Ai, call it A'i + // ------------------------------------------------------------------------------ + last_ci.transform(&event_a_i, true); + + event_a_i.compose(&last_ci); + event_a_i + } else { + event_a_i + }; + if i == spans.len() - 1 { + on_last_event_a(&event_a_prime); + } + // -------------------------------------------------- + // 3. Transform event A'_i based on B_i, call it C_i + // -------------------------------------------------- + event_a_prime.transform(event_b_i, true); + + // println!("event_a_prime: {:?}", event_a_prime); + + let c_i = event_a_prime; + last_ci = Some(c_i); + }); + } + + last_ci.unwrap() +} diff --git a/vendor/loro-internal/src/utils/kv_wrapper.rs b/vendor/loro-internal/src/utils/kv_wrapper.rs new file mode 100644 index 00000000..3da42e60 --- /dev/null +++ b/vendor/loro-internal/src/utils/kv_wrapper.rs @@ -0,0 +1,115 @@ +use crate::sync::Mutex; +use bytes::Bytes; +use loro_kv_store::{mem_store::MemKvConfig, MemKvStore}; +use std::{collections::BTreeSet, ops::Bound, sync::Arc}; + +use crate::kv_store::KvStore; + +/// This thin wrapper aims to limit the ability to modify the kv store and make +/// it easy to find all the modifications. +pub(crate) struct KvWrapper { + kv: Arc>, +} + +impl Clone for KvWrapper { + /// Deep clone the inner kv store. + fn clone(&self) -> Self { + Self { + kv: self.kv.lock().clone_store(), + } + } +} + +impl KvWrapper { + pub fn new_mem() -> Self { + Self { + // kv: Arc::new(Mutex::new(BTreeMap::new())), + kv: Arc::new(Mutex::new(MemKvStore::new( + // set false because it's depended by GC snapshot's import & export + MemKvConfig::default().should_encode_none(false), + ))), + } + } + + pub fn arc_clone(&self) -> Self { + Self { + kv: self.kv.clone(), + } + } + + pub fn import(&self, bytes: Bytes) -> Result<(), String> { + let mut kv = self.kv.lock(); + kv.import_all(bytes) + } + + pub fn export(&self) -> Bytes { + let mut kv = self.kv.lock(); + kv.export_all() + } + + pub fn get(&self, key: &[u8]) -> Option { + let kv = self.kv.lock(); + kv.get(key) + } + + pub fn with_kv(&self, f: impl FnOnce(&dyn KvStore) -> R) -> R { + let kv = self.kv.lock(); + f(&*kv) + } + + pub(crate) fn keys(&self) -> BTreeSet> { + let kv = self.kv.lock(); + kv.scan(Bound::Unbounded, Bound::Unbounded) + .map(|(k, _)| k.to_vec()) + .collect() + } + + pub fn set_all(&self, iter: impl Iterator) { + let mut kv = self.kv.lock(); + for (k, v) in iter { + kv.set(&k, v); + } + } + + #[allow(unused)] + pub(crate) fn contains_key(&self, key: &[u8]) -> bool { + self.kv.lock().contains_key(key) + } + + pub(crate) fn remove_same(&self, old_kv: &KvWrapper) { + let other = old_kv.kv.lock(); + let mut this = self.kv.lock(); + for (k, v) in other.scan(Bound::Unbounded, Bound::Unbounded) { + if this.get(&k) == Some(v) { + this.remove(&k); + } + } + } + + pub(crate) fn remove(&self, k: &[u8]) -> Option { + self.kv.lock().remove(k) + } + + /// Remove all keys not in the given set + pub(crate) fn retain_keys(&self, keys: &BTreeSet>) { + let mut kv = self.kv.lock(); + let mut to_remove = BTreeSet::new(); + for (k, _) in kv.scan(Bound::Unbounded, Bound::Unbounded) { + if !keys.contains(&*k) { + to_remove.insert(k); + } + } + + for k in to_remove { + kv.remove(&k); + } + } + + pub(crate) fn insert(&self, k: &[u8], v: Bytes) { + self.kv.lock().set(k, v); + } + + pub(crate) fn is_empty(&self) -> bool { + self.kv.lock().is_empty() + } +} diff --git a/vendor/loro-internal/src/utils/lazy.rs b/vendor/loro-internal/src/utils/lazy.rs new file mode 100644 index 00000000..175e0d24 --- /dev/null +++ b/vendor/loro-internal/src/utils/lazy.rs @@ -0,0 +1,21 @@ +#[derive(Debug, Clone)] +pub enum LazyLoad> { + Src(Src), + Dst(Dst), +} + +impl> LazyLoad { + pub fn get_mut(&mut self) -> &mut Dst { + match self { + LazyLoad::Src(src) => { + let dst = Dst::from(std::mem::take(src)); + *self = LazyLoad::Dst(dst); + match self { + LazyLoad::Dst(dst) => dst, + _ => unreachable!(), + } + } + LazyLoad::Dst(dst) => dst, + } + } +} diff --git a/vendor/loro-internal/src/utils/mod.rs b/vendor/loro-internal/src/utils/mod.rs new file mode 100644 index 00000000..9feef9be --- /dev/null +++ b/vendor/loro-internal/src/utils/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod kv_wrapper; +pub(crate) mod lazy; +pub(crate) mod query_by_len; +pub mod string_slice; +pub(crate) mod subscription; +pub(crate) mod utf16; diff --git a/vendor/loro-internal/src/utils/query_by_len.rs b/vendor/loro-internal/src/utils/query_by_len.rs new file mode 100644 index 00000000..fd631189 --- /dev/null +++ b/vendor/loro-internal/src/utils/query_by_len.rs @@ -0,0 +1,192 @@ +use std::marker::PhantomData; + +use generic_btree::{BTreeTrait, FindResult, Query}; + +use crate::container::richtext::richtext_state::{RichtextStateChunk, RichtextTreeTrait}; + +/// An easy way to implement [Query] by using key index +/// +/// This query implementation will +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub struct IndexQuery, B: BTreeTrait> { + pub left: usize, + _data: PhantomData<(T, B)>, +} + +/// An easy way to implement [Query] by using key index +/// +/// This query implementation will +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub struct IndexQueryWithEntityIndex, B: BTreeTrait> { + pub left: usize, + pub entity_index: usize, + _data: PhantomData<(T, B)>, +} + +/// An easy way to implement [Query] by using key index +/// +/// This query implementation will +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub struct EntityIndexQueryWithEventIndex { + pub left: usize, + pub event_index: usize, +} + +impl, B: BTreeTrait> IndexQueryWithEntityIndex { + pub fn entity_index(&self) -> usize { + self.entity_index + } +} + +/// The default query implementation will +/// +/// - prefer next element when this element has zero length. +/// - prefer next element rather than previous element when the left is at the middle of two elements. +pub trait QueryByLen { + fn get_cache_entity_len(cache: &B::Cache) -> usize; + fn get_cache_len(cache: &B::Cache) -> usize; + fn get_elem_len(elem: &B::Elem) -> usize; + fn get_offset_and_found(left: usize, elem: &B::Elem) -> (usize, bool); +} + +impl, B: BTreeTrait> Query for IndexQuery { + type QueryArg = usize; + + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + _data: PhantomData, + } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut last_left = self.left; + for (i, cache) in child_caches.iter().enumerate() { + let len = T::get_cache_len(&cache.cache); + if self.left >= len { + last_left = self.left; + self.left -= len; + } else { + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + fn confirm_elem( + &mut self, + _q: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + T::get_offset_and_found(self.left, elem) + } +} + +impl, B: BTreeTrait> Query for IndexQueryWithEntityIndex { + type QueryArg = usize; + + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + entity_index: 0, + _data: PhantomData, + } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut last_left = self.left; + for (i, cache) in child_caches.iter().enumerate() { + let len = T::get_cache_len(&cache.cache); + if self.left >= len { + self.entity_index += T::get_cache_entity_len(&cache.cache); + last_left = self.left; + self.left -= len; + } else { + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + fn confirm_elem( + &mut self, + _q: &Self::QueryArg, + elem: &::Elem, + ) -> (usize, bool) { + let (offset, found) = T::get_offset_and_found(self.left, elem); + self.entity_index += offset; + (offset, found) + } +} + +impl Query for EntityIndexQueryWithEventIndex { + type QueryArg = usize; + + fn init(target: &Self::QueryArg) -> Self { + Self { + left: *target, + event_index: 0, + } + } + + fn find_node( + &mut self, + _: &Self::QueryArg, + child_caches: &[generic_btree::Child], + ) -> generic_btree::FindResult { + let mut last_left = self.left; + let mut last_event_left = self.left; + for (i, cache) in child_caches.iter().enumerate() { + let len = cache.cache.entity_len as usize; + if self.left >= len { + last_event_left = self.event_index; + self.event_index += cache.cache.event_len() as usize; + last_left = self.left; + self.left -= len; + } else { + return FindResult::new_found(i, self.left); + } + } + + self.left = last_left; + self.event_index = last_event_left; + FindResult::new_missing(child_caches.len() - 1, last_left) + } + + fn confirm_elem(&mut self, _q: &Self::QueryArg, elem: &RichtextStateChunk) -> (usize, bool) { + let left = self.left; + match elem { + RichtextStateChunk::Text(s) => { + if s.len() as usize >= left { + self.event_index += s.convert_unicode_offset_to_event_offset(left); + return (left, true); + } + + self.event_index += s.event_len() as usize; + (left, false) + } + RichtextStateChunk::Style { .. } => { + if left == 0 { + return (0, true); + } + + (left, false) + } + } + } +} diff --git a/vendor/loro-internal/src/utils/string_slice.rs b/vendor/loro-internal/src/utils/string_slice.rs new file mode 100644 index 00000000..910297cb --- /dev/null +++ b/vendor/loro-internal/src/utils/string_slice.rs @@ -0,0 +1,506 @@ +use std::{fmt::Debug, ops::Deref}; + +use append_only_bytes::BytesSlice; +use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; +use rle::Mergable; +use serde::{Deserialize, Deserializer, Serialize}; + +use crate::{ + container::richtext::richtext_state::{unicode_to_utf8_index, utf16_to_utf8_index}, + delta::DeltaValue, +}; + +use super::utf16::{count_unicode_chars, count_utf16_len}; + +pub struct StringSlice { + bytes: Variant, + /// §flowstate vendor patch (oom-leads #4): cached unicode char count. + /// `rle_len`/`length`/`len_unicode` each ran a FULL utf8 validation + + /// char-count scan per call, and `generic_btree` recomputes node caches + /// through `rle_len` on every delta-rope edit — the dominant cost of + /// remote-import event assembly on large documents. `usize::MAX` means + /// uncomputed; mutators reset it. + unicode_len_cache: std::sync::atomic::AtomicUsize, +} + +const UNICODE_LEN_UNSET: usize = usize::MAX; + +impl Clone for StringSlice { + fn clone(&self) -> Self { + Self { + bytes: self.bytes.clone(), + unicode_len_cache: std::sync::atomic::AtomicUsize::new( + self.unicode_len_cache.load(std::sync::atomic::Ordering::Relaxed), + ), + } + } +} + +impl PartialEq for StringSlice { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Eq for StringSlice {} + +#[derive(Clone, PartialEq, Eq)] +enum Variant { + BytesSlice(BytesSlice), + Owned(String), +} + +impl From for StringSlice { + fn from(s: String) -> Self { + Self::with_bytes(Variant::Owned(s)) + } +} + +impl From for StringSlice { + fn from(s: BytesSlice) -> Self { + Self::new(s) + } +} + +impl From<&str> for StringSlice { + fn from(s: &str) -> Self { + Self::with_bytes(Variant::Owned(s.to_string())) + } +} + +impl Debug for StringSlice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StringSlice") + .field("bytes", &self.as_str()) + .finish() + } +} + +impl StringSlice { + fn with_bytes(bytes: Variant) -> Self { + Self { + bytes, + unicode_len_cache: std::sync::atomic::AtomicUsize::new(UNICODE_LEN_UNSET), + } + } + + fn invalidate_unicode_len(&self) { + self.unicode_len_cache + .store(UNICODE_LEN_UNSET, std::sync::atomic::Ordering::Relaxed); + } + + /// Cached unicode char count (validated + counted at most once per + /// content; clones carry the cache). + fn cached_unicode_len(&self) -> usize { + let cached = self + .unicode_len_cache + .load(std::sync::atomic::Ordering::Relaxed); + if cached != UNICODE_LEN_UNSET { + return cached; + } + let len = count_unicode_chars(self.bytes()); + self.unicode_len_cache + .store(len, std::sync::atomic::Ordering::Relaxed); + len + } + + pub fn new(s: BytesSlice) -> Self { + std::str::from_utf8(&s).unwrap(); + Self::with_bytes(Variant::BytesSlice(s)) + } + + pub fn as_str(&self) -> &str { + match &self.bytes { + // SAFETY: `bytes` is always valid utf8 + Variant::BytesSlice(s) => unsafe { std::str::from_utf8_unchecked(s) }, + Variant::Owned(s) => s, + } + } + + pub fn len_bytes(&self) -> usize { + match &self.bytes { + Variant::BytesSlice(s) => s.len(), + Variant::Owned(s) => s.len(), + } + } + + fn bytes(&self) -> &[u8] { + match &self.bytes { + Variant::BytesSlice(s) => s.deref(), + Variant::Owned(s) => s.as_bytes(), + } + } + + pub fn len_unicode(&self) -> usize { + self.cached_unicode_len() + } + + pub fn len_utf16(&self) -> usize { + count_utf16_len(self.bytes()) + } + + pub fn is_empty(&self) -> bool { + self.bytes().is_empty() + } + + pub fn extend(&mut self, s: &str) { + self.invalidate_unicode_len(); + match &mut self.bytes { + Variant::BytesSlice(_) => { + *self = Self::with_bytes(Variant::Owned(format!("{}{}", self.as_str(), s))); + } + Variant::Owned(v) => { + v.push_str(s); + } + } + } +} + +impl std::fmt::Display for StringSlice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl<'de> Deserialize<'de> for StringSlice { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + Ok(s.into()) + } +} + +impl Serialize for StringSlice { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_str(self.as_str()) + } +} + +impl DeltaValue for StringSlice { + fn value_extend(&mut self, other: Self) -> Result<(), Self> { + self.invalidate_unicode_len(); + match (&mut self.bytes, &other.bytes) { + (Variant::BytesSlice(s), Variant::BytesSlice(o)) => match s.try_merge(o) { + Ok(_) => Ok(()), + Err(_) => Err(other), + }, + (Variant::Owned(s), _) => { + s.push_str(other.as_str()); + Ok(()) + } + _ => Err(other), + } + } + + fn take(&mut self, length: usize) -> Self { + self.invalidate_unicode_len(); + let length = if cfg!(feature = "wasm") { + utf16_to_utf8_index(self.as_str(), length).unwrap() + } else { + unicode_to_utf8_index(self.as_str(), length).unwrap() + }; + + match &mut self.bytes { + Variant::BytesSlice(s) => { + let mut other = s.slice_clone(length..); + s.slice_(..length); + std::mem::swap(s, &mut other); + Self::with_bytes(Variant::BytesSlice(other)) + } + Variant::Owned(s) => { + let mut other = s.split_off(length); + std::mem::swap(s, &mut other); + Self::with_bytes(Variant::Owned(other)) + } + } + } + + /// Unicode length of the string + /// Utf16 length when in WASM + fn length(&self) -> usize { + if cfg!(feature = "wasm") { + count_utf16_len(self.bytes()) + } else { + self.cached_unicode_len() + } + } +} + +impl HasLength for StringSlice { + fn rle_len(&self) -> usize { + if cfg!(feature = "wasm") { + count_utf16_len(self.bytes()) + } else { + self.cached_unicode_len() + } + } +} + +impl TryInsert for StringSlice { + fn try_insert(&mut self, pos: usize, elem: Self) -> Result<(), Self> + where + Self: Sized, + { + self.invalidate_unicode_len(); + match &mut self.bytes { + Variant::BytesSlice(_) => Err(elem), + Variant::Owned(s) => { + if s.capacity() >= s.len() + elem.len_bytes() { + let pos = if cfg!(feature = "wasm") { + utf16_to_utf8_index(s.as_str(), pos).unwrap() + } else { + unicode_to_utf8_index(s.as_str(), pos).unwrap() + }; + s.insert_str(pos, elem.as_str()); + Ok(()) + } else { + Err(elem) + } + } + } + + // match (&mut self.bytes, &elem.bytes) { + // (Variant::Owned(a), Variant::Owned(b)) + // // TODO: Extract magic num + // if a.capacity() >= a.len() + b.len() && a.capacity() < 128 => + // { + // a.insert_str(pos, b.as_str()); + // Ok(()) + // } + // _ => Err(elem), + // } + } +} + +impl Mergeable for StringSlice { + fn can_merge(&self, rhs: &Self) -> bool { + match (&self.bytes, &rhs.bytes) { + (Variant::BytesSlice(a), Variant::BytesSlice(b)) => a.can_merge(b), + (Variant::Owned(a), Variant::Owned(b)) => a.len() + b.len() <= a.capacity(), + _ => false, + } + } + + fn merge_right(&mut self, rhs: &Self) { + self.invalidate_unicode_len(); + match (&mut self.bytes, &rhs.bytes) { + (Variant::BytesSlice(a), Variant::BytesSlice(b)) => a.merge(b, &()), + (Variant::Owned(a), Variant::Owned(b)) => a.push_str(b.as_str()), + _ => {} + } + } + + fn merge_left(&mut self, left: &Self) { + self.invalidate_unicode_len(); + match (&mut self.bytes, &left.bytes) { + (Variant::BytesSlice(a), Variant::BytesSlice(b)) => { + let mut new = b.clone(); + new.merge(a, &()); + *a = new; + } + (Variant::Owned(a), Variant::Owned(b)) => { + a.insert_str(0, b.as_str()); + } + _ => {} + } + } +} + +impl Sliceable for StringSlice { + fn _slice(&self, range: std::ops::Range) -> Self { + let range = if cfg!(feature = "wasm") { + let start = utf16_to_utf8_index(self.as_str(), range.start).unwrap(); + let end = utf16_to_utf8_index(self.as_str(), range.end).unwrap(); + start..end + } else { + let start = unicode_to_utf8_index(self.as_str(), range.start).unwrap(); + let end = unicode_to_utf8_index(self.as_str(), range.end).unwrap(); + start..end + }; + + let bytes = match &self.bytes { + Variant::BytesSlice(s) => Variant::BytesSlice(s.slice_clone(range)), + Variant::Owned(s) => Variant::Owned(s[range].to_string()), + }; + + Self::with_bytes(bytes) + } + + fn split(&mut self, pos: usize) -> Self { + // The returned half gets a fresh cache via `with_bytes`; SELF shrinks in + // place below, so its cache must be dropped too (a stale full-length + // cache feeds `rle_len()` and corrupts later split positions). + self.invalidate_unicode_len(); + let pos = if cfg!(feature = "wasm") { + utf16_to_utf8_index(self.as_str(), pos).unwrap() + } else { + unicode_to_utf8_index(self.as_str(), pos).unwrap() + }; + + let bytes = match &mut self.bytes { + Variant::BytesSlice(s) => { + let other = s.slice_clone(pos..); + s.slice_(..pos); + Variant::BytesSlice(other) + } + Variant::Owned(s) => { + let other = s.split_off(pos); + Variant::Owned(other) + } + }; + + Self::with_bytes(bytes) + } +} + +impl Default for StringSlice { + fn default() -> Self { + StringSlice::with_bytes(Variant::Owned(String::with_capacity(32))) + } +} + +impl loro_delta::delta_trait::DeltaValue for StringSlice {} +#[allow(dead_code)] +pub fn unicode_range_to_byte_range(s: &str, start: usize, end: usize) -> (usize, usize) { + debug_assert!(start <= end); + let start_unicode_index = start; + let end_unicode_index = end; + let mut current_utf8_index = 0; + let mut start_byte = 0; + let mut end_byte = s.len(); + for (current_unicode_index, c) in s.chars().enumerate() { + if current_unicode_index == start_unicode_index { + start_byte = current_utf8_index; + } + + if current_unicode_index == end_unicode_index { + end_byte = current_utf8_index; + break; + } + + current_utf8_index += c.len_utf8(); + } + + (start_byte, end_byte) +} + +#[cfg(test)] +mod tests { + use append_only_bytes::AppendOnlyBytes; + use generic_btree::rle::{HasLength, Mergeable, Sliceable, TryInsert}; + + use crate::delta::DeltaValue; + + use super::*; + + #[test] + fn owned_slice_lengths_and_unicode_boundaries_use_logical_char_indices() { + let mut slice = StringSlice::from("aé😀b"); + assert_eq!(slice.as_str(), "aé😀b"); + assert_eq!(slice.len_bytes(), 8); + assert_eq!(slice.len_unicode(), 4); + assert_eq!(slice.len_utf16(), 5); + let event_len = if cfg!(feature = "wasm") { 5 } else { 4 }; + assert_eq!(slice.length(), event_len); + assert_eq!(slice.rle_len(), event_len); + + let middle_range = if cfg!(feature = "wasm") { 1..4 } else { 1..3 }; + let middle = slice._slice(middle_range); + assert_eq!(middle.as_str(), "é😀"); + + let right = slice.split(2); + assert_eq!(slice.as_str(), "aé"); + assert_eq!(right.as_str(), "😀b"); + + let prefix = slice.take(1); + assert_eq!(prefix.as_str(), "a"); + assert_eq!(slice.as_str(), "é"); + } + + #[test] + fn owned_slice_insert_and_merge_respect_capacity_contracts() { + let mut backing = String::with_capacity(16); + backing.push_str("a😀"); + let mut slice = StringSlice::from(backing); + slice.try_insert(1, StringSlice::from("é")).unwrap(); + assert_eq!(slice.as_str(), "aé😀"); + + let mut tight = StringSlice::from(String::from("ab")); + let rejected = tight.try_insert(1, StringSlice::from("c")).unwrap_err(); + assert_eq!(tight.as_str(), "ab"); + assert_eq!(rejected.as_str(), "c"); + + let mut left_text = String::with_capacity(8); + left_text.push_str("ab"); + let mut left = StringSlice::from(left_text); + let right = StringSlice::from("cd"); + assert!(left.can_merge(&right)); + left.merge_right(&right); + assert_eq!(left.as_str(), "abcd"); + + let mut suffix = StringSlice::from("cd"); + suffix.merge_left(&StringSlice::from("ab")); + assert_eq!(suffix.as_str(), "abcd"); + } + + #[test] + fn bytes_slice_variant_keeps_append_only_slice_semantics() { + let mut bytes = AppendOnlyBytes::new(); + bytes.push_str("left"); + let left_bytes = bytes.slice(..); + bytes.push_str("right"); + let right_bytes = bytes.slice(4..); + + let mut left = StringSlice::from(left_bytes.clone()); + let right = StringSlice::from(right_bytes.clone()); + assert_eq!(left.as_str(), "left"); + assert_eq!(right.as_str(), "right"); + assert!(left.can_merge(&right)); + left.value_extend(right).unwrap(); + assert_eq!(left.as_str(), "leftright"); + + let mut whole = StringSlice::from(bytes.slice(..)); + let tail = whole.split(4); + assert_eq!(whole.as_str(), "left"); + assert_eq!(tail.as_str(), "right"); + assert_eq!( + StringSlice::from(bytes.slice(..))._slice(1..5).as_str(), + "eftr" + ); + + let mut converted_to_owned = StringSlice::from(left_bytes); + converted_to_owned.extend("!"); + assert_eq!(converted_to_owned.as_str(), "left!"); + + let mut byte_slice = StringSlice::from(right_bytes); + let rejected = byte_slice + .try_insert(1, StringSlice::from("!")) + .unwrap_err(); + assert_eq!(byte_slice.as_str(), "right"); + assert_eq!(rejected.as_str(), "!"); + } + + #[test] + fn serde_display_default_and_range_conversion_are_string_like() { + let empty = StringSlice::default(); + assert!(empty.is_empty()); + + let slice = StringSlice::from("aé😀b"); + assert_eq!(slice.to_string(), "aé😀b"); + assert_eq!(format!("{slice:?}"), r#"StringSlice { bytes: "aé😀b" }"#); + + let encoded = serde_json::to_string(&slice).unwrap(); + assert_eq!(encoded, r#""aé😀b""#); + let decoded: StringSlice = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, slice); + + let (start, end) = unicode_range_to_byte_range("aé😀b", 1, 3); + assert_eq!(&"aé😀b"[start..end], "é😀"); + assert_eq!(unicode_range_to_byte_range("aé😀b", 2, 4), (3, 8)); + } +} diff --git a/vendor/loro-internal/src/utils/subscription.rs b/vendor/loro-internal/src/utils/subscription.rs new file mode 100644 index 00000000..764259ea --- /dev/null +++ b/vendor/loro-internal/src/utils/subscription.rs @@ -0,0 +1,668 @@ +use either::Either; +/* +This file is modified from the original file in the following repo: +https://github.com/zed-industries/zed + + +Copyright 2022 - 2024 Zed Industries, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + + http://www.apache.org/licenses/LICENSE-2.0 + + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + + 1. Definitions. + + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + + END OF TERMS AND CONDITIONS + +*/ +use crate::sync::{thread, AtomicBool, Mutex}; +use smallvec::SmallVec; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Weak}; +use std::{fmt::Debug, mem}; +use thread::ThreadId; + +#[derive(Debug)] +pub enum SubscriptionError { + CannotEmitEventDueToRecursiveCall, +} + +impl std::fmt::Display for Subscription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "SubscriptionError") + } +} + +pub struct SubscriberSet( + Arc>>, +); + +impl Clone for SubscriberSet { + fn clone(&self) -> Self { + SubscriberSet(self.0.clone()) + } +} + +struct SubscriberSetState { + subscribers: BTreeMap>, ThreadId>>, + dropped_subscribers: BTreeSet<(EmitterKey, usize)>, + next_subscriber_id: usize, +} + +struct Subscriber { + active: Arc, + callback: Callback, + /// This field is used to drop the subscription when the subscriber is dropped. + _sub: InnerSubscription, +} + +impl SubscriberSet +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync, +{ + pub fn new() -> Self { + Self(Arc::new(Mutex::new(SubscriberSetState { + subscribers: Default::default(), + dropped_subscribers: Default::default(), + next_subscriber_id: 0, + }))) + } + + /// Inserts a new [`Subscription`] for the given `emitter_key`. + /// + /// By default, subscriptions are inert, meaning that they won't be listed when calling + /// `[SubscriberSet::remove]` or `[SubscriberSet::retain]`. This method returns a tuple + /// of a [`Subscription`] and an `impl FnOnce`, and you can use the latter to activate + /// the [`Subscription`]. + pub fn insert( + &self, + emitter_key: EmitterKey, + callback: Callback, + ) -> (Subscription, impl FnOnce()) { + let active = Arc::new(AtomicBool::new(false)); + let mut lock = self.0.lock(); + let subscriber_id = post_inc(&mut lock.next_subscriber_id); + let this = Arc::downgrade(&self.0); + let emitter_key_1 = emitter_key.clone(); + let inner_sub = InnerSubscription { + unsubscribe: Arc::new(Mutex::new(Some(Box::new(move || { + let Some(this) = this.upgrade() else { + return; + }; + + let mut lock = this.lock(); + let Some(subscribers) = lock.subscribers.get_mut(&emitter_key) else { + // remove was called with this emitter_key + return; + }; + + if let Either::Left(subscribers) = subscribers { + subscribers.remove(&subscriber_id); + if subscribers.is_empty() { + lock.subscribers.remove(&emitter_key); + } + return; + } + + // We didn't manage to remove the subscription, which means it was dropped + // while invoking the callback. Mark it as dropped so that we can remove it + // later. + lock.dropped_subscribers + .insert((emitter_key, subscriber_id)); + })))), + }; + let subscription = Subscription { + unsubscribe: Arc::downgrade(&inner_sub.unsubscribe), + }; + + lock.subscribers + .entry(emitter_key_1) + .or_insert_with(|| Either::Left(BTreeMap::new())) + .as_mut() + .unwrap_left() + .insert( + subscriber_id, + Subscriber { + active: active.clone(), + callback, + _sub: inner_sub, + }, + ); + (subscription, move || active.store(true, Ordering::Relaxed)) + } + + #[allow(unused)] + pub fn remove(&self, emitter: &EmitterKey) -> impl IntoIterator { + let mut lock = self.0.lock(); + let subscribers = lock.subscribers.remove(emitter); + subscribers + .and_then(|x| x.left().map(|s| s.into_values())) + .into_iter() + .flatten() + .filter_map(|subscriber| { + if subscriber.active.load(Ordering::Relaxed) { + Some(subscriber.callback) + } else { + None + } + }) + } + + pub fn is_recursive_calling(&self, emitter: &EmitterKey) -> bool { + if let Some(Either::Right(thread_id)) = self.0.lock().subscribers.get(emitter) { + *thread_id == thread::current().id() + } else { + false + } + } + + /// Call the given callback for each subscriber to the given emitter. + /// If the callback returns false, the subscriber is removed. + pub fn retain( + &self, + emitter: &EmitterKey, + f: &mut dyn FnMut(&mut Callback) -> bool, + ) -> Result<(), SubscriptionError> { + let mut subscribers = { + let inner = loop { + let mut subscriber_set_state = self.0.lock(); + let Some(set) = subscriber_set_state.subscribers.get_mut(emitter) else { + return Ok(()); + }; + match set { + Either::Left(_) => { + break std::mem::replace(set, Either::Right(thread::current().id())) + .unwrap_left(); + } + Either::Right(lock_thread) => { + if thread::current().id() == *lock_thread { + return Err(SubscriptionError::CannotEmitEventDueToRecursiveCall); + } else { + // return Ok(()); + drop(subscriber_set_state); + #[cfg(loom)] + loom::thread::yield_now(); + #[cfg(not(loom))] + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + } + }; + inner + }; + + subscribers.retain(|_, subscriber| { + if subscriber.active.load(Ordering::Relaxed) { + f(&mut subscriber.callback) + } else { + true + } + }); + + let mut lock = self.0.lock(); + + // Add any new subscribers that were added while invoking the callback. + if let Some(Either::Left(new_subscribers)) = lock.subscribers.remove(emitter) { + subscribers.extend(new_subscribers); + } + + // Remove any dropped subscriptions that were dropped while invoking the callback. + for (dropped_emitter, dropped_subscription_id) in mem::take(&mut lock.dropped_subscribers) { + if *emitter == dropped_emitter { + subscribers.remove(&dropped_subscription_id); + } else { + lock.dropped_subscribers + .insert((dropped_emitter, dropped_subscription_id)); + } + } + + lock.subscribers + .insert(emitter.clone(), Either::Left(subscribers)); + Ok(()) + } + + pub fn is_empty(&self) -> bool { + self.0.lock().subscribers.is_empty() + } + + pub fn may_include(&self, emitter: &EmitterKey) -> bool { + self.0.lock().subscribers.contains_key(emitter) + } +} + +impl Default for SubscriberSet +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync, +{ + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Debug for SubscriberSet +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let lock = self.0.lock(); + f.debug_struct("SubscriberSet") + .field("subscriber_count", &lock.subscribers.len()) + .field("dropped_subscribers_count", &lock.dropped_subscribers.len()) + .field("next_subscriber_id", &lock.next_subscriber_id) + .finish() + } +} + +fn post_inc(next_subscriber_id: &mut usize) -> usize { + let ans = *next_subscriber_id; + *next_subscriber_id += 1; + ans +} +type Callback = Box; + +/// A handle to a subscription created by GPUI. When dropped, the subscription +/// is cancelled and the callback will no longer be invoked. +#[must_use] +pub struct Subscription { + unsubscribe: Weak>>, +} + +impl std::fmt::Debug for Subscription { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Subscription").finish() + } +} + +impl Subscription { + /// Detaches the subscription from this handle. The callback will + /// continue to be invoked until the doc has been subscribed to + /// are dropped + pub fn detach(self) { + if let Some(unsubscribe) = self.unsubscribe.upgrade() { + unsubscribe.lock().take(); + } + } + + /// Unsubscribes the subscription. + #[inline] + pub fn unsubscribe(self) { + drop(self) + } +} + +impl Drop for Subscription { + fn drop(&mut self) { + if let Some(unsubscribe) = self.unsubscribe.upgrade() { + let unsubscribe = unsubscribe.lock().take(); + if let Some(unsubscribe) = unsubscribe { + unsubscribe(); + } + } + } +} + +struct InnerSubscription { + unsubscribe: Arc>>, +} + +impl Drop for InnerSubscription { + fn drop(&mut self) { + self.unsubscribe.lock().take(); + } +} + +/// A wrapper around `SubscriberSet` that automatically handles recursive event emission. +/// +/// This struct differs from `SubscriberSet` in the following ways: +/// 1. It automatically handles the `CannotEmitEventDueToRecursiveCall` error that can occur in `SubscriberSet`. +/// 2. When a recursive event emission is detected, it queues the event instead of throwing an error. +/// 3. After the current event processing is complete, it automatically processes the queued events. +/// +/// This behavior ensures that all events are processed in the order they were emitted, even in cases +/// where recursive event emission would normally cause an error. +#[derive(Clone)] +pub struct SubscriberSetWithQueue { + subscriber_set: SubscriberSet, + queue: Arc>>>, +} + +impl Debug + for SubscriberSetWithQueue +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SubscriberSetWithQueue").finish() + } +} + +impl Default + for SubscriberSetWithQueue +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync + for<'a> FnMut(&'a Payload) -> bool, + Payload: Send + Sync + Debug, +{ + fn default() -> Self { + Self::new() + } +} + +pub struct WeakSubscriberSetWithQueue { + subscriber_set: Weak>>, + queue: Weak>>>, +} + +impl Debug + for WeakSubscriberSetWithQueue +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WeakSubscriberSetWithQueue").finish() + } +} + +impl Clone + for WeakSubscriberSetWithQueue +{ + fn clone(&self) -> Self { + Self { + subscriber_set: self.subscriber_set.clone(), + queue: self.queue.clone(), + } + } +} + +impl WeakSubscriberSetWithQueue { + pub fn upgrade(self) -> Option> { + Some(SubscriberSetWithQueue { + subscriber_set: SubscriberSet(self.subscriber_set.upgrade()?), + queue: self.queue.upgrade()?, + }) + } +} + +impl SubscriberSetWithQueue +where + EmitterKey: 'static + Ord + Clone + Debug + Send + Sync, + Callback: 'static + Send + Sync + for<'a> FnMut(&'a Payload) -> bool, + Payload: Send + Sync + Debug, +{ + pub fn new() -> Self { + Self { + subscriber_set: SubscriberSet::new(), + queue: Arc::new(Mutex::new(Default::default())), + } + } + + pub fn downgrade(&self) -> WeakSubscriberSetWithQueue { + WeakSubscriberSetWithQueue { + subscriber_set: Arc::downgrade(&self.subscriber_set.0), + queue: Arc::downgrade(&self.queue), + } + } + + pub fn inner(&self) -> &SubscriberSet { + &self.subscriber_set + } + + pub fn emit(&self, key: &EmitterKey, payload: Payload) { + let mut pending_events: SmallVec<[Payload; 1]> = SmallVec::new(); + pending_events.push(payload); + while let Some(payload) = pending_events.pop() { + let result = self + .subscriber_set + .retain(key, &mut |callback| (callback)(&payload)); + match result { + Ok(_) => { + let mut queue = self.queue.lock(); + if let Some(new_pending_events) = queue.remove(key) { + pending_events.extend(new_pending_events); + } + } + Err(SubscriptionError::CannotEmitEventDueToRecursiveCall) => { + let mut queue = self.queue.lock(); + queue.entry(key.clone()).or_default().push(payload); + } + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_inner_subscription_drop() { + let subscriber_set = SubscriberSet:: bool + Send + Sync>>::new(); + let (subscription, activate) = subscriber_set.insert(1, Box::new(move |_: &i32| true)); + activate(); + drop(subscriber_set); + assert!(subscription.unsubscribe.upgrade().is_none()); + } + + #[test] + fn test_inner_subscription_drop_2() { + let subscriber_set = SubscriberSet:: bool + Send + Sync>>::new(); + let (subscription, activate) = subscriber_set.insert(1, Box::new(move |_: &i32| false)); + activate(); + subscriber_set + .retain(&1, &mut |callback| callback(&1)) + .unwrap(); + assert!(subscription.unsubscribe.upgrade().is_none()); + } +} diff --git a/vendor/loro-internal/src/utils/utf16.rs b/vendor/loro-internal/src/utils/utf16.rs new file mode 100644 index 00000000..cc65d22b --- /dev/null +++ b/vendor/loro-internal/src/utils/utf16.rs @@ -0,0 +1,116 @@ +pub fn count_utf16_len(utf8_str: &[u8]) -> usize { + let mut utf16_count = 0; + + let mut iter = utf8_str.iter(); + + while let Some(&byte) = iter.next() { + if byte & 0b1000_0000 == 0 { + utf16_count += 1; + } else if byte & 0b1110_0000 == 0b1100_0000 { + let _ = iter.next(); + + utf16_count += 1; + } else if byte & 0b1111_0000 == 0b1110_0000 { + let _ = iter.next(); + let _ = iter.next(); + + utf16_count += 1; + } else if byte & 0b1111_1000 == 0b1111_0000 { + let u = (((byte & 0b0000_0111) as u32) << 18) + | (((iter.next().unwrap() & 0b0011_1111) as u32) << 12); + + let _ = iter.next(); + let _ = iter.next(); + if u >= 0x10000 { + utf16_count += 2; + } else { + utf16_count += 1; + } + } else { + unreachable!() + } + } + + utf16_count +} + +// TODO: FIXME: Tests +/// Count unicode chars in a utf8 string +pub fn count_unicode_chars(s: &[u8]) -> usize { + std::str::from_utf8(s).unwrap().chars().count() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ascii() { + let input = "Hello, world!"; + let expected = 13; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_2_byte() { + let input = "Привет, мир!"; + let expected = 12; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_3_byte() { + let input = "こんにちは世界"; + let expected = 7; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_4_byte() { + let input = "👋🌍"; + let expected = 4; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_crazy_emoji_bytes() { + let input = "👩‍👩‍👧‍👧👨‍👨‍👧"; + let expected = 19; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_empty_string() { + let input = ""; + let expected = 0; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_single_char() { + let input = "a"; + let expected = 1; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_utf8_with_bom() { + let input = "\u{FEFF}Hello, world!"; // UTF-8 with BOM + let expected = 14; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_long_string() { + let input = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + let expected = 52; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } + + #[test] + fn test_utf8_with_null_char() { + let input = "Hello\u{0000}world!"; + let expected = 12; + assert_eq!(count_utf16_len(input.as_bytes()), expected); + } +} diff --git a/vendor/loro-internal/src/value.rs b/vendor/loro-internal/src/value.rs new file mode 100644 index 00000000..8104eec6 --- /dev/null +++ b/vendor/loro-internal/src/value.rs @@ -0,0 +1,1222 @@ +use crate::{ + container::richtext::richtext_state::{unicode_to_utf8_index, utf16_to_utf8_index}, + delta::{Delta, DeltaItem, Meta}, + event::{Diff, Index, Path, TextDiff, TextDiffItem, TextMeta}, + handler::ValueOrHandler, + utils::string_slice::StringSlice, +}; +use generic_btree::rle::HasLength; +use loro_common::ContainerType; +pub use loro_common::LoroValue; + +// TODO: rename this trait +pub trait ToJson { + fn to_json_value(&self) -> serde_json::Value; + fn to_json(&self) -> String { + self.to_json_value().to_string() + } + fn to_json_pretty(&self) -> String { + serde_json::to_string_pretty(&self.to_json_value()).unwrap() + } + fn from_json(s: &str) -> Self; +} + +impl ToJson for LoroValue { + fn to_json_value(&self) -> serde_json::Value { + serde_json::to_value(self).unwrap() + } + + fn to_json(&self) -> String { + serde_json::to_string(self).unwrap() + } + + fn to_json_pretty(&self) -> String { + serde_json::to_string_pretty(self).unwrap() + } + + #[allow(unused)] + fn from_json(s: &str) -> Self { + serde_json::from_str(s).unwrap() + } +} + +impl ToJson for DeltaItem { + fn to_json_value(&self) -> serde_json::Value { + match self { + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + let mut map = serde_json::Map::new(); + map.insert("retain".into(), serde_json::to_value(len).unwrap()); + if !meta.is_empty() { + map.insert("attributes".into(), meta.to_json_value()); + } + serde_json::Value::Object(map) + } + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + let mut map = serde_json::Map::new(); + map.insert("insert".into(), serde_json::to_value(value).unwrap()); + if !meta.is_empty() { + map.insert("attributes".into(), meta.to_json_value()); + } + serde_json::Value::Object(map) + } + DeltaItem::Delete { + delete: len, + attributes: _, + } => { + let mut map = serde_json::Map::new(); + map.insert("delete".into(), serde_json::to_value(len).unwrap()); + serde_json::Value::Object(map) + } + } + } + + fn from_json(s: &str) -> Self { + let map: serde_json::Map = serde_json::from_str(s).unwrap(); + if map.contains_key("retain") { + let len = map["retain"].as_u64().unwrap(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + DeltaItem::Retain { + retain: len as usize, + attributes: meta, + } + } else if map.contains_key("insert") { + let value = map["insert"].as_str().unwrap().to_string().into(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + DeltaItem::Insert { + insert: value, + attributes: meta, + } + } else if map.contains_key("delete") { + let len = map["delete"].as_u64().unwrap(); + DeltaItem::Delete { + delete: len as usize, + attributes: Default::default(), + } + } else { + panic!("Invalid delta item: {}", s); + } + } +} + +fn diff_item_to_json_value(item: &TextDiffItem) -> (serde_json::Value, Option) { + match item { + loro_delta::DeltaItem::Retain { len, attr } => { + let mut map = serde_json::Map::new(); + map.insert("retain".into(), serde_json::to_value(len).unwrap()); + if !attr.is_empty() { + map.insert("attributes".into(), attr.to_json_value()); + } + (serde_json::Value::Object(map), None) + } + loro_delta::DeltaItem::Replace { + value, + attr, + delete, + } => { + let mut a = None; + let mut b = None; + if value.rle_len() > 0 { + let mut map = serde_json::Map::new(); + map.insert("insert".into(), serde_json::to_value(value).unwrap()); + if !attr.is_empty() { + map.insert("attributes".into(), attr.to_json_value()); + } + a = Some(serde_json::Value::Object(map)); + } + if *delete > 0 { + let mut map = serde_json::Map::new(); + map.insert("delete".into(), serde_json::to_value(delete).unwrap()); + b = Some(serde_json::Value::Object(map)); + } + + if a.is_none() { + a = std::mem::take(&mut b); + if a.is_none() { + let mut map = serde_json::Map::new(); + map.insert("retain".into(), serde_json::to_value(0).unwrap()); + a = Some(serde_json::Value::Object(map)); + } + } + (a.unwrap(), b) + } + } +} + +fn diff_item_from_json(v: serde_json::Value) -> TextDiffItem { + let serde_json::Value::Object(map) = v else { + panic!("Invalid delta item: {:?}", v); + }; + if map.contains_key("retain") { + let len = map["retain"].as_u64().unwrap(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + TextDiffItem::Retain { + len: len as usize, + attr: meta, + } + } else if map.contains_key("insert") { + let value = map["insert"].as_str().unwrap().to_string().into(); + let meta = if let Some(meta) = map.get("attributes") { + TextMeta::from_json(meta.to_string().as_str()) + } else { + TextMeta::default() + }; + TextDiffItem::Replace { + value, + attr: meta, + delete: 0, + } + } else if map.contains_key("delete") { + let len = map["delete"].as_u64().unwrap(); + TextDiffItem::new_delete(len as usize) + } else { + panic!("Invalid delta item: {:?}", map); + } +} + +impl ToJson for TextDiff { + fn to_json_value(&self) -> serde_json::Value { + let mut vec = Vec::new(); + for item in self.iter() { + let (a, b) = diff_item_to_json_value(item); + vec.push(a); + if let Some(b) = b { + vec.push(b); + } + } + serde_json::Value::Array(vec) + } + + fn from_json(s: &str) -> Self { + let vec: Vec = serde_json::from_str(s).unwrap(); + let mut ans = TextDiff::new(); + for item in vec.into_iter() { + ans.push(diff_item_from_json(item)); + } + ans + } +} + +impl ToJson for Delta { + fn to_json_value(&self) -> serde_json::Value { + let mut vec = Vec::new(); + for item in self.iter() { + vec.push(item.to_json_value()); + } + serde_json::Value::Array(vec) + } + + fn from_json(s: &str) -> Self { + let vec: Vec = serde_json::from_str(s).unwrap(); + let mut ans = Delta::new(); + for item in vec.into_iter() { + ans.push(DeltaItem::from_json(item.to_string().as_str())); + } + ans + } +} + +#[derive(Debug, PartialEq, Eq)] +enum TypeHint { + Map, + Text, + List, + Tree, + #[cfg(feature = "counter")] + Counter, +} + +pub trait ApplyDiff { + fn apply_diff_shallow(&mut self, diff: &[Diff]); + fn apply_diff(&mut self, diff: &[Diff]); + fn apply(&mut self, path: &Path, diff: &[Diff]); +} + +impl ApplyDiff for LoroValue { + fn apply_diff_shallow(&mut self, diff: &[Diff]) { + match self { + LoroValue::String(value) => { + let mut s = value.to_string(); + for item in diff.iter() { + let delta = item.as_text().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, attr: _ } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + let (start, end) = if cfg!(feature = "wasm") { + ( + utf16_to_utf8_index(&s, index).unwrap(), + utf16_to_utf8_index(&s, index + *delete).unwrap(), + ) + } else { + ( + unicode_to_utf8_index(&s, index).unwrap(), + unicode_to_utf8_index(&s, index + *delete).unwrap(), + ) + }; + s.replace_range(start..end, value.as_str()); + index += value.len_bytes(); + } + } + } + } + *value = s.into() + } + LoroValue::List(seq) => { + let is_tree = matches!(diff.first(), Some(Diff::Tree(_))); + if !is_tree { + let seq = seq.make_mut(); + for item in diff.iter() { + let delta = item.as_list().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, attr: _ } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + let len = value.len(); + seq.splice( + index..index + delete, + value.iter().map(|x| x.to_value()), + ); + index += len; + } + } + } + } + } else { + // let seq = Arc::make_mut(seq); + // for item in diff.iter() { + // match item { + // Diff::Tree(tree) => { + // let mut v = TreeValue(seq); + // v.apply_diff(tree); + // } + // _ => unreachable!(), + // } + // } + unimplemented!() + } + } + LoroValue::Map(map) => { + for item in diff.iter() { + match item { + Diff::Map(diff) => { + let map = map.make_mut(); + for (key, value) in diff.updated.iter() { + match &value.value { + Some(value) => { + map.insert(key.to_string(), value.to_value()); + } + None => { + map.remove(&key.to_string()); + } + } + } + } + _ => unreachable!(), + } + } + } + _ => unreachable!(), + } + } + + fn apply_diff(&mut self, diff: &[Diff]) { + match self { + LoroValue::String(value) => { + let mut s = value.to_string(); + for item in diff.iter() { + let delta = item.as_text().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, attr: _ } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + s.replace_range(index..index + *delete, value.as_str()); + index += value.len_bytes(); + } + } + } + } + *value = s.into(); + } + LoroValue::List(seq) => { + let is_tree = matches!(diff.first(), Some(Diff::Tree(_))); + if !is_tree { + let seq = seq.make_mut(); + for item in diff.iter() { + let delta = item.as_list().unwrap(); + let mut index = 0; + for delta_item in delta.iter() { + match delta_item { + loro_delta::DeltaItem::Retain { len, .. } => { + index += len; + } + loro_delta::DeltaItem::Replace { + value, + attr: _, + delete, + } => { + let value_iter = value.iter().map(unresolved_to_collection); + seq.splice(index..index + *delete, value_iter); + index += value.len(); + } + } + } + } + } else { + // let seq = Arc::make_mut(seq); + // for item in diff.iter() { + // match item { + // Diff::Tree(tree) => { + // let mut v = TreeValue(seq); + // v.apply_diff(tree); + // } + // _ => unreachable!(), + // } + // } + unimplemented!() + } + } + LoroValue::Map(map) => { + for item in diff.iter() { + match item { + Diff::Map(diff) => { + let map = map.make_mut(); + for (key, value) in diff.updated.iter() { + match &value.value { + Some(value) => { + map.insert( + key.to_string(), + unresolved_to_collection(value), + ); + } + None => { + map.remove(&key.to_string()); + } + } + } + } + _ => unreachable!(), + } + } + } + _ => unreachable!(), + } + } + + fn apply(&mut self, path: &Path, diff: &[Diff]) { + if diff.is_empty() { + return; + } + + let hint = match diff[0] { + Diff::List(_) => TypeHint::List, + Diff::Text(_) => TypeHint::Text, + Diff::Map(_) => TypeHint::Map, + Diff::Tree(_) => TypeHint::Tree, + #[cfg(feature = "counter")] + Diff::Counter(_) => TypeHint::Counter, + Diff::Unknown => unreachable!(), + }; + let value = { + let mut hints = Vec::with_capacity(path.len()); + for item in path.iter().skip(1) { + match item { + Index::Key(_) => hints.push(TypeHint::Map), + Index::Seq(_) => hints.push(TypeHint::List), + Index::Node(_) => hints.push(TypeHint::Tree), + } + } + + hints.push(hint); + let mut value: &mut LoroValue = self; + for (item, hint) in path.iter().zip(hints.iter()) { + match item { + Index::Key(key) => { + let m = value.as_map_mut().unwrap(); + let map = m.make_mut(); + value = map.entry(key.to_string()).or_insert_with(|| match hint { + TypeHint::Map => LoroValue::Map(Default::default()), + TypeHint::Text => LoroValue::String(Default::default()), + TypeHint::List => LoroValue::List(Default::default()), + TypeHint::Tree => LoroValue::List(Default::default()), + #[cfg(feature = "counter")] + TypeHint::Counter => LoroValue::Double(0.), + }) + } + Index::Seq(index) => { + let l = value.as_list_mut().unwrap(); + let list = l.make_mut(); + value = list.get_mut(*index).unwrap(); + } + Index::Node(tree_id) => { + let l = value.as_list_mut().unwrap(); + let list = l.make_mut(); + let Some(map) = list.iter_mut().find(|x| { + let id = x.as_map().unwrap().get("id").unwrap().as_string().unwrap(); + id.as_ref() == tree_id.to_string() + }) else { + // delete node first + return; + }; + let map_mut = map.as_map_mut().unwrap().make_mut(); + let meta = map_mut.get_mut("meta").unwrap(); + if meta.is_container() { + *meta = ContainerType::Map.default_value(); + } + value = meta + } + } + } + value + }; + value.apply_diff(diff); + } +} + +pub(crate) fn unresolved_to_collection(v: &ValueOrHandler) -> LoroValue { + match v { + ValueOrHandler::Value(v) => v.clone(), + ValueOrHandler::Handler(c) => c.c_type().default_value(), + } +} + +#[cfg(test)] +mod tests { + use loro_common::IdLp; + use loro_delta::{array_vec::ArrayVec, DeltaItem as EventDeltaItem}; + use rustc_hash::FxHashMap; + + use crate::{ + delta::{Delta, DeltaItem}, + event::{ListDeltaMeta, ListDiff, ListDiffInsertItem}, + handler::{Handler, HandlerTrait}, + LoroDoc, + }; + + use super::*; + + fn text_diff(items: Vec) -> TextDiff { + let mut diff = TextDiff::new(); + for item in items { + diff.push(item); + } + diff + } + + fn list_values(values: Vec) -> ListDiffInsertItem { + let mut array = ArrayVec::new(); + for value in values { + array.push(ValueOrHandler::Value(value)).unwrap(); + } + array + } + + fn list_diff(items: Vec) -> ListDiff { + let mut diff = ListDiff::new(); + for item in items { + diff.push(item); + } + diff + } + + fn map_value(value: Option, lamport: u32) -> crate::delta::ResolvedMapValue { + crate::delta::ResolvedMapValue { + value: value.map(ValueOrHandler::Value), + idlp: IdLp::new(1, lamport), + } + } + + #[test] + fn text_delta_json_roundtrip_preserves_attributes_and_operation_order() { + let mut attributes = FxHashMap::default(); + attributes.insert("bold".to_string(), LoroValue::Bool(true)); + let meta = TextMeta(attributes); + + let item = DeltaItem::Insert { + insert: StringSlice::from("hi"), + attributes: meta.clone(), + }; + assert_eq!( + item.to_json_value(), + serde_json::json!({"insert": "hi", "attributes": {"bold": true}}) + ); + assert_eq!( + DeltaItem::::from_json( + r#"{"insert":"hi","attributes":{"bold":true}}"# + ), + item + ); + + let mut delta = Delta::::new(); + delta.push(DeltaItem::Retain { + retain: 2, + attributes: meta.clone(), + }); + delta.push(DeltaItem::Insert { + insert: StringSlice::from("!"), + attributes: TextMeta::default(), + }); + delta.push(DeltaItem::Delete { + delete: 1, + attributes: TextMeta::default(), + }); + + let json = delta.to_json(); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + serde_json::json!([ + {"retain": 2, "attributes": {"bold": true}}, + {"insert": "!"}, + {"delete": 1} + ]) + ); + assert_eq!(Delta::::from_json(&json), delta); + } + + #[test] + fn text_diff_json_splits_replace_with_insert_and_delete_entries() { + let diff = text_diff(vec![TextDiffItem::Replace { + value: StringSlice::from("new"), + attr: TextMeta::default(), + delete: 2, + }]); + + assert_eq!( + diff.to_json_value(), + serde_json::json!([{"insert": "new"}, {"delete": 2}]) + ); + assert_eq!( + TextDiff::from_json(r#"[{"retain":1},{"insert":"x"},{"delete":2}]"#).to_json_value(), + serde_json::json!([{"retain": 1}, {"insert": "x"}, {"delete": 2}]) + ); + + let (empty_replace, follow_up) = diff_item_to_json_value(&TextDiffItem::Replace { + value: StringSlice::from(""), + attr: TextMeta::default(), + delete: 0, + }); + assert_eq!(empty_replace, serde_json::json!({"retain": 0})); + assert_eq!(follow_up, None); + } + + #[test] + fn apply_diff_updates_string_list_and_map_values() { + let mut text = LoroValue::String("aéz".into()); + text.apply_diff_shallow(&[Diff::Text(text_diff(vec![ + TextDiffItem::Retain { + len: 1, + attr: TextMeta::default(), + }, + TextDiffItem::Replace { + value: StringSlice::from("bc"), + attr: TextMeta::default(), + delete: 1, + }, + ]))]); + assert_eq!(text, LoroValue::String("abcz".into())); + + let mut list = + LoroValue::List(vec![LoroValue::I64(1), LoroValue::I64(2), LoroValue::I64(3)].into()); + list.apply_diff(&[Diff::List(list_diff(vec![ + EventDeltaItem::Retain { + len: 1, + attr: ListDeltaMeta::default(), + }, + EventDeltaItem::Replace { + value: list_values(vec![LoroValue::I64(20), LoroValue::I64(21)]), + attr: ListDeltaMeta::default(), + delete: 1, + }, + ]))]); + assert_eq!( + list, + LoroValue::List( + vec![ + LoroValue::I64(1), + LoroValue::I64(20), + LoroValue::I64(21), + LoroValue::I64(3), + ] + .into() + ) + ); + + let mut map = LoroValue::Map( + FxHashMap::from_iter([ + ("keep".to_string(), LoroValue::I64(1)), + ("remove".to_string(), LoroValue::I64(2)), + ]) + .into(), + ); + map.apply_diff(&[Diff::Map( + crate::delta::ResolvedMapDelta::new() + .with_entry("add".into(), map_value(Some(LoroValue::I64(3)), 1)) + .with_entry("remove".into(), map_value(None, 2)), + )]); + let map = map.as_map().unwrap(); + assert_eq!(map.get("keep"), Some(&LoroValue::I64(1))); + assert_eq!(map.get("add"), Some(&LoroValue::I64(3))); + assert_eq!(map.get("remove"), None); + } + + #[test] + fn apply_creates_missing_nested_values_from_path_and_diff_hint() { + let mut root = LoroValue::Map(Default::default()); + let mut title_path = Path::new(); + title_path.push(Index::Key("doc".into())); + title_path.push(Index::Key("title".into())); + root.apply( + &title_path, + &[Diff::Text(text_diff(vec![TextDiffItem::Replace { + value: StringSlice::from("Loro"), + attr: TextMeta::default(), + delete: 0, + }]))], + ); + + assert_eq!( + root.to_json_value(), + serde_json::json!({"doc": {"title": "Loro"}}) + ); + + let mut tags_path = Path::new(); + tags_path.push(Index::Key("doc".into())); + tags_path.push(Index::Key("tags".into())); + root.apply( + &tags_path, + &[Diff::List(list_diff(vec![EventDeltaItem::Replace { + value: list_values(vec![ + LoroValue::String("crdt".into()), + LoroValue::String("rust".into()), + ]), + attr: ListDeltaMeta::default(), + delete: 0, + }]))], + ); + assert_eq!( + root.to_json_value(), + serde_json::json!({"doc": {"title": "Loro", "tags": ["crdt", "rust"]}}) + ); + } + + #[test] + fn unresolved_container_handlers_apply_as_default_collection_values() { + let doc = LoroDoc::new_auto_commit(); + let map_handler = Handler::new_unattached(ContainerType::Map); + let text_handler = Handler::new_unattached(ContainerType::Text); + let attached_list = doc.get_list("list").to_handler(); + + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Handler(map_handler)), + LoroValue::Map(Default::default()) + ); + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Handler(text_handler)), + LoroValue::String(Default::default()) + ); + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Handler(attached_list)), + LoroValue::List(Default::default()) + ); + assert_eq!( + unresolved_to_collection(&ValueOrHandler::Value(LoroValue::Bool(true))), + LoroValue::Bool(true) + ); + } +} + +#[cfg(feature = "wasm")] +pub mod wasm { + use crate::{ + delta::{Delta, DeltaItem, Meta, StyleMeta, TreeDiff, TreeDiffItem, TreeExternalDiff}, + event::{Index, TextDiff, TextDiffItem, TextMeta}, + utils::string_slice::StringSlice, + TreeParentId, + }; + use fractional_index::FractionalIndex; + use generic_btree::rle::HasLength; + use js_sys::{Array, Object}; + use loro_common::{LoroValue, TreeID}; + use wasm_bindgen::{__rt::IntoJsResult, JsCast, JsValue}; + + impl From for JsValue { + fn from(value: Index) -> Self { + match value { + Index::Key(key) => JsValue::from_str(&key), + Index::Seq(num) => JsValue::from_f64(num as f64), + Index::Node(node) => node.into(), + } + } + } + + impl From<&TreeDiff> for JsValue { + fn from(value: &TreeDiff) -> Self { + let array = Array::new(); + for diff in value.diff.iter() { + let obj = Object::new(); + js_sys::Reflect::set(&obj, &"target".into(), &diff.target.into()).unwrap(); + match &diff.action { + TreeExternalDiff::Create { + parent, + index, + position, + } => { + js_sys::Reflect::set(&obj, &"action".into(), &"create".into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"parent".into(), + &JsValue::from(parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"index".into(), &(*index).into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"fractionalIndex".into(), + &position.to_string().into(), + ) + .unwrap(); + } + TreeExternalDiff::Delete { + old_parent, + old_index, + } => { + js_sys::Reflect::set(&obj, &"action".into(), &"delete".into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"oldParent".into(), + &JsValue::from(old_parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"oldIndex".into(), &(*old_index).into()) + .unwrap(); + } + TreeExternalDiff::Move { + parent, + index, + position, + old_parent, + old_index, + } => { + js_sys::Reflect::set(&obj, &"action".into(), &"move".into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"parent".into(), + &JsValue::from(parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"index".into(), &(*index).into()).unwrap(); + js_sys::Reflect::set( + &obj, + &"fractionalIndex".into(), + &position.to_string().into(), + ) + .unwrap(); + js_sys::Reflect::set( + &obj, + &"oldParent".into(), + &JsValue::from(old_parent.tree_id()), + ) + .unwrap(); + js_sys::Reflect::set(&obj, &"oldIndex".into(), &(*old_index).into()) + .unwrap(); + } + } + array.push(&obj); + } + array.into_js_result().unwrap() + } + } + + impl TryFrom<&JsValue> for TreeDiff { + type Error = String; + + fn try_from(value: &JsValue) -> Result { + if !value.is_array() { + return Err("Expected an array".to_string()); + } + + let array = js_sys::Array::from(value); + let mut diff = Vec::new(); + + for i in 0..array.length() { + let item = array.get(i); + if !item.is_object() { + return Err(format!("Item at index {i} is not an object")); + } + + let obj = js_sys::Object::from(item); + let target = js_sys::Reflect::get(&obj, &"target".into()) + .map_err(|e| format!("Failed to get target: {:?}", e))?; + let target = TreeID::try_from(target) + .map_err(|e| format!("Failed to parse target: {:?}", e))?; + + let action = js_sys::Reflect::get(&obj, &"action".into()) + .map_err(|e| format!("Failed to get action: {:?}", e))?; + let action = action + .as_string() + .ok_or_else(|| "action is not a string".to_string())?; + + let action = match action.as_str() { + "create" => { + let parent = js_sys::Reflect::get(&obj, &"parent".into()) + .map_err(|e| format!("Failed to get parent: {:?}", e))?; + let parent_id = if parent.is_null() || parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(parent) + .map_err(|e| format!("Failed to parse parent: {:?}", e))?, + ) + }; + let parent = TreeParentId::from(parent_id); + let index = js_sys::Reflect::get(&obj, &"index".into()) + .map_err(|e| format!("Failed to get index: {:?}", e))?; + let index = index + .as_f64() + .ok_or_else(|| "index is not a number".to_string())? + as usize; + + let position = js_sys::Reflect::get(&obj, &"fractionalIndex".into()) + .map_err(|e| format!("Failed to get fractionalIndex: {:?}", e))?; + let position = position + .as_string() + .ok_or_else(|| "fractionalIndex is not a string".to_string())?; + let position = FractionalIndex::from_hex_string(position); + + TreeExternalDiff::Create { + parent, + index, + position, + } + } + "move" => { + let parent = js_sys::Reflect::get(&obj, &"parent".into()) + .map_err(|e| format!("Failed to get parent: {:?}", e))?; + let parent_id = if parent.is_null() || parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(parent) + .map_err(|e| format!("Failed to parse parent: {:?}", e))?, + ) + }; + let parent = TreeParentId::from(parent_id); + + let index = js_sys::Reflect::get(&obj, &"index".into()) + .map_err(|e| format!("Failed to get index: {:?}", e))?; + let index = index + .as_f64() + .ok_or_else(|| "index is not a number".to_string())? + as usize; + + let position = js_sys::Reflect::get(&obj, &"fractionalIndex".into()) + .map_err(|e| format!("Failed to get fractionalIndex: {:?}", e))?; + let position = position + .as_string() + .ok_or_else(|| "fractionalIndex is not a string".to_string())?; + let position = FractionalIndex::from_hex_string(position); + + let old_parent = js_sys::Reflect::get(&obj, &"oldParent".into()) + .map_err(|e| format!("Failed to get oldParent: {:?}", e))?; + let old_parent_id = if old_parent.is_null() || old_parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(old_parent) + .map_err(|e| format!("Failed to parse oldParent: {:?}", e))?, + ) + }; + let old_parent = TreeParentId::from(old_parent_id); + + let old_index = js_sys::Reflect::get(&obj, &"oldIndex".into()) + .map_err(|e| format!("Failed to get oldIndex: {:?}", e))?; + let old_index = old_index + .as_f64() + .ok_or_else(|| "oldIndex is not a number".to_string())? + as usize; + + TreeExternalDiff::Move { + parent, + index, + position, + old_parent, + old_index, + } + } + "delete" => { + let old_parent = js_sys::Reflect::get(&obj, &"oldParent".into()) + .map_err(|e| format!("Failed to get oldParent: {:?}", e))?; + let old_parent_id = if old_parent.is_null() || old_parent.is_undefined() { + None + } else { + Some( + TreeID::try_from(old_parent) + .map_err(|e| format!("Failed to parse oldParent: {:?}", e))?, + ) + }; + let old_parent = TreeParentId::from(old_parent_id); + + let old_index = js_sys::Reflect::get(&obj, &"oldIndex".into()) + .map_err(|e| format!("Failed to get oldIndex: {:?}", e))?; + let old_index = old_index + .as_f64() + .ok_or_else(|| "oldIndex is not a number".to_string())? + as usize; + + TreeExternalDiff::Delete { + old_parent, + old_index, + } + } + action => Err(format!("Unknown tree diff action: {action}"))?, + }; + + diff.push(TreeDiffItem { target, action }); + } + + Ok(TreeDiff { diff }) + } + } + + impl From<&Delta> for JsValue { + fn from(value: &Delta) -> Self { + let arr = Array::new_with_length(value.len() as u32); + for (i, v) in value.iter().enumerate() { + arr.set(i as u32, JsValue::from(v.clone())); + } + + arr.into_js_result().unwrap() + } + } + + impl From> for JsValue { + fn from(value: DeltaItem) -> Self { + let obj = Object::new(); + match value { + DeltaItem::Retain { + retain: len, + attributes: meta, + } => { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("retain"), + &JsValue::from_f64(len as f64), + ) + .unwrap(); + if !meta.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(&meta), + ) + .unwrap(); + } + } + DeltaItem::Insert { + insert: value, + attributes: meta, + } => { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("insert"), + &JsValue::from_str(value.as_str()), + ) + .unwrap(); + if !meta.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(&meta), + ) + .unwrap(); + } + } + DeltaItem::Delete { + delete: len, + attributes: _, + } => { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("delete"), + &JsValue::from_f64(len as f64), + ) + .unwrap(); + } + } + + obj.into_js_result().unwrap() + } + } + + pub fn text_diff_to_js_value(diff: &TextDiff) -> JsValue { + let arr = Array::new(); + let mut i = 0; + for v in diff.iter() { + let (a, b) = text_diff_item_to_js_value(v); + arr.set(i as u32, a); + i += 1; + if let Some(b) = b { + arr.set(i as u32, b); + i += 1; + } + } + + arr.into_js_result().unwrap() + } + + fn text_diff_item_to_js_value(value: &TextDiffItem) -> (JsValue, Option) { + match value { + loro_delta::DeltaItem::Retain { len, attr } => { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("retain"), + &JsValue::from_f64(*len as f64), + ) + .unwrap(); + if !attr.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(attr), + ) + .unwrap(); + } + (obj.into_js_result().unwrap(), None) + } + loro_delta::DeltaItem::Replace { + value, + attr, + delete, + } => { + let mut a = None; + let mut b = None; + if value.rle_len() > 0 { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("insert"), + &JsValue::from_str(value.as_str()), + ) + .unwrap(); + if !attr.is_empty() { + js_sys::Reflect::set( + &obj, + &JsValue::from_str("attributes"), + &JsValue::from(attr), + ) + .unwrap(); + } + a = Some(obj.into_js_result().unwrap()); + } + + if *delete > 0 { + let obj = Object::new(); + js_sys::Reflect::set( + &obj, + &JsValue::from_str("delete"), + &JsValue::from_f64(*delete as f64), + ) + .unwrap(); + b = Some(obj.into_js_result().unwrap()); + } + + if a.is_none() { + a = std::mem::take(&mut b); + } + + (a.unwrap(), b) + } + } + } + + impl From<&StyleMeta> for JsValue { + fn from(value: &StyleMeta) -> Self { + // TODO: refactor: should we extract the common code of ToJson and ToJsValue + let obj = Object::new(); + for (key, style) in value.iter() { + let value = JsValue::from(style.data); + js_sys::Reflect::set(&obj, &JsValue::from_str(&key), &value).unwrap(); + } + + obj.into_js_result().unwrap() + } + } + + impl From<&TextMeta> for JsValue { + fn from(value: &TextMeta) -> Self { + let obj = Object::new(); + for (key, value) in value.0.iter() { + js_sys::Reflect::set(&obj, &JsValue::from_str(key), &JsValue::from(value.clone())) + .unwrap(); + } + + obj.into_js_result().unwrap() + } + } + + impl TryFrom<&JsValue> for TextMeta { + type Error = JsValue; + + fn try_from(value: &JsValue) -> Result { + if value.is_null() || value.is_undefined() { + return Ok(TextMeta::default()); + } + + let obj = value.dyn_ref::().ok_or("Expected an object")?; + let mut meta = TextMeta::default(); + + let entries = Object::entries(obj); + for i in 0..entries.length() { + let entry = entries.get(i); + let entry_arr = entry.dyn_ref::().ok_or("Expected an array")?; + let key = entry_arr + .get(0) + .as_string() + .ok_or("Expected a string key")?; + let value = entry_arr.get(1); + meta.0.insert(key, LoroValue::from(value)); + } + + Ok(meta) + } + } +} diff --git a/vendor/loro-internal/src/version.rs b/vendor/loro-internal/src/version.rs new file mode 100644 index 00000000..1c6ea17a --- /dev/null +++ b/vendor/loro-internal/src/version.rs @@ -0,0 +1,1197 @@ +mod frontiers; +pub use frontiers::Frontiers; + +use crate::{ + id::{Counter, ID}, + oplog::AppDag, + span::{CounterSpan, IdSpan}, + LoroError, PeerID, +}; +use loro_common::{HasCounter, HasCounterSpan, HasIdSpan, HasLamportSpan, IdFull, IdSpanVector}; +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; +use std::{ + cmp::Ordering, + ops::{ControlFlow, Deref, DerefMut}, +}; + +/// [VersionVector](https://en.wikipedia.org/wiki/Version_vector) +/// is a map from [PeerID] to [Counter]. Its a right-open interval. +/// +/// i.e. a [VersionVector] of `{A: 1, B: 2}` means that A has 1 atomic op and B has 2 atomic ops, +/// thus ID of `{client: A, counter: 1}` is out of the range. +// +// NOTE: though normally it doesn't make sense to have an entry with counter = 0, but it can be 0 in certain cases. +// Like, when using start_vv and end_vv to mark the version range, start_vv may have an entry with counter = 0. +#[repr(transparent)] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VersionVector(FxHashMap); + +#[repr(transparent)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct VersionRange(pub(crate) FxHashMap); + +#[macro_export] +macro_rules! version_range { + ($($peer:expr => ($start:expr, $end:expr)),* $(,)?) => {{ + let mut map = ::rustc_hash::FxHashMap::default(); + $( + map.insert($peer, ($start, $end)); + )* + $crate::version::VersionRange::from_map(map) + }}; +} + +impl VersionRange { + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn from_map(map: FxHashMap) -> Self { + Self(map) + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + pub fn iter_mut(&mut self) -> impl Iterator + '_ { + self.0.iter_mut() + } + + pub fn clear(&mut self) { + self.0.clear() + } + + pub fn get(&self, peer: &PeerID) -> Option<&(Counter, Counter)> { + self.0.get(peer) + } + + pub fn insert(&mut self, peer: PeerID, start: Counter, end: Counter) { + self.0.insert(peer, (start, end)); + } + + pub fn from_vv(vv: &VersionVector) -> Self { + let mut ans = Self::new(); + for (peer, counter) in vv.iter() { + let counter = normalize_vv_counter(*counter); + if counter > 0 { + ans.insert(*peer, 0, counter); + } + } + ans + } + + pub fn contains_ops_between(&self, vv_a: &VersionVector, vv_b: &VersionVector) -> bool { + for span in vv_a.sub_iter(vv_b) { + if !self.contains_id_span(IdSpan::new( + span.peer, + span.counter.start.saturating_sub(1), + span.counter.end, + )) { + return false; + } + } + + for span in vv_b.sub_iter(vv_a) { + if !self.contains_id_span(IdSpan::new( + span.peer, + span.counter.start.saturating_sub(1), + span.counter.end, + )) { + return false; + } + } + + true + } + + pub fn has_overlap_with(&self, mut span: IdSpan) -> bool { + span.normalize_(); + if let Some((start, end)) = self.get(&span.peer) { + start < &span.counter.end && end > &span.counter.start + } else { + false + } + } + + pub fn contains_id(&self, id: ID) -> bool { + if let Some((start, end)) = self.get(&id.peer) { + start <= &id.counter && end > &id.counter + } else { + false + } + } + + pub fn contains_id_span(&self, mut span: IdSpan) -> bool { + span.normalize_(); + if let Some((start, end)) = self.get(&span.peer) { + start <= &span.counter.start && end >= &span.counter.end + } else { + false + } + } + + pub fn extends_to_include_id_span(&mut self, mut span: IdSpan) { + span.normalize_(); + if let Some((start, end)) = self.0.get_mut(&span.peer) { + *start = (*start).min(span.counter.start); + *end = (*end).max(span.counter.end); + } else { + self.insert(span.peer, span.counter.start, span.counter.end); + } + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn inner(&self) -> &FxHashMap { + &self.0 + } +} + +/// Immutable version vector +/// +/// It has O(1) clone time and O(logN) insert/delete/lookup time. +/// +/// It's more memory efficient than [VersionVector] when the version vector +/// can be created from cloning and modifying other similar version vectors. +#[repr(transparent)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ImVersionVector(im::HashMap); + +#[inline] +fn normalize_vv_counter(counter: Counter) -> Counter { + counter.max(0) +} + +#[inline] +fn last_id_to_vv_end(id: ID) -> Counter { + id.counter.saturating_add(1).max(0) +} + +impl ImVersionVector { + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn clear(&mut self) { + self.0.clear() + } + + pub fn get(&self, key: &PeerID) -> Option<&Counter> { + self.0.get(key) + } + + pub fn get_mut(&mut self, key: &PeerID) -> Option<&mut Counter> { + self.0.get_mut(key) + } + + pub fn insert(&mut self, k: PeerID, v: Counter) { + self.0.insert(k, v); + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> im::hashmap::Iter<'_, PeerID, Counter> { + self.0.iter() + } + + pub fn remove(&mut self, k: &PeerID) -> Option { + self.0.remove(k) + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn contains_key(&self, k: &PeerID) -> bool { + self.0.contains_key(k) + } + + pub fn encode(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + pub fn decode(bytes: &[u8]) -> Result { + let vv = VersionVector::decode(bytes)?; + Ok(Self::from_vv(&vv)) + } + + pub fn to_vv(&self) -> VersionVector { + VersionVector(self.0.iter().map(|(&k, &v)| (k, v)).collect()) + } + + pub fn from_vv(vv: &VersionVector) -> Self { + ImVersionVector(vv.0.iter().map(|(&k, &v)| (k, v)).collect()) + } + + pub fn extend_to_include_vv<'a>( + &mut self, + vv: impl Iterator, + ) { + for (&client_id, &counter) in vv { + let counter = normalize_vv_counter(counter); + if counter == 0 { + continue; + } + + if let Some(my_counter) = self.0.get_mut(&client_id) { + *my_counter = normalize_vv_counter(*my_counter); + if *my_counter < counter { + *my_counter = counter; + } + } else { + self.0.insert(client_id, counter); + } + } + } + + #[inline] + pub fn merge(&mut self, other: &Self) { + self.extend_to_include_vv(other.0.iter()); + } + + #[inline] + pub fn merge_vv(&mut self, other: &VersionVector) { + self.extend_to_include_vv(other.0.iter()); + } + + #[inline] + pub fn set_last(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + self.0.remove(&id.peer); + } else { + self.0.insert(id.peer, end); + } + } + + pub fn extend_to_include_last_id(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + return; + } + + if let Some(counter) = self.0.get_mut(&id.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.0.insert(id.peer, end); + } + } + + pub(crate) fn includes_id(&self, x: ID) -> bool { + if self.is_empty() || x.counter < 0 { + return false; + } + + normalize_vv_counter(self.get(&x.peer).copied().unwrap_or(0)) > x.counter + } +} + +impl PartialEq for VersionVector { + fn eq(&self, other: &Self) -> bool { + self.iter().all(|(client, counter)| { + normalize_vv_counter(*other.get(client).unwrap_or(&0)) == normalize_vv_counter(*counter) + }) && other.iter().all(|(client, counter)| { + normalize_vv_counter(*self.get(client).unwrap_or(&0)) == normalize_vv_counter(*counter) + }) + } +} + +impl Eq for VersionVector {} + +impl PartialEq for ImVersionVector { + fn eq(&self, other: &Self) -> bool { + self.0.iter().all(|(client, counter)| { + normalize_vv_counter(*other.0.get(client).unwrap_or(&0)) + == normalize_vv_counter(*counter) + }) && other.0.iter().all(|(client, counter)| { + normalize_vv_counter(*self.0.get(client).unwrap_or(&0)) + == normalize_vv_counter(*counter) + }) + } +} + +impl Eq for ImVersionVector {} + +impl Deref for VersionVector { + type Target = FxHashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[derive(Default, Debug, PartialEq, Eq)] +pub struct VersionVectorDiff { + /// The spans that the `left` side needs to retreat to reach the `right` side + /// + /// these spans are included in the left, but not in the right + pub retreat: IdSpanVector, + /// The spans that the `left` side needs to forward to reach the `right` side + /// + /// these spans are included in the right, but not in the left + pub forward: IdSpanVector, +} + +impl VersionVectorDiff { + #[inline] + pub fn merge_left(&mut self, span: IdSpan) { + merge(&mut self.retreat, span); + } + + #[inline] + pub fn merge_right(&mut self, span: IdSpan) { + merge(&mut self.forward, span); + } + + #[inline] + pub fn subtract_start_left(&mut self, span: IdSpan) { + subtract_start(&mut self.retreat, span); + } + + #[inline] + pub fn subtract_start_right(&mut self, span: IdSpan) { + subtract_start(&mut self.forward, span); + } + + pub fn get_id_spans_left(&self) -> impl Iterator + '_ { + self.retreat.iter().map(|(peer, span)| IdSpan { + peer: *peer, + counter: *span, + }) + } + + pub fn get_id_spans_right(&self) -> impl Iterator + '_ { + self.forward.iter().map(|(peer, span)| IdSpan { + peer: *peer, + counter: *span, + }) + } +} + +fn subtract_start(m: &mut FxHashMap, target: IdSpan) { + if let Some(span) = m.get_mut(&target.peer) { + if span.start < target.counter.end { + span.start = target.counter.end; + } + } +} + +fn merge(m: &mut FxHashMap, mut target: IdSpan) { + target.normalize_(); + if let Some(span) = m.get_mut(&target.peer) { + span.start = span.start.min(target.counter.start); + span.end = span.end.max(target.counter.end); + } else { + m.insert(target.peer, target.counter); + } +} + +impl PartialOrd for VersionVector { + fn partial_cmp(&self, other: &Self) -> Option { + let mut self_greater = true; + let mut other_greater = true; + let mut eq = true; + for (client_id, other_end) in other.iter() { + let other_end = normalize_vv_counter(*other_end); + if let Some(self_end) = self.get(client_id) { + let self_end = normalize_vv_counter(*self_end); + if self_end < other_end { + self_greater = false; + eq = false; + } + if self_end > other_end { + other_greater = false; + eq = false; + } + } else if other_end > 0 { + self_greater = false; + eq = false; + } + } + + for (client_id, self_end) in self.iter() { + if other.contains_key(client_id) { + continue; + } else if normalize_vv_counter(*self_end) > 0 { + other_greater = false; + eq = false; + } + } + + if eq { + Some(Ordering::Equal) + } else if self_greater { + Some(Ordering::Greater) + } else if other_greater { + Some(Ordering::Less) + } else { + None + } + } +} + +impl PartialOrd for ImVersionVector { + fn partial_cmp(&self, other: &Self) -> Option { + let mut self_greater = true; + let mut other_greater = true; + let mut eq = true; + for (client_id, other_end) in other.iter() { + let other_end = normalize_vv_counter(*other_end); + if let Some(self_end) = self.get(client_id) { + let self_end = normalize_vv_counter(*self_end); + if self_end < other_end { + self_greater = false; + eq = false; + } + if self_end > other_end { + other_greater = false; + eq = false; + } + } else if other_end > 0 { + self_greater = false; + eq = false; + } + } + + for (client_id, self_end) in self.iter() { + if other.contains_key(client_id) { + continue; + } else if normalize_vv_counter(*self_end) > 0 { + other_greater = false; + eq = false; + } + } + + if eq { + Some(Ordering::Equal) + } else if self_greater { + Some(Ordering::Greater) + } else if other_greater { + Some(Ordering::Less) + } else { + None + } + } +} + +impl DerefMut for VersionVector { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl VersionVector { + pub fn diff(&self, rhs: &Self) -> VersionVectorDiff { + let mut ans: VersionVectorDiff = Default::default(); + for (client_id, &counter) in self.iter() { + let counter = normalize_vv_counter(counter); + if let Some(&rhs_counter) = rhs.get(client_id) { + let rhs_counter = normalize_vv_counter(rhs_counter); + match counter.cmp(&rhs_counter) { + Ordering::Less => { + ans.forward.insert( + *client_id, + CounterSpan { + start: counter, + end: rhs_counter, + }, + ); + } + Ordering::Greater => { + ans.retreat.insert( + *client_id, + CounterSpan { + start: rhs_counter, + end: counter, + }, + ); + } + Ordering::Equal => {} + } + } else if counter > 0 { + ans.retreat.insert( + *client_id, + CounterSpan { + start: 0, + end: counter, + }, + ); + } + } + for (client_id, &rhs_counter) in rhs.iter() { + let rhs_counter = normalize_vv_counter(rhs_counter); + if rhs_counter > 0 && !self.contains_key(client_id) { + ans.forward.insert( + *client_id, + CounterSpan { + start: 0, + end: rhs_counter, + }, + ); + } + } + + ans + } + + /// Returns two iterators that cover the differences between two version vectors. + /// + /// - The first iterator contains the spans that are in `self` but not in `rhs` + /// - The second iterator contains the spans that are in `rhs` but not in `self` + pub fn diff_iter<'a>( + &'a self, + rhs: &'a Self, + ) -> ( + impl Iterator + 'a, + impl Iterator + 'a, + ) { + (self.sub_iter(rhs), rhs.sub_iter(self)) + } + + /// Returns the spans that are in `self` but not in `rhs` + pub fn sub_iter<'a>(&'a self, rhs: &'a Self) -> impl Iterator + 'a { + self.iter().filter_map(move |(peer, &counter)| { + let counter = normalize_vv_counter(counter); + if let Some(&rhs_counter) = rhs.get(peer) { + let rhs_counter = normalize_vv_counter(rhs_counter); + if counter > rhs_counter { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: rhs_counter, + end: counter, + }, + }) + } else { + None + } + } else if counter > 0 { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: 0, + end: counter, + }, + }) + } else { + None + } + }) + } + + /// Returns the spans that are in `self` but not in `rhs` + pub fn sub_iter_im<'a>( + &'a self, + rhs: &'a ImVersionVector, + ) -> impl Iterator + 'a { + self.iter().filter_map(move |(peer, &counter)| { + let counter = normalize_vv_counter(counter); + if let Some(&rhs_counter) = rhs.get(peer) { + let rhs_counter = normalize_vv_counter(rhs_counter); + if counter > rhs_counter { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: rhs_counter, + end: counter, + }, + }) + } else { + None + } + } else if counter > 0 { + Some(IdSpan { + peer: *peer, + counter: CounterSpan { + start: 0, + end: counter, + }, + }) + } else { + None + } + }) + } + + /// Iter all span from a -> b and b -> a + pub fn iter_between<'a>(&'a self, other: &'a Self) -> impl Iterator + 'a { + // PERF: can be optimized a little + self.sub_iter(other).chain(other.sub_iter(self)) + } + + pub fn sub_vec(&self, rhs: &Self) -> IdSpanVector { + self.sub_iter(rhs).map(|x| (x.peer, x.counter)).collect() + } + + pub fn distance_between(&self, other: &Self) -> usize { + let mut ans = 0usize; + for (client_id, &counter) in self.iter() { + let counter = counter.max(0) as i64; + if let Some(&other_counter) = other.get(client_id) { + let other_counter = other_counter.max(0) as i64; + ans += counter.abs_diff(other_counter) as usize; + } else { + ans += counter as usize; + } + } + + for (client_id, &counter) in other.iter() { + if !self.contains_key(client_id) { + ans += counter.max(0) as usize; + } + } + + ans + } + + pub fn to_spans(&self) -> IdSpanVector { + self.iter() + .filter_map(|(client_id, &counter)| { + let counter = normalize_vv_counter(counter); + (counter > 0).then_some(( + *client_id, + CounterSpan { + start: 0, + end: counter, + }, + )) + }) + .collect() + } + + #[inline] + pub fn get_frontiers(&self) -> Frontiers { + self.iter() + .filter_map(|(client_id, &counter)| { + if counter > 0 { + Some(ID { + peer: *client_id, + counter: counter - 1, + }) + } else { + None + } + }) + .collect() + } + + #[inline] + pub fn new() -> Self { + Self(Default::default()) + } + + /// set the inclusive ending point. target id will be included by self + #[inline] + pub fn set_last(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + self.0.remove(&id.peer); + } else { + self.0.insert(id.peer, end); + } + } + + #[inline] + pub fn get_last(&self, client_id: PeerID) -> Option { + self.0.get(&client_id).and_then(|&x| { + let x = normalize_vv_counter(x); + if x == 0 { + None + } else { + Some(x - 1) + } + }) + } + + /// set the exclusive ending point. target id will NOT be included by self + #[inline] + pub fn set_end(&mut self, id: ID) { + if id.counter <= 0 { + self.0.remove(&id.peer); + } else { + self.0.insert(id.peer, id.counter); + } + } + + /// Update the end counter of the given client if the end is greater. + /// Return whether updated + #[inline] + pub fn try_update_last(&mut self, id: ID) -> bool { + let new_end = last_id_to_vv_end(id); + if new_end == 0 { + if self.0.get(&id.peer).is_some_and(|counter| *counter < 0) { + self.0.remove(&id.peer); + } + return false; + } + + if let Some(end) = self.0.get_mut(&id.peer) { + *end = normalize_vv_counter(*end); + if *end < new_end { + *end = new_end; + true + } else { + false + } + } else { + self.0.insert(id.peer, new_end); + true + } + } + + pub fn get_missing_span(&self, target: &Self) -> Vec { + let mut ans = vec![]; + for (client_id, other_end) in target.iter() { + let other_end = normalize_vv_counter(*other_end); + if other_end == 0 { + continue; + } + + let my_end = self + .get(client_id) + .map(|counter| normalize_vv_counter(*counter)) + .unwrap_or(0); + if my_end < other_end { + ans.push(IdSpan::new(*client_id, my_end, other_end)); + } + } + + ans + } + + pub fn merge(&mut self, other: &Self) { + for (&client_id, &other_end) in other.iter() { + let other_end = normalize_vv_counter(other_end); + if other_end == 0 { + continue; + } + + if let Some(my_end) = self.get_mut(&client_id) { + *my_end = normalize_vv_counter(*my_end); + if *my_end < other_end { + *my_end = other_end; + } + } else { + self.0.insert(client_id, other_end); + } + } + } + + pub fn includes_vv(&self, other: &VersionVector) -> bool { + match self.partial_cmp(other) { + Some(ord) => match ord { + Ordering::Less => false, + Ordering::Equal => true, + Ordering::Greater => true, + }, + None => false, + } + } + + pub fn includes_id(&self, id: ID) -> bool { + if id.counter < 0 { + return false; + } + + if let Some(end) = self.get(&id.peer) { + if normalize_vv_counter(*end) > id.counter { + return true; + } + } + false + } + + pub fn intersect_span(&self, target: IdSpan) -> Option { + if let Some(&end) = self.get(&target.peer) { + let end = normalize_vv_counter(end); + let count_start = target.ctr_start().max(0); + let count_end = target.ctr_end().max(0); + if end > count_start && count_end > count_start { + return Some(CounterSpan { + start: count_start, + end: end.min(count_end), + }); + } + } + + None + } + + pub fn extend_to_include_vv<'a>( + &mut self, + vv: impl Iterator, + ) { + for (&client_id, &counter) in vv { + let counter = normalize_vv_counter(counter); + if counter == 0 { + continue; + } + + if let Some(my_counter) = self.get_mut(&client_id) { + *my_counter = normalize_vv_counter(*my_counter); + if *my_counter < counter { + *my_counter = counter; + } + } else { + self.0.insert(client_id, counter); + } + } + } + + pub fn extend_to_include_last_id(&mut self, id: ID) { + let end = last_id_to_vv_end(id); + if end == 0 { + return; + } + + if let Some(counter) = self.get_mut(&id.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.0.insert(id.peer, end); + } + } + + pub fn extend_to_include_end_id(&mut self, id: ID) { + let end = normalize_vv_counter(id.counter); + if end == 0 { + return; + } + + if let Some(counter) = self.get_mut(&id.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.0.insert(id.peer, end); + } + } + + pub fn extend_to_include(&mut self, span: IdSpan) { + let end = normalize_vv_counter(span.counter.norm_end()); + if end == 0 { + if self.0.get(&span.peer).is_some_and(|counter| *counter < 0) { + self.0.remove(&span.peer); + } + return; + } + + if let Some(counter) = self.get_mut(&span.peer) { + *counter = normalize_vv_counter(*counter); + if *counter < end { + *counter = end; + } + } else { + self.insert(span.peer, end); + } + } + + pub fn shrink_to_exclude(&mut self, span: IdSpan) { + let start = normalize_vv_counter(span.counter.min()); + let end = normalize_vv_counter(span.counter.norm_end()); + if end <= start { + return; + } + + if start == 0 { + self.remove(&span.peer); + return; + } + + if let Some(counter) = self.get_mut(&span.peer) { + *counter = normalize_vv_counter(*counter); + if *counter > start { + *counter = start; + } + if *counter == 0 { + self.remove(&span.peer); + } + } + } + + pub fn forward(&mut self, spans: &IdSpanVector) { + for span in spans.iter() { + self.extend_to_include(IdSpan { + peer: *span.0, + counter: *span.1, + }); + } + } + + pub fn retreat(&mut self, spans: &IdSpanVector) { + for span in spans.iter() { + self.shrink_to_exclude(IdSpan { + peer: *span.0, + counter: *span.1, + }); + } + } + + pub fn intersection(&self, other: &VersionVector) -> VersionVector { + let mut ans = VersionVector::new(); + for (client_id, &counter) in self.iter() { + let counter = normalize_vv_counter(counter); + if let Some(&other_counter) = other.get(client_id) { + let other_counter = normalize_vv_counter(other_counter); + if counter < other_counter { + if counter != 0 { + ans.insert(*client_id, counter); + } + } else if other_counter != 0 { + ans.insert(*client_id, other_counter); + } + } + } + ans + } + + #[inline(always)] + pub fn encode(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + #[inline(always)] + pub fn decode(bytes: &[u8]) -> Result { + postcard::from_bytes(bytes).map_err(|_| LoroError::DecodeVersionVectorError) + } + + pub fn to_im_vv(&self) -> ImVersionVector { + ImVersionVector(self.0.iter().map(|(&k, &v)| (k, v)).collect()) + } + + pub fn from_im_vv(im_vv: &ImVersionVector) -> Self { + VersionVector(im_vv.0.iter().map(|(&k, &v)| (k, v)).collect()) + } +} + +/// Use minimal set of ids to represent the frontiers +#[tracing::instrument(skip(dag))] +pub fn shrink_frontiers(last_ids: &Frontiers, dag: &AppDag) -> Result { + // it only keep the ids of ops that are concurrent to each other + + if last_ids.len() <= 1 { + return Ok(last_ids.clone()); + } + + let mut last_ids = { + let ids = filter_duplicated_peer_id(last_ids); + if last_ids.len() == 1 { + let mut frontiers = Frontiers::default(); + frontiers.push(last_ids.as_single().unwrap()); + return Ok(frontiers); + } + + let mut last_ids = Vec::with_capacity(ids.len()); + for id in ids { + let Some(lamport) = dag.get_lamport(&id) else { + return Err(id); + }; + last_ids.push(IdFull::new(id.peer, id.counter, lamport)) + } + + last_ids + }; + + let mut frontiers = Vec::new(); + // Iterate from the greatest lamport to the smallest + last_ids.sort_by_key(|x| x.lamport); + for id in last_ids.iter().rev() { + let mut should_insert = true; + let mut len = 0; + // travel backward because they have more similar lamport + for f_id in frontiers.iter().rev() { + dag.travel_ancestors(*f_id, &mut |x| { + len += 1; + if x.contains_id(id.id()) { + should_insert = false; + ControlFlow::Break(()) + } else if x.lamport_last() < id.lamport { + // Already travel to a node with smaller lamport, no need to continue, we are sure two ops are concurrent now + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + }); + } + + if should_insert { + frontiers.push(id.id()); + } + } + + Ok(frontiers.into()) +} + +fn filter_duplicated_peer_id(last_ids: &Frontiers) -> Vec { + let mut peer_max_counters = FxHashMap::default(); + for id in last_ids.iter() { + let counter = peer_max_counters.entry(id.peer).or_insert(id.counter); + if id.counter > *counter { + *counter = id.counter; + } + } + + peer_max_counters + .into_iter() + .map(|(peer, counter)| ID::new(peer, counter)) + .collect() +} + +impl Default for VersionVector { + fn default() -> Self { + Self::new() + } +} + +impl From> for VersionVector { + fn from(map: FxHashMap) -> Self { + let mut im_map = FxHashMap::default(); + for (client_id, counter) in map { + im_map.insert(client_id, counter); + } + Self(im_map) + } +} + +impl From> for VersionVector { + fn from(vec: Vec) -> Self { + let mut vv = VersionVector::new(); + for id in vec { + vv.set_last(id); + } + + vv + } +} + +impl FromIterator for VersionVector { + fn from_iter>(iter: T) -> Self { + let iter = iter.into_iter(); + let mut vv = VersionVector(FxHashMap::with_capacity_and_hasher( + iter.size_hint().0, + Default::default(), + )); + for id in iter { + vv.set_last(id); + } + + vv + } +} + +impl FromIterator<(PeerID, Counter)> for VersionVector { + fn from_iter>(iter: T) -> Self { + VersionVector(FxHashMap::from_iter(iter)) + } +} + +// Note: It will be encoded into binary format, so the order of its fields should not be changed. + +pub fn are_frontiers_eq(a: &[ID], b: &[ID]) -> bool { + if a.len() != b.len() { + return false; + } + + let mut a: SmallVec<[ID; 10]> = a.into(); + let mut b: SmallVec<[ID; 10]> = b.into(); + + a.sort(); + b.sort(); + + a == b +} + +#[cfg(test)] +mod tests { + #![allow(clippy::neg_cmp_op_on_partial_ord)] + use super::*; + mod cmp { + use super::*; + #[test] + fn test() { + let a: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), Some(Ordering::Equal)); + assert!(a == b); + + let a: VersionVector = vec![ID::new(1, 2), ID::new(2, 1)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), None); + + assert!(!(a > b)); + assert!(!(b > a)); + assert!(!(b == a)); + + let a: VersionVector = vec![ID::new(1, 2), ID::new(2, 3)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), Some(Ordering::Greater)); + assert!(a > b); + assert!(a >= b); + + let a: VersionVector = vec![ID::new(1, 0), ID::new(2, 2)].into(); + let b: VersionVector = vec![ID::new(1, 1), ID::new(2, 2)].into(); + assert_eq!(a.partial_cmp(&b), Some(Ordering::Less)); + assert!(a < b); + assert!(a <= b); + } + } + + #[test] + fn im() { + let mut a = VersionVector::new(); + a.set_last(ID::new(1, 1)); + a.set_last(ID::new(2, 1)); + let mut b = a.clone(); + b.merge(&vec![ID::new(1, 2), ID::new(2, 2)].into()); + assert!(a != b); + assert_eq!(a.get(&1), Some(&2)); + assert_eq!(a.get(&2), Some(&2)); + assert_eq!(b.get(&1), Some(&3)); + assert_eq!(b.get(&2), Some(&3)); + } + + #[test] + fn test_encode_decode_im_version_vector() { + let vv = VersionVector::from_iter([(1, 1), (2, 2), (3, 3)]); + let im_vv = vv.to_im_vv(); + let decoded_vv = VersionVector::from_im_vv(&im_vv); + assert_eq!(vv, decoded_vv); + } + + #[test] + fn test_version_vector_encoding_decoding() { + let mut vv = VersionVector::new(); + vv.insert(1, 10); + vv.insert(2, 20); + vv.insert(3, 30); + + // Encode VersionVector + let encoded = vv.encode(); + + // Decode to ImVersionVector + let decoded_im_vv = ImVersionVector::decode(&encoded).unwrap(); + + // Convert VersionVector to ImVersionVector for comparison + let im_vv = vv.to_im_vv(); + + // Compare the original ImVersionVector with the decoded one + assert_eq!(im_vv, decoded_im_vv); + + // Convert back to VersionVector and compare + let decoded_vv = VersionVector::from_im_vv(&decoded_im_vv); + assert_eq!(vv, decoded_vv); + } +} diff --git a/vendor/loro-internal/src/version/frontiers.rs b/vendor/loro-internal/src/version/frontiers.rs new file mode 100644 index 00000000..9c26268f --- /dev/null +++ b/vendor/loro-internal/src/version/frontiers.rs @@ -0,0 +1,574 @@ +use super::*; +use either::Either; + +/// Frontiers representation. +// +// Internal Invariance: +// - Frontiers::Map(map) always have at least 2 elements. +#[derive(Clone, Default)] +pub enum Frontiers { + #[default] + None, + ID(ID), + // We use internal map to avoid the module outside accidentally create or modify the map + // to make it empty or only contain 1 element + Map(InternalMap), +} + +use std::{fmt, sync::Arc}; + +impl fmt::Debug for Frontiers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Frontiers") + .field(&FrontiersDebugHelper(self)) + .finish() + } +} + +struct FrontiersDebugHelper<'a>(&'a Frontiers); + +impl fmt::Debug for FrontiersDebugHelper<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut list = f.debug_list(); + match self.0 { + Frontiers::None => {} + Frontiers::ID(id) => { + list.entry(id); + } + Frontiers::Map(map) => { + for id in map.iter() { + list.entry(&id); + } + } + } + list.finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InternalMap(Arc>); + +impl InternalMap { + fn new() -> Self { + Self(Arc::new(FxHashMap::default())) + } + + fn len(&self) -> usize { + self.0.len() + } + + fn iter(&self) -> impl Iterator + '_ { + self.0 + .iter() + .map(|(&peer, &counter)| ID::new(peer, counter)) + } + + fn contains(&self, id: &ID) -> bool { + self.0 + .get(&id.peer) + .is_some_and(|&counter| counter == id.counter) + } + + fn insert(&mut self, id: ID) { + Arc::make_mut(&mut self.0) + .entry(id.peer) + .and_modify(|e| *e = (*e).max(id.counter)) + .or_insert(id.counter); + } + + fn remove(&mut self, id: &ID) -> bool { + let map = Arc::make_mut(&mut self.0); + if let Some(counter) = map.get_mut(&id.peer) { + if *counter == id.counter { + map.remove(&id.peer); + return true; + } + } + false + } + + fn retain(&mut self, mut f: F) + where + F: FnMut(&ID) -> bool, + { + let map = Arc::make_mut(&mut self.0); + map.retain(|&peer, &mut counter| f(&ID::new(peer, counter))); + } +} + +impl Frontiers { + pub fn len(&self) -> usize { + match self { + Frontiers::None => 0, + Frontiers::ID(_) => 1, + Frontiers::Map(map) => map.len(), + } + } + + pub fn iter(&self) -> impl Iterator + '_ { + match self { + Frontiers::None => Either::Left(Either::Left(std::iter::empty())), + Frontiers::ID(id) => Either::Left(Either::Right(std::iter::once(*id))), + Frontiers::Map(map) => Either::Right(map.iter()), + } + } + + pub fn contains(&self, id: &ID) -> bool { + match self { + Frontiers::None => false, + Frontiers::ID(inner_id) => inner_id == id, + Frontiers::Map(map) => map.contains(id), + } + } + + pub fn push(&mut self, id: ID) { + match self { + Frontiers::None => *self = Frontiers::ID(id), + Frontiers::ID(existing_id) => { + if existing_id.peer != id.peer { + let mut map = InternalMap::new(); + map.insert(*existing_id); + map.insert(id); + *self = Frontiers::Map(map); + } else if existing_id.counter < id.counter { + *existing_id = id; + } + } + Frontiers::Map(map) => map.insert(id), + } + } + + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&ID) -> bool, + { + match self { + Frontiers::None => {} + Frontiers::ID(id) => { + if !f(id) { + *self = Frontiers::None; + } + } + Frontiers::Map(map) => { + map.retain(|id| f(id)); + match map.len() { + 0 => *self = Frontiers::None, + 1 => { + let id = map.iter().next().unwrap(); + *self = Frontiers::ID(id); + } + _ => {} + } + } + } + } + + pub fn remove(&mut self, id: &ID) { + match self { + Frontiers::None => {} + Frontiers::ID(existing_id) => { + if existing_id == id { + *self = Frontiers::None; + } + } + Frontiers::Map(map) => { + if map.remove(id) { + match map.len() { + 0 => *self = Frontiers::None, + 1 => { + let id = map.iter().next().unwrap(); + *self = Frontiers::ID(id); + } + _ => {} + } + } + } + } + } +} + +impl PartialEq for Frontiers { + fn eq(&self, other: &Self) -> bool { + let len = self.len(); + if len != other.len() { + return false; + } + + match (self, other) { + (Frontiers::None, Frontiers::None) => true, + (Frontiers::ID(id1), Frontiers::ID(id2)) => id1 == id2, + (Frontiers::Map(map1), Frontiers::Map(map2)) => map1 == map2, + _ => unreachable!(), + } + } +} + +impl Eq for Frontiers {} + +impl Frontiers { + pub fn new() -> Self { + Self::None + } + + #[inline] + pub fn from_id(id: ID) -> Self { + Self::ID(id) + } + + #[inline] + pub fn encode(&self) -> Vec { + let mut vec: Vec = self.iter().collect(); + vec.sort(); + postcard::to_allocvec(&vec).unwrap() + } + + #[inline] + pub fn decode(bytes: &[u8]) -> Result { + let vec: Vec = postcard::from_bytes(bytes).map_err(|_| { + LoroError::DecodeError("Decode Frontiers error".to_string().into_boxed_str()) + })?; + Ok(Self::from(vec)) + } + + pub fn update_frontiers_on_new_change(&mut self, id: ID, deps: &Frontiers) { + if self.len() <= 8 && self == deps { + *self = Frontiers::from_id(id); + return; + } + + // Remove all IDs in deps from self + for dep in deps.iter() { + self.remove(&dep); + } + + // Add the new ID + self.push(id); + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Returns the single ID if the Frontiers contains exactly one ID, otherwise returns None. + pub fn as_single(&self) -> Option { + match self { + Frontiers::ID(id) => Some(*id), + _ => None, + } + } + + /// Returns a reference to the internal map if the Frontiers contains multiple IDs, + /// otherwise returns None. + pub fn as_map(&self) -> Option<&InternalMap> { + match self { + Frontiers::Map(map) => Some(map), + _ => None, + } + } + + /// Merges another Frontiers into this one. + /// + /// Id from other will override the id with the same peer from self. + pub fn merge_with_greater(&mut self, other: &Frontiers) { + if self.is_empty() { + *self = other.clone(); + return; + } + + if let Some(id) = self.as_single() { + match other { + Frontiers::None => {} + Frontiers::ID(other_id) => { + if id.peer == other_id.peer { + *self = Frontiers::ID(ID::new(id.peer, id.counter.max(other_id.counter))); + } else { + self.push(*other_id); + } + return; + } + Frontiers::Map(internal_map) => { + let mut map = internal_map.clone(); + Arc::make_mut(&mut map.0) + .entry(id.peer) + .and_modify(|c| *c = (*c).max(id.counter)) + .or_insert(id.counter); + *self = Frontiers::Map(map); + } + } + + return; + } + + let Frontiers::Map(map) = self else { + unreachable!() + }; + let map = Arc::make_mut(&mut map.0); + for id in other.iter() { + map.entry(id.peer) + .and_modify(|c| *c = (*c).max(id.counter)) + .or_insert(id.counter); + } + } + + pub fn to_vec(&self) -> Vec { + match self { + Frontiers::None => Vec::new(), + Frontiers::ID(id) => vec![*id], + Frontiers::Map(map) => map.iter().collect(), + } + } + + /// Keeps only one element in the Frontiers, deleting all others. + /// If the Frontiers is empty, it remains empty. + /// If it contains multiple elements, it keeps the first one encountered. + pub fn keep_one(&mut self) { + match self { + Frontiers::None => {} + Frontiers::ID(_) => {} + Frontiers::Map(map) => { + if let Some((&peer, &counter)) = map.0.iter().next() { + *self = Frontiers::ID(ID::new(peer, counter)); + } + } + } + } +} +impl From<&[ID]> for Frontiers { + fn from(ids: &[ID]) -> Self { + match ids.len() { + 0 => Frontiers::None, + 1 => Frontiers::ID(ids[0]), + _ => { + let mut map = InternalMap::new(); + for &id in ids { + map.insert(id); + } + Frontiers::Map(map) + } + } + } +} + +impl From> for Frontiers { + fn from(ids: Vec) -> Self { + match ids.len() { + 0 => Frontiers::None, + 1 => Frontiers::ID(ids[0]), + _ => { + let mut map = InternalMap::new(); + for id in ids { + map.insert(id); + } + Frontiers::Map(map) + } + } + } +} + +impl From for Frontiers { + fn from(value: ID) -> Self { + Self::ID(value) + } +} + +impl FromIterator for Frontiers { + fn from_iter>(iter: I) -> Self { + let mut new = Self::new(); + for id in iter { + new.push(id); + } + new + } +} + +impl From> for Frontiers { + fn from(value: Option) -> Self { + match value { + Some(id) => Frontiers::ID(id), + None => Frontiers::None, + } + } +} + +impl From<[ID; N]> for Frontiers { + fn from(value: [ID; N]) -> Self { + match N { + 0 => Frontiers::None, + 1 => Frontiers::ID(value[0]), + _ => { + let mut map = InternalMap::new(); + for id in value { + map.insert(id); + } + Frontiers::Map(map) + } + } + } +} + +impl From<&Vec> for Frontiers { + fn from(ids: &Vec) -> Self { + match ids.len() { + 0 => Frontiers::None, + 1 => Frontiers::ID(ids[0]), + _ => { + let mut map = InternalMap::new(); + for id in ids { + map.insert(*id); + } + Frontiers::Map(map) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_frontiers_push_insert_remove() { + let mut frontiers = Frontiers::None; + + // Test push + frontiers.push(ID::new(1, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 1))); + + frontiers.push(ID::new(2, 1)); + assert!(matches!(frontiers, Frontiers::Map(_))); + assert_eq!(frontiers.len(), 2); + + frontiers.push(ID::new(1, 2)); + assert_eq!(frontiers.len(), 2); + assert!(frontiers.contains(&ID::new(1, 2))); + assert!(!frontiers.contains(&ID::new(1, 1))); + + // Test insert (via InternalMap) + if let Frontiers::Map(ref mut map) = frontiers { + map.insert(ID::new(3, 1)); + } + assert_eq!(frontiers.len(), 3); + assert!(frontiers.contains(&ID::new(3, 1))); + + // Test remove + frontiers.remove(&ID::new(2, 1)); + assert_eq!(frontiers.len(), 2); + assert!(!frontiers.contains(&ID::new(2, 1))); + + frontiers.remove(&ID::new(1, 2)); + assert_eq!(frontiers, Frontiers::ID(ID::new(3, 1))); + + frontiers.remove(&ID::new(3, 1)); + assert_eq!(frontiers, Frontiers::None); + } + + #[test] + fn test_frontiers_edge_cases() { + let mut frontiers = Frontiers::None; + + // Push to empty + frontiers.push(ID::new(1, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 1))); + + // Push same peer, higher counter + frontiers.push(ID::new(1, 2)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 2))); + + // Push same peer, lower counter (should not change) + frontiers.push(ID::new(1, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 2))); + + // Push different peer + frontiers.push(ID::new(2, 1)); + assert!(matches!(frontiers, Frontiers::Map(_))); + assert_eq!(frontiers.len(), 2); + + // Remove non-existent + frontiers.remove(&ID::new(3, 1)); + assert_eq!(frontiers.len(), 2); + + // Remove until only one left + frontiers.remove(&ID::new(2, 1)); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 2))); + + // Remove last + frontiers.remove(&ID::new(1, 2)); + assert_eq!(frontiers, Frontiers::None); + } + + #[test] + fn test_frontiers_retain() { + let mut frontiers = Frontiers::None; + + // Test retain on empty frontiers + frontiers.retain(|_| true); + assert_eq!(frontiers, Frontiers::None); + + // Test retain on single ID + frontiers.push(ID::new(1, 1)); + frontiers.retain(|id| id.peer == 1); + assert_eq!(frontiers, Frontiers::ID(ID::new(1, 1))); + + frontiers.retain(|id| id.peer == 2); + assert_eq!(frontiers, Frontiers::None); + + // Test retain on multiple IDs + frontiers.push(ID::new(1, 1)); + frontiers.push(ID::new(2, 2)); + frontiers.push(ID::new(3, 3)); + + // Retain only even peer IDs + frontiers.retain(|id| id.peer % 2 == 0); + assert_eq!(frontiers, Frontiers::ID(ID::new(2, 2))); + + // Add more IDs and test retaining multiple + frontiers.push(ID::new(1, 1)); + frontiers.push(ID::new(3, 3)); + frontiers.push(ID::new(4, 4)); + + frontiers.retain(|id| id.peer > 2); + assert!(matches!(frontiers, Frontiers::Map(_))); + assert_eq!(frontiers.len(), 2); + assert!(frontiers.contains(&ID::new(3, 3))); + assert!(frontiers.contains(&ID::new(4, 4))); + + // Retain none + frontiers.retain(|_| false); + assert_eq!(frontiers, Frontiers::None); + } + + #[test] + fn test_frontiers_encode_decode() { + let mut frontiers = Frontiers::None; + + // Test encode/decode for empty frontiers + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test encode/decode for single ID + frontiers.push(ID::new(1, 100)); + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test encode/decode for multiple IDs + frontiers.push(ID::new(2, 200)); + frontiers.push(ID::new(3, 300)); + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test encode/decode for many IDs + for i in 4..20 { + frontiers.push(ID::new(i, i as Counter * 100)); + } + let encoded = frontiers.encode(); + let decoded = Frontiers::decode(&encoded).unwrap(); + assert_eq!(frontiers, decoded); + + // Test decode with invalid input + assert!(Frontiers::decode(&[0xFF]).is_err()); + assert!(Frontiers::decode(&[]).is_err()); + } +} diff --git a/vendor/loro-internal/tests/autocommit.rs b/vendor/loro-internal/tests/autocommit.rs new file mode 100644 index 00000000..524ed2fb --- /dev/null +++ b/vendor/loro-internal/tests/autocommit.rs @@ -0,0 +1,78 @@ +use loro_common::ID; +use loro_internal::{ + cursor::PosType, loro::ExportMode, version::Frontiers, HandlerTrait, LoroDoc, TextHandler, + ToJson, +}; +use serde_json::json; + +#[test] +fn auto_commit() { + let doc_a = LoroDoc::default(); + doc_a.set_peer_id(1).unwrap(); + doc_a.start_auto_commit(); + let text_a = doc_a.get_text("text"); + text_a.insert(0, "hello", PosType::Unicode).unwrap(); + text_a.delete(2, 2, PosType::Unicode).unwrap(); + assert_eq!(&**text_a.get_value().as_string().unwrap(), "heo"); + let bytes = doc_a.export(ExportMode::all_updates()).unwrap(); + + let doc_b = LoroDoc::default(); + doc_b.start_auto_commit(); + doc_b.set_peer_id(2).unwrap(); + let text_b = doc_b.get_text("text"); + text_b.insert(0, "100", PosType::Unicode).unwrap(); + doc_b.import(&bytes).unwrap(); + doc_a + .import(&doc_b.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + assert_eq!(text_a.get_value(), text_b.get_value()); + doc_a.check_state_diff_calc_consistency_slow(); +} + +#[test] +fn auto_commit_list() { + let doc_a = LoroDoc::default(); + doc_a.start_auto_commit(); + let list_a = doc_a.get_list("list"); + list_a.insert(0, "hello").unwrap(); + assert_eq!(list_a.get_value().to_json_value(), json!(["hello"])); + let text_a = list_a + .insert_container(0, TextHandler::new_detached()) + .unwrap(); + let text = text_a; + text.insert(0, "world", PosType::Unicode).unwrap(); + let value = doc_a.get_deep_value(); + assert_eq!(value.to_json_value(), json!({"list": ["world", "hello"]})) +} + +#[ctor::ctor] +fn init() { + dev_utils::setup_test_log(); +} + +#[test] +fn auto_commit_with_checkout() { + let doc = LoroDoc::default(); + doc.set_peer_id(1).unwrap(); + doc.start_auto_commit(); + let map = doc.get_map("a"); + map.insert("0", 0).unwrap(); + map.insert("1", 1).unwrap(); + map.insert("2", 2).unwrap(); + map.insert("3", 3).unwrap(); + doc.checkout(&Frontiers::from(ID::new(1, 0))).unwrap(); + assert_eq!(map.get_value().to_json_value(), json!({"0": 0})); + // assert error if insert after checkout + map.insert("4", 4).unwrap_err(); + doc.checkout_to_latest(); + // assert ok if doc is attached + map.insert("4", 4).unwrap(); + let expected = json!({"0": 0, "1": 1, "2": 2, "3": 3, "4": 4}); + + // should include all changes + let new = LoroDoc::default(); + let a = new.get_map("a"); + new.import(&doc.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + assert_eq!(a.get_value().to_json_value(), expected,); +} diff --git a/vendor/loro-internal/tests/contracts.rs b/vendor/loro-internal/tests/contracts.rs new file mode 100644 index 00000000..7cd6daa6 --- /dev/null +++ b/vendor/loro-internal/tests/contracts.rs @@ -0,0 +1,14 @@ +#[path = "contracts/event_diff.rs"] +mod event_diff; +#[path = "contracts/handler.rs"] +mod handler; +#[path = "contracts/jsonpath_value.rs"] +mod jsonpath_value; +#[path = "contracts/public_doc.rs"] +mod public_doc; +#[path = "contracts/text_richtext_behavior.rs"] +mod text_richtext_behavior; +#[path = "contracts/text_richtext_handler.rs"] +mod text_richtext_handler; +#[path = "contracts/tree_handler.rs"] +mod tree_handler; diff --git a/vendor/loro-internal/tests/contracts/event_diff.rs b/vendor/loro-internal/tests/contracts/event_diff.rs new file mode 100644 index 00000000..0689821b --- /dev/null +++ b/vendor/loro-internal/tests/contracts/event_diff.rs @@ -0,0 +1,160 @@ +use fractional_index::FractionalIndex; +use loro_delta::{array_vec::ArrayVec, delta_trait::DeltaAttr}; +use loro_internal::{ + delta::{ + DeltaType, Meta, ResolvedMapDelta, ResolvedMapValue, TreeDiff, TreeDiffItem, + TreeExternalDiff, + }, + event::{ + path_to_str, str_to_path, Diff, Index, ListDeltaMeta, ListDiff, ListDiffInsertItem, + TextDiff, TextMeta, + }, + handler::ValueOrHandler, + FxHashMap, IdLp, LoroValue, StringSlice, TreeID, TreeParentId, +}; +use pretty_assertions::assert_eq; + +fn list_values(values: impl IntoIterator) -> ListDiffInsertItem { + let mut array = ArrayVec::new(); + for value in values { + array.push(ValueOrHandler::Value(value)).unwrap(); + } + array +} + +fn list_insert(values: impl IntoIterator, from_move: bool) -> ListDiff { + let mut diff = ListDiff::new(); + diff.push_insert(list_values(values), ListDeltaMeta { from_move }); + diff +} + +fn text_insert(value: &str) -> TextDiff { + let mut diff = TextDiff::new(); + diff.push_insert(StringSlice::from(value), TextMeta::default()); + diff +} + +fn map_diff(key: &str, value: Option, lamport: u32) -> ResolvedMapDelta { + ResolvedMapDelta::new().with_entry( + key.into(), + ResolvedMapValue { + value: value.map(ValueOrHandler::Value), + idlp: IdLp::new(1, lamport), + }, + ) +} + +fn tree_create(target: TreeID, index: usize) -> TreeDiff { + TreeDiff { + diff: vec![TreeDiffItem { + target, + action: TreeExternalDiff::Create { + parent: TreeParentId::Root, + index, + position: FractionalIndex::default(), + }, + }], + } +} + +#[test] +fn event_diff_helpers_compose_transform_and_report_empty_by_kind() { + assert_eq!(format!("{:?}", Index::Seq(7)), "Index::Seq(7)"); + assert_eq!( + format!("{:?}", Index::Node(TreeID::new(3, 9))), + "Index::Node(9@3)" + ); + assert_eq!(Index::from(4).to_string(), "4"); + assert_eq!( + path_to_str(&[Index::Key("root".into()), Index::Seq(2)]), + "root/2" + ); + assert_eq!( + str_to_path("root/2/9@3"), + Some(vec![ + Index::Key("root".into()), + Index::Seq(2), + Index::Node(TreeID::new(3, 9)), + ]) + ); + + let mut attrs = FxHashMap::default(); + attrs.insert("bold".to_string(), LoroValue::Bool(true)); + let text_meta = TextMeta(attrs.clone()); + let roundtrip_attrs: FxHashMap = text_meta.clone().into(); + assert_eq!(roundtrip_attrs, attrs); + assert_eq!(TextMeta::from(attrs), text_meta); + + let mut list_meta = ListDeltaMeta::default(); + assert!(::is_empty(&list_meta)); + ::compose( + &mut list_meta, + &ListDeltaMeta { from_move: true }, + (DeltaType::Retain, DeltaType::Retain), + ); + assert!(list_meta.from_move); + assert!(::is_mergeable( + &list_meta, + &ListDeltaMeta { from_move: true } + )); + ::merge(&mut list_meta, &ListDeltaMeta { from_move: true }); + + let mut delta_attr = ListDeltaMeta::default(); + assert!(DeltaAttr::attr_is_empty(&delta_attr)); + DeltaAttr::compose(&mut delta_attr, &ListDeltaMeta { from_move: true }); + assert!(delta_attr.from_move); + assert!(!DeltaAttr::attr_is_empty(&delta_attr)); + + let mut list = Diff::List(list_insert([LoroValue::I64(1)], false)); + list.compose_ref(&Diff::List(list_insert([LoroValue::I64(2)], true))); + assert!(!list.is_empty()); + let list_composed = Diff::List(list_insert([LoroValue::I64(3)], false)) + .compose(Diff::List(list_insert([LoroValue::I64(4)], false))) + .expect("same diff kinds should compose"); + assert!(!list_composed.is_empty()); + + let mut text = Diff::Text(text_insert("a")); + text.compose_ref(&Diff::Text(text_insert("b"))); + assert!(!text.is_empty()); + let text_composed = Diff::Text(text_insert("c")) + .compose(Diff::Text(text_insert("d"))) + .expect("text diffs should compose"); + assert!(!text_composed.is_empty()); + + let map_a = Diff::Map(map_diff("title", Some("old".into()), 1)); + let map_b = Diff::Map(map_diff("title", Some("new".into()), 2)); + let mut map_composed = map_a + .clone() + .compose(map_b.clone()) + .expect("map diffs should compose"); + assert!(!map_composed.is_empty()); + map_composed.transform(&map_b, false); + assert!(map_composed.is_empty()); + + let target = TreeID::new(1, 2); + let tree_a = Diff::Tree(tree_create(target, 0)); + let tree_b = Diff::Tree(tree_create(target, 1)); + let mut tree_composed = tree_a + .clone() + .compose(tree_b.clone()) + .expect("tree diffs should compose"); + assert!(!tree_composed.is_empty()); + tree_composed.transform(&tree_b, false); + assert!(tree_composed.is_empty()); + + assert!(Diff::Unknown.is_empty()); + let wrong_kind = Diff::Text(text_insert("x")).compose(Diff::List(list_insert([], false))); + assert!(matches!(wrong_kind, Err(Diff::Text(_)))); + + #[cfg(feature = "counter")] + { + let counter = Diff::Counter(1.0) + .compose(Diff::Counter(2.0)) + .expect("counter diffs should compose"); + assert!(matches!(counter, Diff::Counter(3.0))); + let mut transformed = Diff::Counter(1.0); + transformed.transform(&Diff::Counter(2.0), false); + assert!(matches!(transformed, Diff::Counter(1.0))); + assert!(Diff::Counter(0.0).is_empty()); + } +} diff --git a/vendor/loro-internal/tests/contracts/handler.rs b/vendor/loro-internal/tests/contracts/handler.rs new file mode 100644 index 00000000..bbd882be --- /dev/null +++ b/vendor/loro-internal/tests/contracts/handler.rs @@ -0,0 +1,179 @@ +use loro_internal::{ + handler::{Handler, ListHandler, MapHandler, MovableListHandler, TextHandler, TreeHandler}, + HandlerTrait, LoroDoc, ToJson, TreeID, TreeParentId, +}; +use pretty_assertions::assert_eq; +use serde_json::json; + +#[test] +fn detached_handlers_expose_local_values_ids_and_clear_contracts() -> loro_internal::LoroResult<()> +{ + let map = MapHandler::new_detached(); + map.insert("title", "draft")?; + let text = map.insert_container("body", TextHandler::new_detached())?; + text.insert_unicode(0, "hello")?; + + let list = ListHandler::new_detached(); + list.insert(0, "a")?; + list.insert_container(1, MapHandler::new_detached())? + .insert("kind", "nested")?; + + let movable = MovableListHandler::new_detached(); + movable.insert(0, "first")?; + movable.insert(1, "second")?; + movable.mov(1, 0)?; + + let tree = TreeHandler::new_detached(); + let root = tree.create(TreeParentId::Root)?; + tree.get_meta(root)?.insert("name", "root")?; + + assert_eq!(format!("{map:?}"), "MapHandler Detached"); + assert_eq!(format!("{list:?}"), "ListHandler Detached"); + assert_eq!( + format!("{movable:?}"), + format!("MovableListHandler {}", movable.id()) + ); + assert_eq!(format!("{text:?}"), "TextHandler(Unattached)"); + assert_eq!(format!("{tree:?}"), "TreeHandler Detached"); + + assert!(!map.is_attached()); + assert!(!list.is_attached()); + assert!(!movable.is_attached()); + assert!(!text.is_attached()); + assert!(!tree.is_attached()); + assert!(map.attached_handler().is_none()); + assert!(list.attached_handler().is_none()); + assert!(movable.attached_handler().is_none()); + assert!(text.attached_handler().is_none()); + assert!(tree.attached_handler().is_none()); + assert!(map.doc().is_none()); + assert!(list.doc().is_none()); + assert!(movable.doc().is_none()); + assert!(text.doc().is_none()); + assert!(tree.doc().is_none()); + assert!(map.parent().is_none()); + assert!(list.parent().is_none()); + assert!(movable.parent().is_none()); + assert!(text.parent().is_none()); + assert!(tree.parent().is_none()); + + assert_eq!( + map.get_deep_value().to_json_value(), + json!({"body": "hello", "title": "draft"}) + ); + assert_eq!( + list.get_deep_value().to_json_value(), + json!(["a", {"kind": "nested"}]) + ); + assert_eq!( + movable.get_deep_value().to_json_value(), + json!(["second", "first"]) + ); + assert_eq!(text.get_deep_value().to_json_value(), json!("hello")); + assert_eq!( + tree.get_deep_value().to_json_value()[0]["meta"], + json!({"name": "root"}) + ); + + for handler in [ + Handler::Map(map.clone()), + Handler::List(list.clone()), + Handler::MovableList(movable.clone()), + Handler::Text(text.clone()), + Handler::Tree(tree.clone()), + ] { + assert!(!handler.is_attached()); + assert!(handler.attached_handler().is_none()); + assert!(handler.doc().is_none()); + assert!(handler.get_attached().is_none()); + assert_eq!(handler.id().container_type(), handler.kind()); + assert_eq!(handler.c_type(), handler.kind()); + assert_eq!(handler.to_handler().kind(), handler.kind()); + assert!(!handler.get_value().is_null()); + } + + Handler::Text(text).clear()?; + Handler::List(list).clear()?; + Handler::MovableList(movable).clear()?; + Handler::Tree(tree).clear()?; + Handler::Map(map).clear()?; + + Ok(()) +} + +#[test] +fn attached_handlers_report_parent_types_and_roundtrip_through_handler_enum( +) -> loro_internal::LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(66)?; + + let root = doc.get_map("root"); + root.insert("title", "draft")?; + let text = root.insert_container("text", TextHandler::new_detached())?; + text.insert_unicode(0, "hello")?; + + let list = root.insert_container("list", ListHandler::new_detached())?; + let map_under_list = list.insert_container(0, MapHandler::new_detached())?; + map_under_list.insert("kind", "list-child")?; + + let movable = root.insert_container("movable", MovableListHandler::new_detached())?; + let text_under_movable = movable.insert_container(0, TextHandler::new_detached())?; + text_under_movable.insert_unicode(0, "move-child")?; + + let tree = root.insert_container("tree", TreeHandler::new_detached())?; + let node = tree.create(TreeParentId::Root)?; + let meta = tree.get_meta(node)?; + meta.insert("kind", "tree-meta")?; + + doc.commit_then_renew(); + + assert!(matches!(text.parent(), Some(Handler::Map(_)))); + assert!(matches!(list.parent(), Some(Handler::Map(_)))); + assert!(matches!(map_under_list.parent(), Some(Handler::List(_)))); + assert!(matches!( + text_under_movable.parent(), + Some(Handler::MovableList(_)) + )); + assert!(matches!(tree.parent(), Some(Handler::Map(_)))); + assert!(matches!(meta.parent(), Some(Handler::Tree(_)))); + + assert!(::from_handler(text.to_handler()).is_some()); + assert!(::from_handler(map_under_list.to_handler()).is_some()); + assert!(::from_handler(list.to_handler()).is_some()); + assert!(::from_handler(movable.to_handler()).is_some()); + assert!(::from_handler(tree.to_handler()).is_some()); + assert!(::from_handler(root.to_handler()).is_none()); + + for handler in [ + root.to_handler(), + text.to_handler(), + list.to_handler(), + map_under_list.to_handler(), + movable.to_handler(), + text_under_movable.to_handler(), + tree.to_handler(), + meta.to_handler(), + ] { + assert!(handler.is_attached()); + assert!(handler.attached_handler().is_some()); + assert!(handler.doc().is_some()); + assert!(handler.get_attached().is_some()); + assert_eq!(handler.to_handler().id(), handler.id()); + assert_eq!(handler.c_type(), handler.kind()); + assert!(!handler.get_value().is_null()); + assert!(!handler.get_deep_value().is_null()); + } + + Handler::Text(text).clear()?; + Handler::List(list).clear()?; + Handler::MovableList(movable).clear()?; + Handler::Tree(tree).clear()?; + Handler::Map(root).clear()?; + assert_eq!(doc.get_deep_value().to_json_value(), json!({"root": {}})); + + assert!(doc + .get_handler(TreeID::new(123, 0).associated_meta_container()) + .is_none()); + + Ok(()) +} diff --git a/vendor/loro-internal/tests/contracts/jsonpath_value.rs b/vendor/loro-internal/tests/contracts/jsonpath_value.rs new file mode 100644 index 00000000..438dacc6 --- /dev/null +++ b/vendor/loro-internal/tests/contracts/jsonpath_value.rs @@ -0,0 +1,172 @@ +#![cfg(feature = "jsonpath")] + +use loro_internal::{ + handler::ValueOrHandler, jsonpath::jsonpath_impl::evaluate_jsonpath, loro_value, HandlerTrait, + LoroValue, ToJson, +}; +use pretty_assertions::assert_eq; +use serde_json::{json, Value}; +use std::error::Error; + +fn query_json(root: &LoroValue, path: &str) -> Result> { + Ok(Value::Array( + evaluate_jsonpath(root, path)? + .into_iter() + .map(|value| match value { + ValueOrHandler::Value(value) => value.to_json_value(), + ValueOrHandler::Handler(handler) => handler.get_deep_value().to_json_value(), + }) + .collect(), + )) +} + +fn root_value() -> LoroValue { + loro_value!({ + "store": { + "books": [ + { + "title": "1984", + "author": "George Orwell", + "price": 10, + "rating": 4.5, + "available": true, + "tags": ["classic", "dystopia"], + "meta": {"pages": 328, "edition": "paperback"}, + "note": null, + "zero": 0, + "empty": "", + "empty_list": [], + "empty_map": {} + }, + { + "title": "Animal Farm", + "author": "George Orwell", + "price": 6, + "rating": 4.0, + "available": true, + "tags": ["classic"], + "meta": {"pages": 112}, + "note": "satire", + "zero": 0, + "empty": "", + "empty_list": [], + "empty_map": {} + }, + { + "title": "Brave New World", + "author": "Aldous Huxley", + "price": 12, + "rating": 4.2, + "available": false, + "tags": [], + "meta": {}, + "note": null, + "zero": 0, + "empty": "", + "empty_list": [], + "empty_map": {} + } + ], + "featured_authors": ["George Orwell"], + "min_price": 9, + "empty": "", + "truth": true, + "nothing": null + }, + "numbers": [0, 1, 2, 3] + }) +} + +#[test] +fn jsonpath_evaluates_raw_loro_values_with_comparisons_and_truthiness() -> Result<(), Box> +{ + let root = root_value(); + + assert_eq!(query_json(&root, "$")?, json!([root.to_json_value()])); + assert_eq!( + query_json(&root, "$.store.books[-1].title")?, + json!(["Brave New World"]) + ); + assert_eq!(query_json(&root, "$.store.books[-99]")?, json!([])); + + assert_eq!( + query_json(&root, "$.store.books[?(@.price != 10)].title")?, + json!(["Animal Farm", "Brave New World"]) + ); + assert_eq!( + query_json( + &root, + "$.store.books[?(@.price <= $.store.min_price)].title" + )?, + json!(["Animal Farm"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(@.price > 10)].title")?, + json!(["Brave New World"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(@.rating == 4.5)].title")?, + json!(["1984"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(@.rating >= 4.2)].title")?, + json!(["1984", "Brave New World"]) + ); + assert_eq!( + query_json( + &root, + "$.store.books[?(@.title >= 'Brave New World')].title" + )?, + json!(["Brave New World"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(@.title < 'Brave New World')].title")?, + json!(["1984", "Animal Farm"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(@.available != false)].title")?, + json!(["1984", "Animal Farm"]) + ); + assert_eq!( + query_json( + &root, + "$.store.books[?(@.author in $.store.featured_authors)].title" + )?, + json!(["1984", "Animal Farm"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(@.author in 'George Orwell')].title")?, + json!([]) + ); + + assert_eq!( + query_json(&root, "$.store.books[?(count(@.title) == 1)].title")?, + json!(["1984", "Animal Farm", "Brave New World"]) + ); + assert!(evaluate_jsonpath(&root, "$.store.books[?(count(1) == 0)].title").is_err()); + assert_eq!( + query_json(&root, "$.store.books[?(length(value(@.tags)) > 0)].title")?, + json!(["1984", "Animal Farm"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(length(value(@.meta)) >= 2)].title")?, + json!(["1984"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(length(value(@.price)) == 0)].title")?, + json!(["1984", "Animal Farm", "Brave New World"]) + ); + assert_eq!( + query_json(&root, "$.store.books[?(value(@.tags[*]) == null)].title")?, + json!(["1984", "Brave New World"]) + ); + + assert!(evaluate_jsonpath(&root, "$.store.books[?(value(@.title))].title").is_err()); + assert_eq!( + query_json(&root, "$.store.books[?(@.missing)].title")?, + json!([]) + ); + assert_eq!(query_json(&root, "$.numbers[?(@ >= 2)]")?, json!([2, 3])); + + Ok(()) +} diff --git a/vendor/loro-internal/tests/contracts/public_doc.rs b/vendor/loro-internal/tests/contracts/public_doc.rs new file mode 100644 index 00000000..856a2c2e --- /dev/null +++ b/vendor/loro-internal/tests/contracts/public_doc.rs @@ -0,0 +1,243 @@ +use std::sync::{Arc, Mutex}; + +use loro_common::{ContainerID, ContainerType, LoroResult, TreeID, ID}; +use loro_internal::{ + cursor::Side, + event::{path_to_str, str_to_path, EventTriggerKind, Index}, + handler::{Handler, ListHandler, MapHandler, MovableListHandler, TextHandler, ValueOrHandler}, + loro::ExportMode, + version::VersionVector, + HandlerTrait, LoroDoc, ToJson, TreeParentId, +}; +use pretty_assertions::assert_eq; +use serde_json::{json, Value}; + +fn deep_json(doc: &LoroDoc) -> Value { + doc.get_deep_value().to_json_value() +} + +fn value_or_handler_deep_json(value: ValueOrHandler) -> Value { + match value { + ValueOrHandler::Value(value) => value.to_json_value(), + ValueOrHandler::Handler(handler) => handler.get_deep_value().to_json_value(), + } +} + +#[test] +fn internal_doc_introspection_uses_the_same_container_path_contracts() -> LoroResult<()> { + let pristine = LoroDoc::new(); + assert!(pristine.can_reset_with_snapshot()); + + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(901)?; + + let root = doc.get_map("root"); + root.insert("title", "draft")?; + + let body = root.insert_container("body", TextHandler::new_detached())?; + body.insert_unicode(0, "hello")?; + + let list = root.insert_container("items", ListHandler::new_detached())?; + list.insert(0, "one")?; + let nested_map = list.insert_container(1, MapHandler::new_detached())?; + nested_map.insert("kind", "nested")?; + + let movable = root.insert_container("order", MovableListHandler::new_detached())?; + movable.insert(0, "first")?; + movable.insert(1, "second")?; + movable.mov(1, 0)?; + + let tree = doc.get_tree("tree"); + tree.enable_fractional_index(0); + let root_node = tree.create(TreeParentId::Root)?; + tree.get_meta(root_node)?.insert("label", "root")?; + + let uncommitted = doc + .get_uncommitted_ops_as_json() + .expect("pending local ops should be visible before commit"); + assert!(!uncommitted.changes.is_empty()); + + doc.commit_then_renew(); + assert!(doc.get_uncommitted_ops_as_json().is_none()); + assert!(!doc.can_reset_with_snapshot()); + + let value_with_ids = doc.app_state().lock().get_deep_value_with_id(); + let value_with_ids_json = value_with_ids.to_json_value(); + assert_eq!( + value_with_ids_json["root"]["value"]["title"], + json!("draft") + ); + assert!(value_with_ids_json["root"]["cid"] + .as_str() + .expect("cid should be a string") + .contains("cid:")); + + let flat = doc.app_state().lock().get_all_container_value_flat(); + let flat_json = flat.to_json_value(); + assert_eq!(flat_json[body.id().to_string()], json!("hello")); + assert_eq!( + flat_json[nested_map.id().to_string()], + json!({"kind": "nested"}) + ); + + let body_path = doc + .get_path_to_container(&body.id()) + .expect("body should have a root path"); + let indexes = body_path + .iter() + .map(|(_, index)| index.clone()) + .collect::>(); + assert_eq!(path_to_str(&indexes), "root/body"); + assert_eq!(str_to_path("root/body"), Some(indexes.clone())); + assert_eq!( + value_or_handler_deep_json(doc.get_by_path(&indexes).expect("body path should resolve")), + json!("hello") + ); + assert!(doc + .get_path_to_container(&ContainerID::new_normal( + ID::new(777, 0), + ContainerType::Text + )) + .is_none()); + + assert_eq!(EventTriggerKind::Local.to_string(), "local"); + assert_eq!(EventTriggerKind::Import.to_string(), "import"); + assert_eq!(EventTriggerKind::Checkout.to_string(), "checkout"); + assert!(EventTriggerKind::Local.is_local()); + assert!(EventTriggerKind::Import.is_import()); + assert!(EventTriggerKind::Checkout.is_checkout()); + + assert_eq!(Index::try_from("").unwrap(), Index::Key("".into())); + assert_eq!(Index::try_from("12").unwrap(), Index::Seq(12)); + assert_eq!(Index::try_from("12x").unwrap(), Index::Key("12x".into())); + assert_eq!(Index::Node(TreeID::new(5, 7)).to_string(), "5@7"); + assert_eq!( + format!("{:?}", Index::Key("plain".into())), + "Index::Key(\"plain\")" + ); + assert_eq!( + str_to_path("root/1/5@7"), + Some(vec![ + Index::Key("root".into()), + Index::Seq(1), + Index::Node(TreeID::new(7, 5)), + ]) + ); + + let fork = doc.fork(); + assert_eq!(deep_json(&fork), deep_json(&doc)); + doc.detach(); + let detached_fork = doc.fork(); + assert_eq!(deep_json(&detached_fork), deep_json(&doc)); + doc.attach(); + + let text_cursor = body + .get_cursor(0, Side::Middle) + .expect("text cursor should be created"); + assert_eq!( + doc.query_pos(&text_cursor) + .expect("cursor should resolve") + .current + .pos, + 0 + ); + + Ok(()) +} + +#[test] +fn import_diff_apply_and_checkout_events_follow_doc_contracts() -> LoroResult<()> { + let source = LoroDoc::new_auto_commit(); + source.set_peer_id(902)?; + + let source_events = Arc::new(Mutex::new(Vec::<(EventTriggerKind, String, usize)>::new())); + let source_events_ref = Arc::clone(&source_events); + let _source_sub = source.subscribe_root(Arc::new(move |event| { + source_events_ref.lock().unwrap().push(( + event.event_meta.by, + event.event_meta.origin.to_string(), + event.events.len(), + )); + })); + + let text = source.get_text("text"); + text.insert_unicode(0, "a")?; + source.set_next_commit_origin("seed"); + source.commit_then_renew(); + let v1 = source.state_frontiers(); + + text.insert_unicode(1, "b")?; + let list = source.get_list("list"); + list.insert(0, "item")?; + let nested = list.insert_container(1, TextHandler::new_detached())?; + nested.insert_unicode(0, "nested")?; + source.set_next_commit_origin("second"); + source.commit_then_renew(); + let v2 = source.state_frontiers(); + + source.checkout(&v1)?; + assert_eq!(deep_json(&source), json!({"text": "a", "list": []})); + source.checkout_to_latest(); + assert_eq!( + deep_json(&source), + json!({"text": "ab", "list": ["item", "nested"]}) + ); + + let observed = source_events.lock().unwrap().clone(); + assert!(observed + .iter() + .any(|(kind, origin, _)| kind.is_local() && origin == "seed")); + assert!(observed + .iter() + .any(|(kind, origin, _)| kind.is_local() && origin == "second")); + assert!(observed.iter().any(|(kind, _, _)| kind.is_checkout())); + + let at_v1 = source.fork_at(&v1)?; + assert_eq!(deep_json(&at_v1), json!({"text": "a", "list": []})); + let diff = source.diff(&v1, &v2)?; + at_v1.apply_diff(diff)?; + assert_eq!( + deep_json(&at_v1), + json!({"text": "ab", "list": ["item", "nested"]}) + ); + + let reverse = source.diff(&v2, &v1)?; + at_v1.apply_diff(reverse)?; + assert_eq!(deep_json(&at_v1), json!({"text": "a", "list": []})); + + let snapshot = source.export(ExportMode::Snapshot)?; + let imported = LoroDoc::new_auto_commit(); + let import_events = Arc::new(Mutex::new(Vec::<(EventTriggerKind, String)>::new())); + let import_events_ref = Arc::clone(&import_events); + let _import_sub = imported.subscribe_root(Arc::new(move |event| { + import_events_ref + .lock() + .unwrap() + .push((event.event_meta.by, event.event_meta.origin.to_string())); + })); + imported.import_with(&snapshot, "sync".into())?; + assert_eq!(deep_json(&imported), deep_json(&source)); + assert_eq!( + import_events.lock().unwrap().as_slice(), + &[(EventTriggerKind::Import, "sync".to_string())] + ); + + let batch_target = LoroDoc::new_auto_commit(); + let first_vv = source + .frontiers_to_vv(&v1) + .expect("v1 should be in the source oplog"); + let first_updates = source.export(ExportMode::updates(&VersionVector::default()))?; + let second_updates = source.export(ExportMode::updates(&first_vv))?; + let status = batch_target.import_batch(&[second_updates, first_updates])?; + assert!(status.pending.is_none()); + assert_eq!(deep_json(&batch_target), deep_json(&source)); + + let handler = imported + .get_handler(text.id()) + .expect("text handler should exist"); + assert!(matches!(handler, Handler::Text(_))); + assert!(matches!(handler.to_handler(), Handler::Text(_))); + assert_eq!(handler.kind(), ContainerType::Text); + + Ok(()) +} diff --git a/vendor/loro-internal/tests/contracts/text_richtext_behavior.rs b/vendor/loro-internal/tests/contracts/text_richtext_behavior.rs new file mode 100644 index 00000000..b1415d43 --- /dev/null +++ b/vendor/loro-internal/tests/contracts/text_richtext_behavior.rs @@ -0,0 +1,213 @@ +use loro_common::{LoroError, LoroValue}; +use loro_internal::{ + cursor::PosType, + handler::{TextDelta, TextHandler, UpdateOptions}, + HandlerTrait, LoroDoc, ToJson, +}; +use pretty_assertions::assert_eq; +use rustc_hash::FxHashMap; +use serde_json::json; + +fn attrs( + pairs: impl IntoIterator, +) -> FxHashMap { + pairs + .into_iter() + .map(|(key, value)| (key.to_string(), value)) + .collect() +} + +#[test] +fn attached_text_versions_styles_and_coordinate_errors_follow_contract( +) -> loro_internal::LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(78)?; + let text = doc.get_text("text"); + + let initial_version = text.version_id().expect("attached text has a version id"); + text.insert_unicode(0, "A😀文")?; + assert_ne!(text.version_id(), Some(initial_version)); + assert_eq!(text.to_string(), "A😀文"); + + assert!(matches!( + text.insert_utf16(2, "x"), + Err(LoroError::UTF16InUnicodeCodePoint { pos: 2 }) + )); + assert!(matches!( + text.insert_utf8(2, "x"), + Err(LoroError::UTF8InUnicodeCodePoint { pos: 2 }) + )); + assert_eq!(text.to_string(), "A😀文"); + assert_eq!( + text.convert_pos(2, PosType::Unicode, PosType::Utf16), + Some(3) + ); + assert_eq!( + text.convert_pos(5, PosType::Bytes, PosType::Unicode), + Some(2) + ); + assert_eq!(text.convert_pos(1, PosType::Unicode, PosType::Entity), None); + assert_eq!( + text.convert_pos(100, PosType::Unicode, PosType::Bytes), + None + ); + + let before_missing_unmark = text.version_id(); + text.unmark(0, 1, "bold", PosType::Unicode)?; + assert_eq!(text.version_id(), before_missing_unmark); + + text.mark(1, 3, "bold", true.into(), PosType::Unicode)?; + let after_mark = text.version_id(); + text.mark(1, 3, "bold", true.into(), PosType::Unicode)?; + assert_eq!(text.version_id(), after_mark); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([ + { "insert": "A" }, + { "insert": "😀文", "attributes": { "bold": true } } + ]) + ); + assert_eq!( + text.slice_delta(0, text.len_unicode(), PosType::Unicode)?, + vec![ + TextDelta::Insert { + insert: "A".to_string(), + attributes: None, + }, + TextDelta::Insert { + insert: "😀文".to_string(), + attributes: Some(attrs([("bold", true.into())])), + }, + ] + ); + + text.unmark(2, 3, "bold", PosType::Unicode)?; + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([ + { "insert": "A" }, + { "insert": "😀", "attributes": { "bold": true } }, + { "insert": "文" } + ]) + ); + + text.check(); + text.diagnose(); + Ok(()) +} + +#[test] +fn apply_delta_and_update_keep_text_and_attribute_contracts() -> loro_internal::LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(79)?; + let text = doc.get_text("patch"); + + text.apply_delta(&[TextDelta::Insert { + insert: "hello".to_string(), + attributes: Some(attrs([("bold", true.into())])), + }])?; + assert_eq!( + text.get_delta(), + vec![TextDelta::Insert { + insert: "hello".to_string(), + attributes: Some(attrs([("bold", true.into())])), + }] + ); + + text.apply_delta(&[ + TextDelta::Retain { + retain: 2, + attributes: Some(attrs([("italic", true.into())])), + }, + TextDelta::Delete { delete: 2 }, + TextDelta::Insert { + insert: "Y".to_string(), + attributes: None, + }, + ])?; + assert_eq!(text.to_string(), "heYo"); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([ + { "insert": "he", "attributes": { "bold": true, "italic": true } }, + { "insert": "Y" }, + { "insert": "o", "attributes": { "bold": true } } + ]) + ); + + text.update("plain\nnew\ntext", UpdateOptions::default()) + .expect("character update should finish without timeout"); + assert_eq!(text.to_string(), "plain\nnew\ntext"); + text.update_by_line("plain\nline\ntext\n", UpdateOptions::default()) + .expect("line update should finish without timeout"); + assert_eq!(text.to_string(), "plain\nline\ntext\n"); + + Ok(()) +} + +#[test] +fn detached_text_richtext_state_survives_attach() -> loro_internal::LoroResult<()> { + let detached = TextHandler::new_detached(); + assert_eq!(detached.version_id(), None); + assert!(detached.is_empty()); + assert!(detached.apply_delta(&[]).is_err()); + + detached.insert_unicode(0, "abc文")?; + detached.mark(1, 3, "bold", true.into(), PosType::Unicode)?; + detached.mark(1, 3, "bold", true.into(), PosType::Unicode)?; + detached.unmark(0, 1, "bold", PosType::Unicode)?; + assert!(matches!( + detached.char_at(99, PosType::Unicode), + Err(LoroError::OutOfBound { .. }) + )); + assert!(matches!( + detached.slice(3, 2, PosType::Unicode), + Err(LoroError::EndIndexLessThanStartIndex { .. }) + )); + assert!(detached + .mark(2, 2, "bold", true.into(), PosType::Unicode) + .is_err()); + assert_eq!(detached.slice(0, 0, PosType::Unicode)?, ""); + + let expected_delta = vec![ + TextDelta::Insert { + insert: "a".to_string(), + attributes: None, + }, + TextDelta::Insert { + insert: "bc".to_string(), + attributes: Some(attrs([("bold", true.into())])), + }, + TextDelta::Insert { + insert: "文".to_string(), + attributes: None, + }, + ]; + assert_eq!(detached.get_delta(), expected_delta); + + let mut first_chunk = String::new(); + detached.iter(|chunk| { + first_chunk.push_str(chunk); + false + }); + assert_eq!(first_chunk, "a"); + detached.check(); + detached.diagnose(); + + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(80)?; + let root = doc.get_map("root"); + let attached = root.insert_container("body", detached.clone())?; + let attached_from_detached = detached + .get_attached() + .expect("detached text should resolve to attached handler after insertion"); + assert_eq!(attached.id(), attached_from_detached.id()); + assert!(attached.version_id().is_some()); + assert_eq!(attached.get_delta(), expected_delta); + assert_eq!( + root.get_deep_value().to_json_value(), + json!({ "body": "abc文" }) + ); + + Ok(()) +} diff --git a/vendor/loro-internal/tests/contracts/text_richtext_handler.rs b/vendor/loro-internal/tests/contracts/text_richtext_handler.rs new file mode 100644 index 00000000..4e7be2b8 --- /dev/null +++ b/vendor/loro-internal/tests/contracts/text_richtext_handler.rs @@ -0,0 +1,266 @@ +use loro_internal::{ + cursor::{PosType, Side}, + handler::Handler, + handler::TextDelta, + ContainerType, HandlerTrait, LoroDoc, LoroResult, TextHandler, ToJson, +}; +use pretty_assertions::assert_eq; +use serde_json::json; + +fn richtext_json(text: &TextHandler) -> serde_json::Value { + text.get_richtext_value().to_json_value() +} + +#[test] +fn detached_text_handler_contract_covers_local_branch() -> LoroResult<()> { + let text = TextHandler::new_detached(); + + assert!(!text.is_attached()); + assert!(text.doc().is_none()); + assert!(text.version_id().is_none()); + assert!(text.get_attached().is_none()); + assert!(text.is_empty()); + assert_eq!(format!("{:?}", text), "TextHandler(Unattached)"); + + text.insert_utf8(0, "a😀文b")?; + text.mark(1, 3, "bold", true.into(), PosType::Unicode)?; + + assert_eq!(text.len_utf8(), 9); + assert_eq!(text.len_unicode(), 4); + assert_eq!(text.len_utf16(), 5); + let event_len = if cfg!(feature = "wasm") { 5 } else { 4 }; + assert_eq!(text.len_event(), event_len); + assert_eq!(text.char_at(1, PosType::Unicode)?, '😀'); + assert_eq!(text.slice(1, 3, PosType::Unicode)?, "😀文"); + assert_eq!(text.slice_utf16(1, 4)?, "😀文"); + assert_eq!( + text.convert_pos(2, PosType::Unicode, PosType::Bytes), + Some(5) + ); + assert_eq!( + text.convert_pos(5, PosType::Bytes, PosType::Unicode), + Some(2) + ); + assert_eq!( + text.convert_pos(1, PosType::Unicode, PosType::Utf16), + Some(1) + ); + assert!(text.char_at(99, PosType::Unicode).is_err()); + assert!(text.slice(3, 2, PosType::Unicode).is_err()); + + let mut seen = Vec::new(); + text.iter(|chunk| { + seen.push(chunk.to_owned()); + false + }); + assert_eq!(seen, vec!["a".to_string()]); + + assert_eq!( + richtext_json(&text), + json!([ + {"insert": "a"}, + {"insert": "😀文", "attributes": {"bold": true}}, + {"insert": "b"} + ]) + ); + assert_eq!( + text.slice_delta(0, text.len_unicode(), PosType::Unicode)?, + vec![ + TextDelta::Insert { + insert: "a".to_string(), + attributes: None, + }, + TextDelta::Insert { + insert: "😀文".to_string(), + attributes: Some([("bold".to_string(), true.into())].into_iter().collect()), + }, + TextDelta::Insert { + insert: "b".to_string(), + attributes: None, + }, + ] + ); + + let handler = text.to_handler(); + assert_eq!(handler.c_type(), ContainerType::Text); + assert!(!handler.is_attached()); + assert!(handler.attached_handler().is_none()); + assert!(handler.doc().is_none()); + assert_eq!(handler.get_value().to_json_value(), json!("a😀文b")); + assert_eq!(handler.get_deep_value().to_json_value(), json!("a😀文b")); + assert!(::from_handler(handler.clone()).is_some()); + assert!(::from_handler(handler).is_some()); + + text.diagnose(); + + let plain = TextHandler::new_detached(); + plain.insert_utf8(0, "clear me")?; + plain.clear()?; + assert_eq!(plain.len_unicode(), 0); + assert_eq!(plain.len_utf8(), 0); + assert_eq!(richtext_json(&plain), json!([])); + + Ok(()) +} + +#[test] +fn attached_text_handler_contract_covers_document_branch_and_handler_enum() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + let root = doc.get_map("root"); + + let detached = TextHandler::new_detached(); + detached.insert_utf8(0, "abcd")?; + detached.mark(1, 3, "bold", true.into(), PosType::Unicode)?; + + let attached = root.insert_container("body", detached.clone())?; + + assert!(!detached.is_attached()); + assert!(detached.doc().is_none()); + assert!(detached.get_attached().is_some()); + assert!(attached.is_attached()); + assert!(attached.doc().is_some()); + assert!(attached.version_id().is_some()); + assert_eq!(attached.len_utf8(), 4); + assert_eq!(attached.len_unicode(), 4); + assert_eq!(attached.len_utf16(), 4); + assert_eq!(attached.char_at(1, PosType::Unicode)?, 'b'); + assert_eq!(attached.slice(1, 3, PosType::Unicode)?, "bc"); + assert_eq!(attached.slice_utf16(1, 3)?, "bc"); + assert_eq!( + attached.convert_pos(2, PosType::Unicode, PosType::Bytes), + Some(2) + ); + assert_eq!( + attached.convert_pos(2, PosType::Bytes, PosType::Unicode), + Some(2) + ); + + let mut seen = Vec::new(); + attached.iter(|chunk| { + seen.push(chunk.to_owned()); + false + }); + assert_eq!(seen, vec!["a".to_string()]); + + assert_eq!( + attached.slice_delta(0, attached.len_unicode(), PosType::Unicode)?, + vec![ + TextDelta::Insert { + insert: "a".to_string(), + attributes: None, + }, + TextDelta::Insert { + insert: "bc".to_string(), + attributes: Some([("bold".to_string(), true.into())].into_iter().collect()), + }, + TextDelta::Insert { + insert: "d".to_string(), + attributes: None, + }, + ] + ); + assert_eq!( + attached.get_richtext_value().to_json_value(), + json!([ + {"insert": "a"}, + {"insert": "bc", "attributes": {"bold": true}}, + {"insert": "d"} + ]) + ); + assert_eq!( + root.get_deep_value().to_json_value(), + json!({"body": "abcd"}) + ); + + let handler = attached.to_handler(); + assert_eq!(handler.kind(), ContainerType::Text); + assert!(handler.is_attached()); + assert!(handler.attached_handler().is_some()); + assert!(handler.doc().is_some()); + assert_eq!(handler.get_value().to_json_value(), json!("abcd")); + assert_eq!(handler.get_deep_value().to_json_value(), json!("abcd")); + assert!(::from_handler(handler.clone()).is_some()); + assert!(::from_handler(handler.clone()).is_some()); + + let before = attached.version_id(); + attached.insert_utf8(attached.len_utf8(), "!")?; + attached.delete_utf16(1, 2)?; + assert_ne!(attached.version_id(), before); + assert_eq!( + attached.slice(0, attached.len_unicode(), PosType::Unicode)?, + "ad!" + ); + assert_eq!( + root.get_deep_value().to_json_value(), + json!({"body": "ad!"}) + ); + + handler.clear()?; + assert_eq!(attached.len_unicode(), 0); + assert_eq!(attached.len_utf8(), 0); + assert!(attached.version_id().is_some()); + let cursor = attached + .get_cursor(0, Side::Middle) + .expect("empty attached text"); + assert_eq!(cursor.id, None); + assert_eq!(cursor.side, Side::Left); + assert_eq!(attached.char_at(0, PosType::Unicode).is_err(), true); + assert_eq!(attached.get_richtext_value().to_json_value(), json!([])); + + attached.diagnose(); + assert!(attached.char_at(0, PosType::Unicode).is_err()); + assert!(attached.slice(1, 0, PosType::Unicode).is_err()); + + Ok(()) +} + +#[test] +fn text_handler_position_edge_cases_cover_unsupported_and_empty_ranges() -> LoroResult<()> { + let text = TextHandler::new_detached(); + text.insert_utf8(0, "ab")?; + + assert_eq!( + text.get_delta(), + vec![TextDelta::Insert { + insert: "ab".to_string(), + attributes: None, + }] + ); + assert_eq!( + text.convert_pos(0, PosType::Unicode, PosType::Unicode), + Some(0) + ); + assert_eq!(text.convert_pos(0, PosType::Entity, PosType::Unicode), None); + assert_eq!(text.convert_pos(0, PosType::Unicode, PosType::Entity), None); + assert_eq!(text.convert_pos(10, PosType::Unicode, PosType::Bytes), None); + assert_eq!(text.slice(1, 1, PosType::Unicode)?, ""); + assert_eq!(text.slice_utf16(1, 1)?, ""); + + let doc = LoroDoc::new_auto_commit(); + let root = doc.get_map("root"); + let attached = root.insert_container("body", text.clone())?; + + assert!(attached.is_attached()); + assert_eq!( + attached.get_delta(), + vec![TextDelta::Insert { + insert: "ab".to_string(), + attributes: None, + }] + ); + assert_eq!( + attached.convert_pos(0, PosType::Entity, PosType::Unicode), + None + ); + assert_eq!( + attached.convert_pos(attached.len_unicode(), PosType::Unicode, PosType::Unicode), + Some(attached.len_unicode()) + ); + assert_eq!( + attached.convert_pos(attached.len_unicode() + 1, PosType::Unicode, PosType::Bytes), + None + ); + assert_eq!(attached.slice(0, 0, PosType::Unicode)?, ""); + + Ok(()) +} diff --git a/vendor/loro-internal/tests/contracts/tree_handler.rs b/vendor/loro-internal/tests/contracts/tree_handler.rs new file mode 100644 index 00000000..5c363145 --- /dev/null +++ b/vendor/loro-internal/tests/contracts/tree_handler.rs @@ -0,0 +1,119 @@ +use loro_common::{LoroResult, PeerID}; +use loro_internal::{handler::TreeHandler, LoroDoc, ToJson, TreeID, TreeParentId}; +use pretty_assertions::assert_eq; +use serde_json::{json, Value}; + +fn deep_json(doc: &LoroDoc) -> Value { + doc.get_deep_value().to_json_value() +} + +#[test] +fn detached_tree_handler_queries_and_clear_follow_contracts() -> LoroResult<()> { + let tree = TreeHandler::new_detached(); + assert_eq!(format!("{tree:?}"), "TreeHandler Detached"); + assert!(tree.is_fractional_index_enabled()); + tree.enable_fractional_index(2); + tree.disable_fractional_index(); + assert!(!tree.is_deleted()); + assert!(tree.is_empty()); + assert_eq!(tree.__internal__next_tree_id(), TreeID::new(PeerID::MAX, 0)); + + let root = tree.create(TreeParentId::Root)?; + let first = tree.create_at(root.into(), 0)?; + let second = tree.create_at(root.into(), 1)?; + let third = tree.create_at(root.into(), 2)?; + + assert_eq!(tree.__internal__next_tree_id(), TreeID::new(PeerID::MAX, 4)); + assert_eq!(tree.get_child_at(&TreeParentId::Root, 0), Some(root)); + assert_eq!(tree.get_child_at(&root.into(), 0), Some(first)); + assert_eq!(tree.get_child_at(&root.into(), 1), Some(second)); + assert_eq!(tree.get_child_at(&root.into(), 2), Some(third)); + assert_eq!(tree.get_child_at(&root.into(), 3), None); + assert_eq!(tree.get_index_by_tree_id(&second), Some(1)); + assert!(tree.get_position_by_tree_id(&second).is_some()); + assert!(tree + .get_position_by_tree_id(&TreeID::new(PeerID::MAX, 99)) + .is_none()); + assert!(tree.is_parent(&second, &root.into())); + assert_eq!(tree.get_nodes_under(TreeParentId::Root).len(), 4); + assert_eq!(tree.get_nodes_under(root.into()).len(), 3); + assert_eq!(tree.nodes().len(), 4); + + tree.mov_before(third, first)?; + assert_eq!( + tree.children(&root.into()), + Some(vec![third, first, second]) + ); + tree.mov_after(third, second)?; + assert_eq!( + tree.children(&root.into()), + Some(vec![first, second, third]) + ); + tree.move_to(third, TreeParentId::Root, 0)?; + assert_eq!(tree.children(&TreeParentId::Root), Some(vec![third, root])); + + tree.delete(third)?; + assert!(!tree.contains(third)); + assert_eq!(tree.get_node_parent(&third), None); + + tree.clear()?; + assert!(tree.is_empty()); + assert_eq!(tree.roots(), Vec::::new()); + assert_eq!(tree.nodes(), Vec::::new()); + + Ok(()) +} + +#[test] +fn attached_tree_hierarchy_positions_and_diff_apply_follow_contracts() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(77)?; + let tree = doc.get_tree("outline"); + tree.enable_fractional_index(2); + + let root = tree.create(TreeParentId::Root)?; + let first = tree.create_at(root.into(), 0)?; + let second = tree.create_at(root.into(), 1)?; + tree.get_meta(root)?.insert("name", "root")?; + tree.get_meta(first)?.insert("name", "first")?; + tree.get_meta(second)?.insert("name", "second")?; + + assert!(format!("{tree:?}").contains("TreeHandler")); + assert_eq!(tree.get_child_at(&TreeParentId::Root, 0), Some(root)); + assert_eq!(tree.get_child_at(&root.into(), 0), Some(first)); + assert_eq!(tree.get_child_at(&root.into(), 1), Some(second)); + assert_eq!(tree.get_index_by_tree_id(&second), Some(1)); + assert!(tree.get_position_by_tree_id(&second).is_some()); + assert!(tree.get_last_move_id(&root).is_some()); + + let hierarchy = tree.get_all_hierarchy_nodes_under(TreeParentId::Root); + assert_eq!(hierarchy.len(), 1); + assert_eq!(hierarchy[0].id, root); + assert_eq!(hierarchy[0].children.len(), 2); + assert_eq!(hierarchy[0].children[0].id, first); + assert_eq!(hierarchy[0].children[1].id, second); + + doc.commit_then_renew(); + let v1 = doc.state_frontiers(); + let before = deep_json(&doc); + + tree.move_to(second, TreeParentId::Root, 0)?; + tree.delete(first)?; + tree.get_meta(second)?.insert("status", "promoted")?; + doc.commit_then_renew(); + let v2 = doc.state_frontiers(); + let after = deep_json(&doc); + assert_ne!(before, after); + + let fork = doc.fork_at(&v1)?; + assert_eq!(deep_json(&fork), before); + fork.apply_diff(doc.diff(&v1, &v2)?)?; + assert_eq!(deep_json(&fork), after); + + tree.clear()?; + doc.commit_then_renew(); + assert!(tree.is_empty()); + assert_eq!(deep_json(&doc), json!({"outline": []})); + + Ok(()) +} diff --git a/vendor/loro-internal/tests/ephemeral_store.rs b/vendor/loro-internal/tests/ephemeral_store.rs new file mode 100644 index 00000000..f991ad85 --- /dev/null +++ b/vendor/loro-internal/tests/ephemeral_store.rs @@ -0,0 +1,94 @@ +use loro_internal::awareness::EphemeralStore; +use loro_internal::LoroValue; +use serde::Serialize; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[derive(Serialize)] +struct WireState { + key: String, + value: Option, + timestamp: i64, +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64 +} + +#[test] +fn import_skips_entries_past_timeout() { + let store = EphemeralStore::new(100); + let stale_timestamp = now_ms() - 500; + + let payload = postcard::to_allocvec(&vec![WireState { + key: "stale".into(), + value: Some(LoroValue::from(1)), + timestamp: stale_timestamp, + }]) + .unwrap(); + + store.apply(&payload).unwrap(); + + assert!(store.get("stale").is_none()); + assert!(store.get_all_states().is_empty()); +} + +#[test] +fn import_preserves_remote_timestamp_for_timeout() { + let timeout_ms = 100; + let store = EphemeralStore::new(timeout_ms); + let remote_timestamp = now_ms() - (timeout_ms - 20); + + let payload = postcard::to_allocvec(&vec![WireState { + key: "cursor".into(), + value: Some(LoroValue::from("v")), + timestamp: remote_timestamp, + }]) + .unwrap(); + + store.apply(&payload).unwrap(); + assert_eq!(store.get("cursor"), Some(LoroValue::from("v"))); + + std::thread::sleep(Duration::from_millis(40)); + store.remove_outdated(); + + assert!(store.get("cursor").is_none()); + assert!(store.get_all_states().is_empty()); +} + +#[test] +fn import_rejects_too_deep_loro_value() { + let mut payload = vec![0x01u8, 0x00, 0x01]; + for _ in 0..600 { + payload.push(0x05); + payload.push(0x01); + } + payload.extend_from_slice(&[0x00, 0x00]); + + let store = EphemeralStore::new(30_000); + let err = store.apply(&payload).unwrap_err(); + + assert!(err.contains("Failed to decode data")); +} + +#[test] +fn import_accepts_loro_value_at_depth_511() { + let mut value = LoroValue::Null; + for _ in 0..511 { + value = LoroValue::List(vec![value].into()); + } + + let payload = postcard::to_allocvec(&vec![WireState { + key: "deep".into(), + value: Some(value), + timestamp: now_ms(), + }]) + .unwrap(); + + let store = EphemeralStore::new(30_000); + + assert!(store.apply(&payload).is_ok()); + assert!(store.get("deep").is_some()); +} diff --git a/vendor/loro-internal/tests/fugue.rs b/vendor/loro-internal/tests/fugue.rs new file mode 100644 index 00000000..661a7449 --- /dev/null +++ b/vendor/loro-internal/tests/fugue.rs @@ -0,0 +1,90 @@ +use loro_common::LoroResult; +use loro_internal::{LoroDoc, ToJson}; +use serde_json::json; + +#[test] +fn test_forward_interleaving() -> LoroResult<()> { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(0)?; + a.get_text("text").insert_unicode(0, "Hello")?; + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(1)?; + b.get_text("text").insert_unicode(0, " World!")?; + a.merge(&b)?; + assert_eq!( + a.get_deep_value().to_json_value(), + json!({"text": "Hello World!"}) + ); + Ok(()) +} + +#[test] +fn test_backward_interleaving() -> LoroResult<()> { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(0)?; + a.get_text("text").insert_unicode(0, "o")?; + a.get_text("text").insert_unicode(0, "l")?; + a.get_text("text").insert_unicode(0, "l")?; + a.get_text("text").insert_unicode(0, "e")?; + a.get_text("text").insert_unicode(0, "H")?; + dbg!(a.get_deep_value()); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(1)?; + b.get_text("text").insert_unicode(0, "!")?; + b.get_text("text").insert_unicode(0, "d")?; + b.get_text("text").insert_unicode(0, "l")?; + b.get_text("text").insert_unicode(0, "r")?; + b.get_text("text").insert_unicode(0, "o")?; + b.get_text("text").insert_unicode(0, "W")?; + b.get_text("text").insert_unicode(0, " ")?; + dbg!(b.get_deep_value()); + a.merge(&b)?; + assert_eq!( + a.get_deep_value().to_json_value(), + json!({"text": "Hello World!"}) + ); + Ok(()) +} + +#[test] +fn test_forward_backward() -> LoroResult<()> { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(0)?; + a.get_text("text").insert_unicode(0, "ll")?; + a.get_text("text").insert_unicode(0, "He")?; + a.get_text("text").insert_unicode(4, "o")?; + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(1)?; + b.get_text("text").insert_unicode(0, " !")?; + b.get_text("text").insert_unicode(1, "W")?; + b.get_text("text").insert_unicode(2, "d")?; + b.get_text("text").insert_unicode(2, "l")?; + b.get_text("text").insert_unicode(2, "r")?; + b.get_text("text").insert_unicode(2, "o")?; + a.merge(&b)?; + assert_eq!( + a.get_deep_value().to_json_value(), + json!({"text": "Hello World!"}) + ); + + Ok(()) +} + +#[test] +fn test_yjs_interleave() -> LoroResult<()> { + // As stated in the Fugue paper, Yjs has a interleaving anomaly in the following case: + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(0)?; + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(1)?; + let c = LoroDoc::new_auto_commit(); + c.set_peer_id(2)?; + c.get_text("text").insert_unicode(0, "2")?; + a.merge(&c)?; + a.get_text("text").insert_unicode(0, "1")?; + // b should not be between a and c + b.get_text("text").insert_unicode(0, "b")?; + a.merge(&b)?; + assert_eq!(a.get_deep_value().to_json_value(), json!({"text": "b12"})); + Ok(()) +} diff --git a/vendor/loro-internal/tests/mergeable_cid_encoding.rs b/vendor/loro-internal/tests/mergeable_cid_encoding.rs new file mode 100644 index 00000000..4dd4d713 --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_cid_encoding.rs @@ -0,0 +1,314 @@ +//! Encoding and decoding of mergeable container ids. + +#[test] +#[cfg(feature = "counter")] +fn mergeable_container_id_roundtrips_parent_key_and_type() { + use loro_internal::ContainerType; + + let parent = loro_common::ContainerID::new_root("state", ContainerType::Map); + let key = "field\u{1}with/slash:and:semicolon"; + let cid = loro_common::ContainerID::new_mergeable(&parent, key, ContainerType::Counter); + + assert!(cid.is_mergeable()); + let (decoded_parent, decoded_key, decoded_type) = cid.parse_mergeable().unwrap(); + assert_eq!(decoded_parent, parent); + assert_eq!(decoded_key, key); + assert_eq!(decoded_type, ContainerType::Counter); +} + +#[test] +fn user_root_names_cannot_use_mergeable_namespace() { + assert!(!loro_common::check_root_container_name( + loro_common::MERGEABLE_NAMESPACE_PREFIX + )); +} + +/// `parse_mergeable` is a pure decoder and must return `None` (not panic, not +/// silently misinterpret) for every malformed payload. This guards against +/// future drift in the encoder + decoder pair. +/// +/// The cid does NOT encode the container kind in its name — kind already lives in +/// `ContainerID::Root::container_type`. Distinct kinds at the same `(parent, key)` therefore share +/// the same name string but compare unequal at the `ContainerID` level, which is what callers see. +#[test] +#[cfg(feature = "counter")] +fn parse_mergeable_rejects_malformed_payloads() { + use loro_common::{ContainerID, ContainerType}; + + // Non-mergeable root: returns None. + let plain_root = ContainerID::new_root("ordinary", ContainerType::Map); + assert!(plain_root.parse_mergeable().is_none()); + + // Mergeable prefix but missing the required key segment. + let no_key = ContainerID::Root { + name: "🤝:$state".into(), + container_type: ContainerType::Counter, + }; + assert!( + no_key.parse_mergeable().is_none(), + "payload without a key segment must reject" + ); + + // Mergeable prefix, but empty payload. + let truncated = ContainerID::Root { + name: "🤝:".into(), + container_type: ContainerType::Counter, + }; + assert!( + truncated.parse_mergeable().is_none(), + "empty payload must reject" + ); + + // Unknown escapes, raw slash, and dangling backslash must reject. + let malformed = ContainerID::Root { + name: "🤝:$state>bad\\xescape".into(), + container_type: ContainerType::Counter, + }; + assert!( + malformed.parse_mergeable().is_none(), + "unknown escape in payload must reject" + ); + + let raw_slash = ContainerID::Root { + name: "🤝:$state>raw/slash".into(), + container_type: ContainerType::Counter, + }; + assert!(raw_slash.parse_mergeable().is_none()); + + let dangling = ContainerID::Root { + name: "🤝:$state>dangling\\".into(), + container_type: ContainerType::Counter, + }; + assert!(dangling.parse_mergeable().is_none()); + + for name in ["🤝:$root\\sname>key", "🤝:$root\\0name>key"] { + let invalid_root_name = ContainerID::Root { + name: name.into(), + container_type: ContainerType::Counter, + }; + assert!( + invalid_root_name.parse_mergeable().is_none(), + "root base parent must reject decoded slash/NUL: {name:?}" + ); + } + + for name in [ + "🤝:@Z:0>key", + "🤝:@001:0>key", + "🤝:@1:00>key", + "🤝:@1:-0>key", + ] { + let non_canonical = ContainerID::Root { + name: name.into(), + container_type: ContainerType::Counter, + }; + assert!( + non_canonical.parse_mergeable().is_none(), + "non-canonical base36 payload must reject: {name:?}" + ); + } +} + +/// `LoroDoc::get_map` must reject names in the mergeable namespace at the +/// call site, not just in `check_root_container_name`. Otherwise user code +/// could fabricate a Root cid that masquerades as a mergeable child and +/// confuse the parent-edge walks. +/// +/// This test runs `check_root_container_name` directly on a variety of +/// user-supplied strings; the runtime `get_map` / `get_text` / etc. calls +/// route through this validator (see callers in `crates/loro-internal/src/`). +/// If the validator is bypassed by a runtime path, that's a separate bug — +/// but it's not something this test can prove without intentionally writing +/// `🤝:` keys, which is what we're trying to prevent in the first place. +#[test] +fn root_name_validator_rejects_mergeable_namespace_inputs() { + use loro_common::{check_root_container_name, MERGEABLE_NAMESPACE_PREFIX}; + + // Bare prefix. + assert!(!check_root_container_name(MERGEABLE_NAMESPACE_PREFIX)); + // Prefix + arbitrary payload. + assert!(!check_root_container_name("🤝:deadbeef")); + assert!(!check_root_container_name("🤝:")); + // Prefix-as-substring is OK (not a prefix), validator still allows it. + assert!(check_root_container_name("foo🤝:bar")); + // Prefix with a leading zero-width space is NOT a prefix match, allowed. + assert!(check_root_container_name("\u{200B}🤝:abc")); + // Sanity: ordinary user names still pass. + assert!(check_root_container_name("state")); + assert!(check_root_container_name("ordinary-name_with-symbols")); + // Empty is still rejected (pre-existing behavior). + assert!(!check_root_container_name("")); + // Slash and NUL still rejected (pre-existing behavior). + assert!(!check_root_container_name("a/b")); + assert!(!check_root_container_name("a\0b")); +} + +/// For each supported container kind, `new_mergeable` produces a deterministic +/// cid that decodes back to the same `(parent, key, kind)`. Counter is gated +/// on the feature; the rest are unconditional. +#[test] +fn mergeable_cid_roundtrips_for_every_container_kind() { + use loro_common::{ContainerID, ContainerType}; + let parent = ContainerID::new_root("state", ContainerType::Map); + + for kind in ContainerType::ALL_TYPES { + let cid = ContainerID::new_mergeable(&parent, "field", kind); + assert!(cid.is_mergeable(), "kind {kind:?}: must be mergeable"); + let again = ContainerID::new_mergeable(&parent, "field", kind); + assert_eq!(cid, again, "kind {kind:?}: cid must be deterministic"); + let (decoded_parent, decoded_key, decoded_kind) = cid + .parse_mergeable() + .unwrap_or_else(|| panic!("kind {kind:?}: parse_mergeable returned None")); + assert_eq!(decoded_parent, parent, "kind {kind:?}: parent roundtrip"); + assert_eq!(decoded_key, "field", "kind {kind:?}: key roundtrip"); + assert_eq!(decoded_kind, kind, "kind {kind:?}: kind roundtrip"); + } +} + +/// Key encoding is escaped path segments, so it must round-trip cleanly for +/// degenerate inputs: empty, long, delimiters, embedded NUL, and embedded +/// mergeable prefix substring. Catches off-by-one and ad-hoc string-split +/// mistakes in any future decoder change. +#[test] +fn mergeable_cid_roundtrips_for_degenerate_keys() { + use loro_common::{ContainerID, ContainerType}; + let parent = ContainerID::new_root("state", ContainerType::Map); + + let long_key: String = std::iter::repeat('k').take(2048).collect(); + let cases: Vec<&str> = vec![ + "", + long_key.as_str(), + "with\0nul\0bytes", + "embedded 🤝: substring in the middle", + "starts_with_🤝:_prefix", + "trailing_emoji_🤝:", + "ascii/slash/looking", + "contains>separator", + "contains\\backslash", + "mixed>\\/\0chars", + ]; + + for key in cases { + let cid = ContainerID::new_mergeable(&parent, key, ContainerType::Map); + assert!(cid.is_mergeable(), "key {key:?}: must be mergeable"); + let (decoded_parent, decoded_key, decoded_kind) = cid + .parse_mergeable() + .unwrap_or_else(|| panic!("key {key:?}: parse_mergeable returned None")); + assert_eq!(decoded_parent, parent); + assert_eq!(decoded_key, key, "key {key:?}: round-trip mismatch"); + assert_eq!(decoded_kind, ContainerType::Map); + } +} + +#[test] +fn mergeable_cid_uses_flattened_path_payload() { + use loro_common::{ContainerID, ContainerType, MERGEABLE_NAMESPACE_PREFIX}; + + let parent = ContainerID::new_root("state", ContainerType::Map); + let child = ContainerID::new_mergeable(&parent, "note>1", ContainerType::Map); + let grandchild = ContainerID::new_mergeable(&child, "body/slash\\nul\0", ContainerType::Text); + + let name = match &grandchild { + ContainerID::Root { name, .. } => name.as_str(), + _ => panic!("expected Root"), + }; + assert!( + name.starts_with(MERGEABLE_NAMESPACE_PREFIX), + "mergeable cid must use the reserved namespace" + ); + assert_eq!( + name, "🤝:$state>note\\>1>body\\sslash\\\\nul\\0", + "nested mergeable ids must extend the flattened path, not embed parent cid bytes" + ); + assert!( + !name.contains('/'), + "synthetic root names must not contain raw slash" + ); + assert!( + !name.contains('\0'), + "synthetic root names must not contain raw NUL" + ); + + let (decoded_parent, decoded_key, decoded_kind) = grandchild.parse_mergeable().unwrap(); + assert_eq!(decoded_parent, child); + assert_eq!(decoded_key, "body/slash\\nul\0"); + assert_eq!(decoded_kind, ContainerType::Text); +} + +#[test] +fn mergeable_cid_roundtrips_normal_base_parent() { + use loro_common::{ContainerID, ContainerType, ID}; + + let parent = ContainerID::new_normal( + ID { + peer: u64::MAX, + counter: i32::MIN, + }, + ContainerType::Map, + ); + let cid = ContainerID::new_mergeable(&parent, "field", ContainerType::List); + let name = match &cid { + ContainerID::Root { name, .. } => name.as_str(), + _ => panic!("expected Root"), + }; + assert!( + name.starts_with("🤝:@3w5e11264sgsf:-zik0zk>"), + "normal parent ids should use compact base36 encoding; got {name:?}" + ); + + let (decoded_parent, decoded_key, decoded_kind) = cid.parse_mergeable().unwrap(); + assert_eq!(decoded_parent, parent); + assert_eq!(decoded_key, "field"); + assert_eq!(decoded_kind, ContainerType::List); +} + +#[test] +fn mergeable_cid_roundtrips_escaped_root_base_parent() { + use loro_common::{ContainerID, ContainerType}; + + let parent = ContainerID::new_root("root>\\name", ContainerType::Map); + let cid = ContainerID::new_mergeable(&parent, "field", ContainerType::Tree); + let name = match &cid { + ContainerID::Root { name, .. } => name.as_str(), + _ => panic!("expected Root"), + }; + assert_eq!(name, "🤝:$root\\>\\\\name>field"); + + let (decoded_parent, decoded_key, decoded_kind) = cid.parse_mergeable().unwrap(); + assert_eq!(decoded_parent, parent); + assert_eq!(decoded_key, "field"); + assert_eq!(decoded_kind, ContainerType::Tree); +} + +#[test] +#[should_panic(expected = "mergeable child parent must be a map")] +fn mergeable_cid_rejects_non_map_parent() { + use loro_common::{ContainerID, ContainerType}; + + let parent = ContainerID::new_root("content", ContainerType::Text); + let _ = ContainerID::new_mergeable(&parent, "field", ContainerType::Map); +} + +#[test] +fn nested_mergeable_cid_size_grows_linearly() { + use loro_common::{ContainerID, ContainerType}; + + let mut cid = ContainerID::new_root("state", ContainerType::Map); + let mut sizes = Vec::new(); + for _ in 0..=8 { + cid = ContainerID::new_mergeable(&cid, "k", ContainerType::Map); + sizes.push(cid.to_bytes().len()); + } + + for window in sizes.windows(2) { + assert_eq!( + window[1] - window[0], + 2, + "each additional one-byte key should add only separator + key bytes; sizes={sizes:?}" + ); + } + assert!( + sizes[8] < 64, + "depth 8 should stay compact with flattened encoding; sizes={sizes:?}" + ); +} diff --git a/vendor/loro-internal/tests/mergeable_container.rs b/vendor/loro-internal/tests/mergeable_container.rs new file mode 100644 index 00000000..bbb6c4cc --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container.rs @@ -0,0 +1,15 @@ +//! Integration tests for mergeable container support. The actual tests live in focused files +//! under `tests/mergeable_container/`; this harness aggregates them into a single test target. + +#[path = "mergeable_container/convergence.rs"] +mod convergence; +#[path = "mergeable_container/delete.rs"] +mod delete; +#[path = "mergeable_container/discriminator.rs"] +mod discriminator; +#[path = "mergeable_container/events_and_paths.rs"] +mod events_and_paths; +#[path = "mergeable_container/snapshot.rs"] +mod snapshot; +#[path = "mergeable_container/type_conflict.rs"] +mod type_conflict; diff --git a/vendor/loro-internal/tests/mergeable_container/common.rs b/vendor/loro-internal/tests/mergeable_container/common.rs new file mode 100644 index 00000000..5add3778 --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/common.rs @@ -0,0 +1,25 @@ +//! Shared helpers for the mergeable-container integration tests. +//! +//! Each split test file uses `#[path = "mergeable_container/common.rs"] mod common;` from +//! the harness in `tests/mergeable_container.rs`, so test fns can reach `common::doc` and +//! `common::sync` regardless of which sub-file they live in. + +// Each split test file imports this module via `#[path = "common.rs"]`, so each helper appears +// as a separate copy per file. Suppress the dead-code warning that fires when an individual file +// only uses a subset of the helpers. +#![allow(dead_code)] + +use loro_internal::{loro::ExportMode, LoroDoc}; + +pub fn doc(peer: u64) -> LoroDoc { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(peer).unwrap(); + doc +} + +pub fn sync(a: &LoroDoc, b: &LoroDoc) { + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + b.import(&a.export(ExportMode::updates(&b.oplog_vv())).unwrap()) + .unwrap(); +} diff --git a/vendor/loro-internal/tests/mergeable_container/convergence.rs b/vendor/loro-internal/tests/mergeable_container/convergence.rs new file mode 100644 index 00000000..d2ad233f --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/convergence.rs @@ -0,0 +1,288 @@ +//! Concurrent first-create convergence and basic merging. + +#[path = "common.rs"] +mod common; +use common::{doc, sync}; + +use loro_internal::{cursor::PosType, event::Index, handler::ValueOrHandler, HandlerTrait, ToJson}; +use serde_json::json; + +#[test] +#[cfg(feature = "counter")] +fn concurrent_counter_increments_show_current_lost_update_bug() { + let a = doc(1); + let b = doc(2); + + let a_root = a.get_map("state"); + let b_root = b.get_map("state"); + + let a_counter = a_root.ensure_mergeable_counter("revision").unwrap(); + let b_counter = b_root.ensure_mergeable_counter("revision").unwrap(); + + assert_eq!( + a_counter.id(), + b_counter.id(), + "both peers should produce the same deterministic cid" + ); + assert!( + a_counter.id().is_mergeable(), + "counter cid should be in the mergeable namespace" + ); + + a_counter.increment(1.0).unwrap(); + b_counter.increment(1.0).unwrap(); + + sync(&a, &b); + + assert_eq!( + a.get_deep_value().to_json_value(), + json!({ "state": { "revision": 2.0 } }), + "both concurrent increments should survive on the merged counter", + ); + assert_eq!( + b.get_deep_value().to_json_value(), + a.get_deep_value().to_json_value() + ); + + let a_cid = match a_root.get_("revision") { + Some(ValueOrHandler::Handler(handler)) => Some(handler.id()), + _ => None, + }; + let b_cid = match b_root.get_("revision") { + Some(ValueOrHandler::Handler(handler)) => Some(handler.id()), + _ => None, + }; + assert_eq!( + a_cid, b_cid, + "both peers should resolve the same logical counter cid" + ); +} + +#[test] +fn concurrent_text_updates_show_current_lost_update_bug() { + let a = doc(1); + let b = doc(2); + let a_text = a.get_map("state").ensure_mergeable_text("notes").unwrap(); + let b_text = b.get_map("state").ensure_mergeable_text("notes").unwrap(); + + assert_eq!( + a_text.id(), + b_text.id(), + "both peers should produce the same deterministic cid" + ); + assert!( + a_text.id().is_mergeable(), + "text cid should be in the mergeable namespace" + ); + + a_text.insert(0, "A", PosType::Unicode).unwrap(); + b_text.insert(0, "B", PosType::Unicode).unwrap(); + sync(&a, &b); + + let value = a.get_deep_value().to_json_value(); + assert!( + value == json!({ "state": { "notes": "AB" } }) + || value == json!({ "state": { "notes": "BA" } }), + "both concurrent text edits should survive on the merged text; got {value}", + ); +} + +#[test] +fn concurrent_list_inserts_show_current_lost_update_bug() { + let a = doc(1); + let b = doc(2); + let a_list = a.get_map("state").ensure_mergeable_list("items").unwrap(); + let b_list = b.get_map("state").ensure_mergeable_list("items").unwrap(); + + assert_eq!( + a_list.id(), + b_list.id(), + "both peers should produce the same deterministic cid" + ); + assert!( + a_list.id().is_mergeable(), + "list cid should be in the mergeable namespace" + ); + + a_list.insert(0, "A").unwrap(); + b_list.insert(0, "B").unwrap(); + sync(&a, &b); + + let value = a.get_deep_value().to_json_value(); + assert!( + value == json!({ "state": { "items": ["A", "B"] } }) + || value == json!({ "state": { "items": ["B", "A"] } }), + "both concurrent list inserts should survive on the merged list; got {value}", + ); +} + +/// Two peers each obtain the "profile" Map via `ensure_mergeable_map` and write +/// to *different* keys. With non-mergeable child Maps, each peer creates a +/// distinct peer-specific cid, so LWW drops one peer's Map entirely even +/// though the key sets are disjoint. Once child Maps are mergeable the merged +/// value should contain both keys. +#[test] +fn concurrent_map_writes_show_current_lost_update_bug() { + let a = doc(1); + let b = doc(2); + let a_map = a.get_map("state").ensure_mergeable_map("profile").unwrap(); + let b_map = b.get_map("state").ensure_mergeable_map("profile").unwrap(); + + assert_eq!( + a_map.id(), + b_map.id(), + "both peers should produce the same deterministic cid" + ); + assert!( + a_map.id().is_mergeable(), + "map cid should be in the mergeable namespace" + ); + + a_map.insert("name", "Ada").unwrap(); + b_map.insert("title", "Engineer").unwrap(); + sync(&a, &b); + + assert_eq!( + a.get_deep_value().to_json_value(), + json!({ "state": { "profile": { "name": "Ada", "title": "Engineer" } } }) + ); +} + +/// Three peers each increment the same mergeable counter once. After a full +/// round-robin sync, every peer must observe `3.0` and the same deterministic +/// cid. Two-peer tests can't catch ordering or idempotency bugs that fire +/// only when more than two histories overlap. +#[test] +#[cfg(feature = "counter")] +fn three_peer_mergeable_counter_convergence() { + let a = doc(1); + let b = doc(2); + let c = doc(3); + + let a_counter = a + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + let b_counter = b + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + let c_counter = c + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + assert_eq!(a_counter.id(), b_counter.id()); + assert_eq!(b_counter.id(), c_counter.id()); + + a_counter.increment(1.0).unwrap(); + b_counter.increment(1.0).unwrap(); + c_counter.increment(1.0).unwrap(); + + // Full round-robin: every pair syncs. + sync(&a, &b); + sync(&b, &c); + sync(&a, &c); + sync(&a, &b); + + let expected = json!({ "state": { "revision": 3.0 } }); + assert_eq!(a.get_deep_value().to_json_value(), expected); + assert_eq!(b.get_deep_value().to_json_value(), expected); + assert_eq!(c.get_deep_value().to_json_value(), expected); +} + +/// After A and B sync once, both peers concurrently mutate the same +/// mergeable counter again, then sync. Convergence must hold on the second +/// round — the deterministic cid plus CRDT merge must keep working after +/// the logical mergeable edge has already been resolved and used. +#[test] +#[cfg(feature = "counter")] +fn post_merge_concurrent_counter_increments_converge() { + let a = doc(1); + let b = doc(2); + + let a_counter = a + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + let b_counter = b + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + a_counter.increment(1.0).unwrap(); + b_counter.increment(1.0).unwrap(); + sync(&a, &b); + assert_eq!( + a.get_deep_value().to_json_value(), + json!({ "state": { "revision": 2.0 } }) + ); + + // Round 2: concurrent edits on the already-merged child. + a_counter.increment(10.0).unwrap(); + b_counter.increment(100.0).unwrap(); + sync(&a, &b); + + let expected = json!({ "state": { "revision": 112.0 } }); + assert_eq!(a.get_deep_value().to_json_value(), expected); + assert_eq!(b.get_deep_value().to_json_value(), expected); +} + +/// Two peers independently navigate `state → mergeable map "profile" → +/// mergeable counter "revision"` and increment. The deterministic cid for +/// "revision" is the same on both peers (it's a function of the "profile" +/// cid, which is itself deterministic from "state"'s cid + "profile" + Map). +/// After sync, both peers see the counter at 2.0 nested correctly. +#[test] +#[cfg(feature = "counter")] +fn nested_mergeable_concurrent_counter_converges() { + let a = doc(1); + let b = doc(2); + + let a_profile = a.get_map("state").ensure_mergeable_map("profile").unwrap(); + let b_profile = b.get_map("state").ensure_mergeable_map("profile").unwrap(); + assert_eq!(a_profile.id(), b_profile.id()); + + let a_rev = a_profile.ensure_mergeable_counter("revision").unwrap(); + let b_rev = b_profile.ensure_mergeable_counter("revision").unwrap(); + assert_eq!(a_rev.id(), b_rev.id()); + + a_rev.increment(1.0).unwrap(); + b_rev.increment(1.0).unwrap(); + sync(&a, &b); + + let expected = json!({ "state": { "profile": { "revision": 2.0 } } }); + assert_eq!(a.get_deep_value().to_json_value(), expected); + assert_eq!(b.get_deep_value().to_json_value(), expected); + + // Path resolution still walks both mergeable hops. + let path = a.get_path_to_container(&a_rev.id()).expect("path"); + let indexes = path.iter().map(|(_, idx)| idx.clone()).collect::>(); + assert_eq!( + indexes, + vec![ + Index::Key("state".into()), + Index::Key("profile".into()), + Index::Key("revision".into()), + ] + ); +} + +/// `ensure_mergeable_*` cannot work on a detached handler: the deterministic cid is computed from +/// the parent's cid, which a detached parent doesn't have. Falling back to a non-deterministic +/// regular child would silently break the mergeable guarantee once the handler attaches, so the +/// detached call must surface that misuse explicitly instead. +#[test] +#[cfg(feature = "counter")] +fn detached_map_ensure_mergeable_counter_rejects_misuse() { + use loro_internal::MapHandler; + let detached = MapHandler::new_detached(); + let err = detached + .ensure_mergeable_counter("revision") + .expect_err("detached ensure_mergeable_* must error"); + assert!( + matches!( + err, + loro_common::LoroError::MisuseDetachedContainer { .. } + ), + "detached error must be MisuseDetachedContainer; got {err:?}" + ); +} diff --git a/vendor/loro-internal/tests/mergeable_container/delete.rs b/vendor/loro-internal/tests/mergeable_container/delete.rs new file mode 100644 index 00000000..a032e7d5 --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/delete.rs @@ -0,0 +1,339 @@ +//! Delete semantics for mergeable children. +//! +//! `delete(key)` overwrites the binary marker in the parent map's value slot with `None`, exactly +//! as deleting a regular child container clears its value-table entry. The child +//! becomes unreachable; its container state is preserved in history. Re-calling +//! `ensure_mergeable_(key)` rewrites the marker and resurfaces the preserved state, and +//! Map LWW resolves any concurrent delete-vs-recreate race — symmetric with normal Loro +//! write/delete (loro-dev/loro#759). + +#[path = "common.rs"] +mod common; +use common::{doc, sync}; + +use loro_internal::{loro::ExportMode, HandlerTrait, ToJson}; +use serde_json::json; + +/// `delete(key)` on a mergeable key clears the marker, so the child no longer appears in +/// deep value. The child's container state is preserved: re-getting it resolves to the same +/// deterministic cid and resurfaces the prior value (delete detaches, it does not reset). +#[test] +#[cfg(feature = "counter")] +fn delete_clears_marker_and_recreate_resurfaces_state() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(3.0).unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "revision": 3.0 } }) + ); + + root.delete("revision").unwrap(); + doc.commit_then_renew(); + + // Cleared: not in deep value, exactly like any deleted container. + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": {} }), + "delete must clear the mergeable child from deep value" + ); + + // Re-get rewrites the marker. The child resurfaces with its PRESERVED value (3.0), + // not a reset to 0.0 — delete detaches, it does not destroy the container state. + let counter2 = root.ensure_mergeable_counter("revision").unwrap(); + doc.commit_then_renew(); + assert_eq!(counter2.id(), counter.id(), "deterministic cid is stable"); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "revision": 3.0 } }), + "re-get rewrites the marker and resurfaces the preserved state" + ); +} + +/// `delete(key)` records nothing special in side state — the slot value (`None`) is the whole +/// story. After delete, the parent map simply has no value at the key. +#[test] +#[cfg(feature = "counter")] +fn delete_leaves_no_value_at_key() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + assert!(root.get("revision").is_some(), "marker present"); + + root.delete("revision").unwrap(); + doc.commit_then_renew(); + + assert_eq!( + root.get("revision"), + None, + "after delete the slot holds no value" + ); +} + +/// A local `delete` clears the child from deep value immediately, without waiting for any +/// remote import cycle. +#[test] +#[cfg(feature = "counter")] +fn local_delete_immediately_clears_mergeable_child() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "revision": 1.0 } }) + ); + + root.delete("revision").unwrap(); + doc.commit_then_renew(); + + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": {} }), + "local delete must remove the mergeable child from deep value" + ); +} + +/// Recreate after seeing a delete wins: peer A deletes the key; peer B imports the delete (its +/// slot is now cleared), then re-calls `ensure_mergeable_`, which re-emits a FRESH marker +/// because the slot is empty. That marker's IdLp dominates the delete, so the child is +/// visible again and its preserved state resurfaces — symmetric with re-`set_container` after a +/// regular delete. +/// +/// (Note the symmetry: if B re-calls `ensure_mergeable_` while still holding the OLD +/// marker it imported earlier — i.e. before seeing the delete — the call is idempotent and +/// emits no op, so the concurrent delete wins, exactly as a regular concurrent delete beats a +/// stale local handle. That case is covered by `concurrent_delete_wins_against_earlier_increment`.) +#[test] +#[cfg(feature = "counter")] +fn recreate_after_seen_delete_resurfaces_preserved_state() { + let a = doc(1); + let b = doc(2); + + let a_root = a.get_map("state"); + let a_counter = a_root.ensure_mergeable_counter("revision").unwrap(); + a_counter.increment(1.0).unwrap(); + a.commit_then_renew(); + sync(&a, &b); + + // A deletes; B imports the delete so B's slot is cleared. + a_root.delete("revision").unwrap(); + a.commit_then_renew(); + sync(&a, &b); + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": {} }), + "B sees the cleared slot after importing the delete" + ); + + // B re-creates: the slot is empty, so this re-emits a fresh marker dominating the + // delete. The preserved counter state (1.0) resurfaces, plus B's new increment. + let b_counter = b + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + b_counter.increment(100.0).unwrap(); + b.commit_then_renew(); + + sync(&a, &b); + + let va = a.get_deep_value().to_json_value(); + let vb = b.get_deep_value().to_json_value(); + assert_eq!(va, vb, "peers must converge"); + assert_eq!( + va["state"]["revision"], + json!(101.0), + "recreate after seeing the delete resurfaces preserved state (1.0) + new increment (100.0)" + ); +} + +/// Concurrent delete-wins: peer A increments, peer B deletes with a higher IdLp and never +/// recreates. The delete dominates the marker via Map LWW, so the child is gone on both +/// peers — symmetric with a concurrent delete winning over a regular write. +#[test] +#[cfg(feature = "counter")] +fn concurrent_delete_wins_against_earlier_increment() { + let a = doc(1); + let b = doc(2); + + let a_root = a.get_map("state"); + let a_counter = a_root.ensure_mergeable_counter("revision").unwrap(); + a_counter.increment(1.0).unwrap(); + a.commit_then_renew(); + sync(&a, &b); + + // B increments first (low IdLp). + let b_root = b.get_map("state"); + let b_counter = b_root.ensure_mergeable_counter("revision").unwrap(); + b_counter.increment(100.0).unwrap(); + b.commit_then_renew(); + + // A advances its clock past B's increment, then deletes. + for i in 0..5 { + a_root.insert(&format!("noise_{i}"), i).unwrap(); + a.commit_then_renew(); + } + a_root.delete("revision").unwrap(); + a.commit_then_renew(); + + sync(&a, &b); + + let va = a.get_deep_value().to_json_value(); + let vb = b.get_deep_value().to_json_value(); + assert_eq!(va, vb); + assert!( + va["state"].get("revision").is_none(), + "delete with higher IdLp must dominate the marker; got {va}" + ); +} + +/// Remote delete clears the child on the receiver: peer B has a visible mergeable counter; peer A +/// deletes the key with a higher IdLp. After sync, B's deep value drops the counter because A's +/// delete wins the Map LWW for the marker slot. +#[test] +#[cfg(feature = "counter")] +fn remote_delete_clears_mergeable_child_on_receiver() { + let a = doc(1); + let b = doc(2); + + let b_root = b.get_map("state"); + let b_counter = b_root.ensure_mergeable_counter("revision").unwrap(); + b_counter.increment(1.0).unwrap(); + b.commit_then_renew(); + sync(&a, &b); + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "revision": 1.0 } }) + ); + + let a_root = a.get_map("state"); + for i in 0..5 { + a_root.insert(&format!("noise_{i}"), i).unwrap(); + a.commit_then_renew(); + } + a_root.delete("revision").unwrap(); + a.commit_then_renew(); + + sync(&a, &b); + + let vb = b.get_deep_value().to_json_value(); + assert!( + vb["state"].get("revision").is_none(), + "remote delete must clear B's mergeable child; got {vb}" + ); +} + +/// Three-peer convergence across a delete and a competing-kind recreate: B creates a counter, A +/// deletes it with a higher IdLp, C recreates the key as a text. All peers converge on the text. +#[test] +#[cfg(feature = "counter")] +fn three_peer_delete_then_recreate_converges() { + use loro_internal::cursor::PosType; + + let a = doc(1); + let b = doc(2); + let c = doc(3); + + let b_counter = b + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + b_counter.increment(1.0).unwrap(); + b.commit_then_renew(); + + sync(&a, &b); + sync(&b, &c); + + let a_root = a.get_map("state"); + for i in 0..6 { + a_root.insert(&format!("noise_{i}"), i).unwrap(); + a.commit_then_renew(); + } + a_root.delete("revision").unwrap(); + a.commit_then_renew(); + + sync(&a, &b); + sync(&a, &c); + + // C recreates "revision" as a text with a post-delete IdLp. + let c_text = c.get_map("state").ensure_mergeable_text("revision").unwrap(); + c_text.insert(0, "after-delete", PosType::Unicode).unwrap(); + c.commit_then_renew(); + + sync(&b, &c); + sync(&a, &c); + sync(&a, &b); + + let expected = json!({ + "state": { + "noise_0": 0, + "noise_1": 1, + "noise_2": 2, + "noise_3": 3, + "noise_4": 4, + "noise_5": 5, + "revision": "after-delete", + } + }); + assert_eq!(a.get_deep_value().to_json_value(), expected, "A converges"); + assert_eq!(b.get_deep_value().to_json_value(), expected, "B converges"); + assert_eq!(c.get_deep_value().to_json_value(), expected, "C converges"); +} + +/// Snapshot round-trip preserves the delete: a peer creates and deletes a mergeable counter, then +/// exports a snapshot. The receiver sees no counter — the `None` slot rides through the snapshot +/// as ordinary map state, so no special recovery is needed. +#[test] +#[cfg(feature = "counter")] +fn snapshot_roundtrip_preserves_delete() { + let a = doc(1); + let a_root = a.get_map("state"); + let a_counter = a_root.ensure_mergeable_counter("revision").unwrap(); + a_counter.increment(5.0).unwrap(); + a_root.delete("revision").unwrap(); + a.commit_then_renew(); + assert_eq!(a.get_deep_value().to_json_value(), json!({ "state": {} })); + + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + let b = doc(2); + b.import(&snapshot).unwrap(); + + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": {} }), + "snapshot import must preserve the cleared slot; no mergeable child appears" + ); + assert_eq!( + b.get_map("state").get("revision"), + None, + "the cleared slot has no value after snapshot import" + ); +} + +/// `delete` on a mergeable key emits exactly one op — a regular `MapSet { value: None }` — by the +/// local peer. No new op types are introduced; older peers apply it as an ordinary map-key clear. +#[test] +#[cfg(feature = "counter")] +fn delete_on_mergeable_key_emits_only_existing_op_types() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + let counter_before = doc.oplog_vv().get(&1).copied().unwrap_or(0); + + root.delete("revision").unwrap(); + doc.commit_then_renew(); + let counter_after = doc.oplog_vv().get(&1).copied().unwrap_or(0); + + let new_ops = counter_after - counter_before; + assert_eq!( + new_ops, 1, + "delete must emit exactly one op (the MapSet clearing the slot); got {new_ops}" + ); +} diff --git a/vendor/loro-internal/tests/mergeable_container/discriminator.rs b/vendor/loro-internal/tests/mergeable_container/discriminator.rs new file mode 100644 index 00000000..b15695e0 --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/discriminator.rs @@ -0,0 +1,313 @@ +//! Marker-based mergeable type-conflict resolution. +//! +//! `ensure_mergeable_(key)` records which container kind is active at +//! `(parent, key)` by writing a compact binary marker op against the parent map. +//! The active mergeable child is whichever marker the parent map's regular LWW +//! resolves to. See loro-dev/loro#759. + +#[path = "common.rs"] +mod common; +use common::{doc, sync}; + +use loro_common::{mergeable_marker, ContainerType, LoroError, LoroValue, MERGEABLE_MARKER_MAGIC}; +use loro_internal::{HandlerTrait, ToJson}; +use serde_json::json; + +/// `ensure_mergeable_list(key)` activates a List child at the parent map's slot. From the +/// public API, `map.get(key)` reports it as a Container value carrying the deterministic +/// mergeable cid — symmetric with what an explicit `set_container` would produce. +#[test] +fn ensure_mergeable_writes_marker_into_parent_slot() { + let d = doc(1); + let root = d.get_map("state"); + + assert_eq!(root.get("tags"), None, "key starts empty"); + + let list = root.ensure_mergeable_list("tags").unwrap(); + + assert_eq!( + root.get("tags"), + Some(LoroValue::Container(list.id())), + "ensure_mergeable_list must surface the activated List child as a Container value" + ); +} + +/// `ensure_mergeable_(key)` must not silently clobber an existing non-mergeable value at the +/// key. A plain scalar or a regular child container occupying the slot makes the call an error, +/// so the marker overwrite is never a hidden side effect of a `get_`-named API. +#[test] +fn ensure_mergeable_rejects_overwriting_a_scalar_value() { + let d = doc(1); + let root = d.get_map("state"); + root.insert("field", 5).unwrap(); + + let err = root + .ensure_mergeable_list("field") + .expect_err("ensure_mergeable over a scalar must error"); + assert!( + matches!(err, LoroError::ArgErr(_)), + "expected ArgErr, got {err:?}" + ); + assert_eq!( + root.get("field"), + Some(5.into()), + "the existing scalar value must be left untouched" + ); +} + +/// The same guard applies when the slot holds a regular child container: requesting a mergeable +/// child there is an error and must not overwrite the container reference. +#[test] +fn ensure_mergeable_rejects_overwriting_a_regular_child_container() { + let d = doc(1); + let root = d.get_map("state"); + root.insert_container("field", loro_internal::handler::MapHandler::new_detached()) + .unwrap(); + + let err = root + .ensure_mergeable_list("field") + .expect_err("ensure_mergeable over a regular child container must error"); + assert!( + matches!(err, LoroError::ArgErr(_)), + "expected ArgErr, got {err:?}" + ); +} + +/// An empty slot, or a slot already holding a mergeable marker, is fair game: +/// `ensure_mergeable_` creates on empty and is idempotent / kind-changes over an existing +/// marker. Only non-mergeable occupants are rejected. +#[test] +fn ensure_mergeable_allows_empty_slot_and_existing_marker() { + let d = doc(1); + let root = d.get_map("state"); + + // Empty slot: creates. + let _list = root.ensure_mergeable_list("a").unwrap(); + // Same kind again: idempotent, still Ok. + let _list2 = root.ensure_mergeable_list("a").unwrap(); + // Different kind over an existing marker: kind change, still Ok. + let _map = root.ensure_mergeable_map("a").unwrap(); +} + +/// A user string must stay a plain scalar and block mergeable child creation. +#[test] +fn user_string_is_not_mergeable() { + let d = doc(1); + let root = d.get_map("state"); + + root.insert("field", "not-a-marker").unwrap(); + d.commit_then_renew(); + + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "field": "not-a-marker" } }) + ); + let err = root + .ensure_mergeable_map("field") + .expect_err("user string must not be treated as a mergeable marker"); + assert!( + matches!(err, LoroError::ArgErr(_)), + "expected ArgErr, got {err:?}" + ); +} + +/// A marker is bound to its exact `(parent, key, kind)` by the digest. Copying +/// the binary payload to another key leaves it as an inert user binary value. +#[test] +fn marker_for_another_key_is_not_mergeable() { + let d = doc(1); + let root = d.get_map("state"); + let marker_for_other = mergeable_marker(&root.id(), "other", ContainerType::Map); + + root.insert("field", marker_for_other.clone()).unwrap(); + d.commit_then_renew(); + + assert_eq!(root.get("field"), Some(marker_for_other)); + let err = root + .ensure_mergeable_map("field") + .expect_err("marker digest for another key must not activate this key"); + assert!( + matches!(err, LoroError::ArgErr(_)), + "expected ArgErr, got {err:?}" + ); +} + +/// A second `ensure_mergeable_(key)` with the SAME kind is idempotent: it +/// does not emit another op (the parent slot already holds the matching marker). +#[test] +fn repeated_ensure_mergeable_same_kind_is_idempotent() { + let d = doc(1); + let root = d.get_map("state"); + + let _first = root.ensure_mergeable_map("profile").unwrap(); + d.commit_then_renew(); + let ops_after_first = d.len_ops(); + + let _second = root.ensure_mergeable_map("profile").unwrap(); + d.commit_then_renew(); + let ops_after_second = d.len_ops(); + + assert_eq!( + ops_after_second, ops_after_first, + "a second same-kind ensure_mergeable must not emit another op" + ); +} + +/// The resolver substitutes the resolved child for the binary marker in deep value: the (empty) +/// child is visible immediately as its default, and the raw marker never leaks into the read view. +#[test] +fn marker_resolves_to_child_in_deep_value() { + let d = doc(1); + let root = d.get_map("state"); + let _list = root.ensure_mergeable_list("tags").unwrap(); + d.commit_then_renew(); + + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "tags": [] } }), + "the marker must resolve to the child's value, not leak as a binary value" + ); + + // Now add an item; the resolved child must reflect it. + let list = root.ensure_mergeable_list("tags").unwrap(); + list.insert(0, "hello").unwrap(); + d.commit_then_renew(); + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "tags": ["hello"] } }) + ); +} + +/// A nested chain of mergeable children resolves at every hop: the intermediate mergeable map is +/// visible (it has a marker on its parent) even though it never receives a direct content op, only +/// its nested child does. This is the case the old `has_ops` gate broke. +#[test] +#[cfg(feature = "counter")] +fn nested_mergeable_chain_resolves_at_every_hop() { + let d = doc(1); + let profile = d.get_map("state").ensure_mergeable_map("profile").unwrap(); + let rev = profile.ensure_mergeable_counter("revision").unwrap(); + rev.increment(5.0).unwrap(); + d.commit_then_renew(); + + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "profile": { "revision": 5.0 } } }), + "intermediate mergeable map must be visible via its marker" + ); +} + +/// Two peers concurrently create the SAME kind under the same key. Both write the identical +/// marker, so the Map LWW merge is a no-op and both peers' contributions land in the one +/// deterministic cid. +#[test] +fn concurrent_same_kind_creation_merges_via_identical_marker() { + let a = doc(1); + let b = doc(2); + + let a_list = a.get_map("state").ensure_mergeable_list("tags").unwrap(); + a_list.insert(0, "from_a").unwrap(); + a.commit_then_renew(); + + let b_list = b.get_map("state").ensure_mergeable_list("tags").unwrap(); + b_list.insert(0, "from_b").unwrap(); + b.commit_then_renew(); + + sync(&a, &b); + + let va = a.get_deep_value().to_json_value(); + let vb = b.get_deep_value().to_json_value(); + assert_eq!(va, vb, "same-kind concurrent creation must converge"); + + let tags = &va["state"]["tags"]; + assert!(tags.is_array(), "tags must be a list, got {tags:?}"); + let items: Vec<&str> = tags + .as_array() + .unwrap() + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!( + items.contains(&"from_a") && items.contains(&"from_b"), + "both peers' items must be preserved in the merged list; got {items:?}" + ); +} + +/// Kind-change cycle: a key cycles List -> Map -> List. Each `ensure_mergeable_` rewrites the +/// marker. When the marker returns to List, the ORIGINAL List's contents resurface (its container +/// state was preserved by name across the cycle), proving the cycle is non-lossy. +#[test] +fn kind_change_cycle_resurfaces_original_list_contents() { + let d = doc(1); + let root = d.get_map("state"); + + // v1: List with an item. + let list = root.ensure_mergeable_list("field").unwrap(); + list.insert(0, "original").unwrap(); + d.commit_then_renew(); + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "field": ["original"] } }) + ); + + // v2: Map (kind change). The List is now hidden, but its state is preserved by name. + let map = root.ensure_mergeable_map("field").unwrap(); + map.insert("k", 1).unwrap(); + d.commit_then_renew(); + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "field": { "k": 1 } } }), + "Map marker active; List hidden" + ); + + // v3: back to List. The marker returns to List and the ORIGINAL contents resurface. + let list_again = root.ensure_mergeable_list("field").unwrap(); + d.commit_then_renew(); + assert_eq!( + list_again.id(), + list.id(), + "List cid is deterministic across the cycle" + ); + assert_eq!( + d.get_deep_value().to_json_value(), + json!({ "state": { "field": ["original"] } }), + "original List contents resurface after the List -> Map -> List cycle" + ); +} + +/// Forward-compatibility envelope: the marker is stored as a real `LoroValue::Binary` in the +/// parent map's value table, so a client that doesn't understand mergeable containers — for +/// example one that consumes only the shallow JSON value — sees it as an inert binary scalar +/// rather than a fake container edge or a reserved-looking user string. +/// +/// New clients translate that binary marker into a Container value at the `MapHandler` boundary +/// (see `ensure_mergeable_writes_marker_into_parent_slot`); this test inspects the underlying +/// representation via `get_value`, which preserves the wire-level form. +#[test] +fn marker_is_observable_as_raw_binary_for_old_clients() { + let d = doc(1); + let root = d.get_map("state"); + let _list = root.ensure_mergeable_list("tags").unwrap(); + d.commit_then_renew(); + + // `get_value` materializes the parent map as a `LoroValue::Map`, which preserves the raw + // binary marker the way it travels in snapshots and updates to older peers. + let shallow = root.get_value(); + let LoroValue::Map(entries) = shallow else { + panic!("parent value must be a map"); + }; + let raw = entries.get("tags").expect("slot has a marker").clone(); + let LoroValue::Binary(bytes) = &raw else { + panic!("marker must be a binary value at the wire level; got {raw:?}"); + }; + assert_eq!(bytes.len(), 8); + assert!(bytes.starts_with(&MERGEABLE_MARKER_MAGIC)); + assert_eq!( + bytes[MERGEABLE_MARKER_MAGIC.len()], + ContainerType::List.to_u8() + ); + + // The same marker is the inputs to the public mergeable parser. + let expected = mergeable_marker(&root.id(), "tags", ContainerType::List); + assert_eq!(raw, expected); +} diff --git a/vendor/loro-internal/tests/mergeable_container/events_and_paths.rs b/vendor/loro-internal/tests/mergeable_container/events_and_paths.rs new file mode 100644 index 00000000..9abf50e8 --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/events_and_paths.rs @@ -0,0 +1,266 @@ +//! Path resolution, deep value, subscriptions, undo, and has_container. + +#[path = "common.rs"] +mod common; +use common::doc; + +use std::sync::{Arc, Mutex}; + +use loro_internal::{event::Index, HandlerTrait, ToJson}; +use serde_json::json; + +#[test] +#[cfg(feature = "counter")] +fn get_path_returns_logical_parent_path_for_mergeable_child() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + // Exercise the cid path directly; child-side wiring needed to keep + // mutating ops alive arrives in a later commit. + let _ = counter.increment(1.0); + + let path = doc + .get_path_to_container(&counter.id()) + .expect("mergeable counter should have a logical path"); + let indexes = path + .iter() + .map(|(_, index)| index.clone()) + .collect::>(); + assert_eq!( + indexes, + vec![Index::Key("state".into()), Index::Key("revision".into()),], + "mergeable child path should walk logical parent edges, not the synthetic Root name", + ); +} + +#[test] +#[cfg(feature = "counter")] +fn deep_value_nests_mergeable_child_under_parent_and_hides_synthetic_root() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(2.0).unwrap(); + + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "revision": 2.0 } }) + ); +} + +/// `ensure_mergeable_*` writes a marker op into the parent map, which +/// realizes the child immediately (loro-dev/loro#759). So an unmutated +/// mergeable child IS visible in deep value, rendered as its empty default. +/// Because the marker is a real op, every peer that imports it agrees +/// on this view — there is no originating-peer divergence. +#[test] +fn deep_value_shows_unmutated_mergeable_child_as_empty() { + let a = doc(1); + let _child = a.get_map("state").ensure_mergeable_map("nested").unwrap(); + a.commit_then_renew(); + + assert_eq!( + a.get_deep_value().to_json_value(), + json!({ "state": { "nested": {} } }), + "the marker realizes the child; it shows as its empty default", + ); +} + +/// Subscribing to the *parent* map must receive events when one of its +/// mergeable children is mutated: subscriptions on ancestor containers should +/// observe deltas from mergeable descendants, not only subscriptions on the +/// mergeable child itself. +#[test] +#[cfg(feature = "counter")] +fn parent_map_subscription_receives_mergeable_child_events() { + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + + let received: Arc>>> = Arc::new(Mutex::new(Vec::new())); + let received_clone = received.clone(); + let _sub = doc.subscribe( + &root.id(), + Arc::new(move |event| { + let mut g = received_clone.lock().unwrap(); + for container_diff in event.events.iter() { + g.push( + container_diff + .path + .iter() + .map(|(_, idx)| idx.clone()) + .collect::>(), + ); + } + }), + ); + + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + + let captured = received.lock().unwrap(); + assert!( + captured.iter().any(|path| path + .iter() + .any(|idx| matches!(idx, Index::Key(k) if &**k == "revision"))), + "parent map subscriber should see an event whose path includes the mergeable child's key 'revision'; got {captured:?}", + ); +} + +/// A subscription on the mergeable child's cid must receive events when the +/// child is mutated, symmetric to the existing parent-map subscription test. +#[test] +#[cfg(feature = "counter")] +fn mergeable_child_subscription_receives_own_events() { + let doc = doc(1); + let counter = doc + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + + let count: Arc> = Arc::new(Mutex::new(0)); + let count_clone = count.clone(); + let _sub = doc.subscribe( + &counter.id(), + Arc::new(move |_event| { + *count_clone.lock().unwrap() += 1; + }), + ); + + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + counter.increment(2.0).unwrap(); + doc.commit_then_renew(); + + let observed = *count.lock().unwrap(); + assert!( + observed >= 2, + "mergeable-child subscription must fire at least once per commit; got {observed} events" + ); +} + +/// Mutations on a mergeable child must be undoable. Undoing reverts the +/// child's value; the parent marker persists (because it isn't the +/// increment op being undone). +#[test] +#[cfg(feature = "counter")] +fn undo_manager_reverts_mergeable_counter_mutation() { + use loro_internal::UndoManager; + + let doc = doc(1); + let counter = doc + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + let undo = UndoManager::new(&doc); + + counter.increment(5.0).unwrap(); + doc.commit_then_renew(); + assert_eq!(counter.get_value().to_json_value(), json!(5.0)); + + let did_undo = undo.undo().expect("undo must succeed"); + assert!(did_undo, "undo must report it did something"); + + // The mergeable child still exists (registration is not an op), but its + // value is back to zero. + assert_eq!( + counter.get_value().to_json_value(), + json!(0.0), + "undo must revert the increment" + ); +} + +/// Mergeable Roots have a deterministic `ContainerID::Root` in a reserved namespace, +/// but they are conceptually parented to a regular Map. The top-level root enumeration +/// (driven by `DocState::preferred_root_containers`, surfaced through +/// `LoroDoc::get_value`) must NOT include the synthetic mergeable Root — otherwise +/// the doc would expose a top-level key in the `🤝:` namespace alongside the real +/// roots. +/// +/// This guards the `id.is_mergeable()` skip in `preferred_root_containers` against +/// accidental removal: without it, peers would see two top-level entries (the actual +/// parent map plus the synthetic mergeable root) instead of one. +#[test] +#[cfg(feature = "counter")] +fn top_level_root_enumeration_skips_mergeable_roots() { + use loro_common::MERGEABLE_NAMESPACE_PREFIX; + + let doc = doc(1); + let root = doc.get_map("state"); + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + doc.commit_then_renew(); + + // Sanity: the mergeable cid carries the synthetic namespace prefix. + let mergeable_cid = counter.id(); + assert!(mergeable_cid.is_mergeable()); + + // `get_value()` returns a Map keyed by top-level root names. The mergeable cid's + // synthetic Root name must NOT appear here. + let top_level = doc.get_value().to_json_value(); + let map = top_level + .as_object() + .expect("top-level doc value must be a JSON object"); + let keys: Vec<&str> = map.keys().map(|s| s.as_str()).collect(); + assert!( + keys.iter().any(|k| *k == "state"), + "real parent root 'state' must appear in top-level enumeration; got {keys:?}" + ); + assert!( + keys.iter() + .all(|k| !k.starts_with(MERGEABLE_NAMESPACE_PREFIX)), + "no mergeable-namespace Root name must appear at the top level; got {keys:?}" + ); + // Even more strictly: the synthetic root's exact name must not be a top-level key. + let synthetic_name = match &mergeable_cid { + loro_common::ContainerID::Root { name, .. } => name.to_string(), + _ => unreachable!("mergeable cid is always a Root"), + }; + assert!( + !keys.iter().any(|k| *k == synthetic_name), + "synthetic mergeable root name {synthetic_name:?} must not be in top-level keys {keys:?}" + ); + + // Deep value must show the mergeable child nested under its logical parent, not as a + // sibling top-level entry. + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "revision": 1.0 } }), + "mergeable child must be nested under its logical parent, not surfaced at the top level" + ); +} + +/// `LoroDoc::has_container(cid)` must return `false` for a mergeable cid +/// that has never been written to, and `true` after the child has been +/// mutated. Mergeable existence depends on state, not on the name shape, so +/// the short-circuit for plain `Root` ids must skip mergeable namespace cids. +#[test] +#[cfg(feature = "counter")] +fn has_container_reports_false_for_unwritten_mergeable_child() { + use loro_common::{ContainerID, ContainerType}; + + let doc = doc(1); + let root = doc.get_map("state"); + + // Build the deterministic mergeable cid by hand, WITHOUT calling + // ensure_mergeable_counter (which would write the marker). + let parent_id = root.id(); + let unwritten_cid = ContainerID::new_mergeable(&parent_id, "revision", ContainerType::Counter); + assert!(unwritten_cid.is_mergeable()); + assert!( + !doc.has_container(&unwritten_cid), + "has_container must report false for an unwritten mergeable cid" + ); + + // Now actually create and mutate the child. has_container must flip to true. + let counter = root.ensure_mergeable_counter("revision").unwrap(); + counter.increment(1.0).unwrap(); + assert_eq!(counter.id(), unwritten_cid, "cid is deterministic"); + assert!( + doc.has_container(&unwritten_cid), + "has_container must report true after the mergeable child has state" + ); + + // Regular root containers still report true as before (regression guard). + let regular_root = ContainerID::new_root("state", ContainerType::Map); + assert!(doc.has_container(®ular_root)); +} diff --git a/vendor/loro-internal/tests/mergeable_container/snapshot.rs b/vendor/loro-internal/tests/mergeable_container/snapshot.rs new file mode 100644 index 00000000..f45a5228 --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/snapshot.rs @@ -0,0 +1,359 @@ +//! Snapshot and update-import round-trips, including LWW recovery. + +#[path = "common.rs"] +mod common; +use common::{doc, sync}; + +use loro_internal::{cursor::PosType, event::Index, loro::ExportMode, HandlerTrait, ToJson}; +use serde_json::json; + +/// Snapshot round-trip must preserve the parent edges (logical path) and +/// state values for mergeable child containers nested inside other mergeable +/// child containers. +/// +/// Source peer creates `state` → mergeable map `profile` → mergeable counter +/// `revision`, mutates each, then exports a snapshot. Importing into a fresh +/// peer must reproduce the same deep value and the same logical path for +/// the counter. +#[test] +#[cfg(feature = "counter")] +fn snapshot_roundtrip_preserves_mergeable_parent_edges_and_values() { + let source = doc(1); + let root = source.get_map("state"); + let nested = root.ensure_mergeable_map("profile").unwrap(); + nested.insert("name", "Ada").unwrap(); + let counter = nested.ensure_mergeable_counter("revision").unwrap(); + counter.increment(3.0).unwrap(); + let counter_id = counter.id(); + + let snapshot = source.export(ExportMode::Snapshot).unwrap(); + let imported = doc(2); + imported.import(&snapshot).unwrap(); + + let path = imported + .get_path_to_container(&counter_id) + .expect("mergeable counter must have a logical path after snapshot import"); + let indexes = path + .iter() + .map(|(_, index)| index.clone()) + .collect::>(); + assert_eq!( + indexes, + vec![ + Index::Key("state".into()), + Index::Key("profile".into()), + Index::Key("revision".into()), + ], + "imported counter should walk logical parent edges across two mergeable hops" + ); + + assert_eq!( + imported.get_deep_value().to_json_value(), + source.get_deep_value().to_json_value(), + "deep value of the imported doc must match the source after snapshot round-trip" + ); + + let imported_counter = imported + .get_map("state") + .ensure_mergeable_map("profile") + .unwrap() + .ensure_mergeable_counter("revision") + .unwrap(); + assert_eq!(imported_counter.id(), counter_id); +} + +/// Peer B imports updates that originated from peer A's `ensure_mergeable_counter` +/// + `increment` calls, but peer B never locally called `ensure_mergeable_*`. +/// After import, peer B's deep value, container enumeration, and path +/// resolution for the mergeable child must all reflect the imported state. +#[test] +#[cfg(feature = "counter")] +fn update_import_resolves_mergeable_logical_edge_on_receiver() { + let a = doc(1); + let b = doc(2); + + let a_counter = a + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + a_counter.increment(5.0).unwrap(); + let a_counter_id = a_counter.id(); + a.commit_then_renew(); + + // Peer B imports A's updates WITHOUT first calling ensure_mergeable_counter. + let updates = a.export(ExportMode::updates(&b.oplog_vv())).unwrap(); + b.import(&updates).unwrap(); + + // Path resolution must work directly from the deterministic cid. Calling the getter first + // would rewrite the marker and could hide resolver bugs after import. + let path = b.get_path_to_container(&a_counter_id).expect("path"); + let indexes = path.iter().map(|(_, idx)| idx.clone()).collect::>(); + assert_eq!( + indexes, + vec![Index::Key("state".into()), Index::Key("revision".into())] + ); + + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "revision": 5.0 } }), + "after update import, peer B's deep value must include the mergeable child" + ); + + // Peer B then locally resolves the mergeable handler — this must return + // the same cid as the one peer A wrote, and the existing value. + let b_counter = b + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + assert_eq!(b_counter.id(), a_counter_id); + assert_eq!(b_counter.get_value().to_json_value(), json!(5.0)); +} + +/// Create a mergeable counter but never mutate it, then export a snapshot. +/// `ensure_mergeable_*` writes a marker into the parent map's value table, +/// which is ordinary map state and rides through the snapshot like any other +/// value (loro-dev/loro#759). So the receiving peer resolves the marker +/// to the same deterministic cid and sees the child as its empty default — no +/// special recovery is needed. +#[test] +#[cfg(feature = "counter")] +fn unmutated_mergeable_child_survives_snapshot_via_marker() { + let a = doc(1); + let counter = a + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + let counter_id = counter.id(); + // Deliberately no increment. Commit anyway so any pending state is flushed. + a.commit_then_renew(); + + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + let b = doc(2); + b.import(&snapshot).unwrap(); + + let has_container_before_path = b.has_container(&counter_id); + let path = b + .get_path_to_container(&counter_id) + .expect("unmutated mergeable child must resolve from marker"); + let indexes = path.iter().map(|(_, idx)| idx.clone()).collect::>(); + assert_eq!( + indexes, + vec![Index::Key("state".into()), Index::Key("revision".into())] + ); + assert_eq!( + b.has_container(&counter_id), + has_container_before_path, + "path lookup must not change has_container semantics for mergeable cids" + ); + + // The binary marker rides through the snapshot as a normal map value, so B + // resolves the same deterministic cid and renders it as an empty counter. + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "revision": 0.0 } }), + "marker survives snapshot round-trip; child resolves to its empty default", + ); + + let b_counter = b + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + assert_eq!(b_counter.id(), counter_id, "cid still deterministic"); + assert_eq!(b_counter.get_value().to_json_value(), json!(0.0)); +} + +/// Snapshot import where both peers registered the same `(key, kind)`: +/// deterministic cids match, recovery walk converges, content from peer A +/// wins through normal CRDT merge. +#[test] +fn snapshot_import_same_type_collision_converges() { + let a = doc(1); + let a_text = a.get_map("state").ensure_mergeable_text("notes").unwrap(); + a_text.insert(0, "A", PosType::Unicode).unwrap(); + a.commit_then_renew(); + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + + let b = doc(2); + let b_text = b.get_map("state").ensure_mergeable_text("notes").unwrap(); + b_text.insert(0, "B", PosType::Unicode).unwrap(); + b.commit_then_renew(); + assert_eq!(a_text.id(), b_text.id(), "cids must match before import"); + + b.import(&snapshot).unwrap(); + + // Sync back so A sees both. + sync(&a, &b); + let value = a.get_deep_value().to_json_value(); + assert!( + value == json!({ "state": { "notes": "AB" } }) + || value == json!({ "state": { "notes": "BA" } }), + "both edits must survive on same-type collision; got {value}" + ); + assert_eq!(b.get_deep_value().to_json_value(), value); +} + +/// Snapshot import where the LOCAL peer registered a different kind for the same key than the +/// SNAPSHOT peer. The two binary markers compete in the parent map's slot for "k"; the parent +/// map's regular LWW deterministically resolves to one, so exactly one kind is visible. Both +/// containers' states are preserved; the loser is reachable by explicit lookup. +#[test] +fn snapshot_import_different_type_collision_resolves_by_lww() { + let a = doc(1); + let a_text = a.get_map("state").ensure_mergeable_text("k").unwrap(); + a_text.insert(0, "hello", PosType::Unicode).unwrap(); + a.commit_then_renew(); + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + + let b = doc(2); + let b_map = b.get_map("state").ensure_mergeable_map("k").unwrap(); + b_map.insert("flag", true).unwrap(); + b.commit_then_renew(); + assert_ne!( + a_text.id(), + b_map.id(), + "different kinds under the same key MUST produce different cids" + ); + + b.import(&snapshot).expect("import must not fail"); + + // Exactly one kind is visible in deep value (whichever marker won the Map LWW). + let value = b.get_deep_value().to_json_value(); + let k = &value["state"]["k"]; + let visible_kinds = [k.is_string(), k.is_object()]; + assert_eq!( + visible_kinds.iter().filter(|x| **x).count(), + 1, + "exactly one kind must be visible after LWW; got {k:?}" + ); + + // Both getters still succeed (each rewrites the marker to its kind). Neither errors — + // requesting a kind is a kind change, not a conflict. + let _ = b.get_map("state").ensure_mergeable_text("k").unwrap(); + let _ = b.get_map("state").ensure_mergeable_map("k").unwrap(); +} + +/// After snapshot import resolves `"k"` to a Text marker on the receiver, a local +/// `ensure_mergeable_map("k")` does NOT error — it rewrites the marker to Map (a deliberate +/// kind change). The Text container stays reachable by name; requesting it again rewrites the +/// marker back to Text and resurfaces its preserved contents. +#[test] +fn different_kind_request_after_snapshot_is_a_kind_change() { + let a = doc(1); + let a_text = a.get_map("state").ensure_mergeable_text("k").unwrap(); + a_text.insert(0, "x", PosType::Unicode).unwrap(); + a.commit_then_renew(); + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + + let b = doc(2); + b.import(&snapshot).unwrap(); + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "k": "x" } }), + "imported Text marker resolves to its content" + ); + + // Requesting a Map rewrites the marker to Map; no error. + let b_map = b.get_map("state").ensure_mergeable_map("k").unwrap(); + b_map.insert("flag", true).unwrap(); + b.commit_then_renew(); + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "k": { "flag": true } } }), + "different-kind request is a kind change, not an error" + ); + + // The Text is still reachable; requesting it again resurfaces its preserved content. + let b_text = b.get_map("state").ensure_mergeable_text("k").unwrap(); + assert_eq!(b_text.id(), a_text.id()); + b.commit_then_renew(); + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "k": "x" } }), + "re-requesting Text rewrites the marker back and resurfaces preserved content" + ); +} + +/// Shallow snapshot export must also preserve the LOSING kind's state in a concurrent-kind +/// conflict. The loser is not reachable through the parent's winning marker — its only +/// addressability is via the deterministic mergeable cid. The snapshot's alive-container walk has +/// to include mergeable cids as retention roots, otherwise `retain_keys` would silently GC the +/// loser, and a later `ensure_mergeable_` on the receiver would resurface an empty +/// child instead of preserved contents. +#[test] +fn shallow_snapshot_preserves_losing_kind_state() { + let a = doc(1); + let a_text = a.get_map("state").ensure_mergeable_text("k").unwrap(); + a_text.insert(0, "from_a", PosType::Unicode).unwrap(); + let a_text_id = a_text.id(); + a.commit_then_renew(); + + // Advance A's clock past A's text marker, then rewrite the marker to Map (a kind change). + // The Map marker now wins LWW on A and is the visible kind; the Text child becomes the loser + // whose state lives only at its deterministic cid. + for i in 0..6 { + a.get_map("state") + .insert(&format!("noise_{i}"), i) + .unwrap(); + a.commit_then_renew(); + } + let a_map = a.get_map("state").ensure_mergeable_map("k").unwrap(); + a_map.insert("from_a_map", true).unwrap(); + a.commit_then_renew(); + + let frontiers = a.state_frontiers(); + let snapshot = a + .export(ExportMode::ShallowSnapshot(std::borrow::Cow::Owned( + frontiers, + ))) + .unwrap(); + + let b = doc(2); + b.import(&snapshot).unwrap(); + + // Sanity: Map marker won on B; Text is not the visible kind under "k". + let value = b.get_deep_value().to_json_value(); + assert!( + value["state"]["k"].is_object(), + "Map marker wins on B; got {value}" + ); + + // The loser's Text state must have ridden through the shallow snapshot. Re-requesting it + // rewrites the marker to Text and resurfaces the preserved "from_a" content. + let b_text = b.get_map("state").ensure_mergeable_text("k").unwrap(); + assert_eq!(b_text.id(), a_text_id, "deterministic cid is stable"); + assert_eq!( + b_text.get_value().to_json_value(), + json!("from_a"), + "losing-kind state must survive shallow snapshot export" + ); +} + +/// Shallow snapshot export should preserve mergeable child state and parent +/// edges on the receiver, the same as a full snapshot. +#[test] +#[cfg(feature = "counter")] +fn shallow_snapshot_roundtrip_preserves_mergeable_child() { + let a = doc(1); + let counter = a + .get_map("state") + .ensure_mergeable_counter("revision") + .unwrap(); + counter.increment(4.0).unwrap(); + a.commit_then_renew(); + + // ShallowSnapshot at current frontiers. + let frontiers = a.state_frontiers(); + let snapshot = a + .export(ExportMode::ShallowSnapshot(std::borrow::Cow::Owned( + frontiers, + ))) + .unwrap(); + + let b = doc(2); + b.import(&snapshot).unwrap(); + assert_eq!( + b.get_deep_value().to_json_value(), + json!({ "state": { "revision": 4.0 } }), + "shallow snapshot must carry mergeable child state and logical edge resolution" + ); +} diff --git a/vendor/loro-internal/tests/mergeable_container/type_conflict.rs b/vendor/loro-internal/tests/mergeable_container/type_conflict.rs new file mode 100644 index 00000000..e0091f3d --- /dev/null +++ b/vendor/loro-internal/tests/mergeable_container/type_conflict.rs @@ -0,0 +1,182 @@ +//! Concurrent different-kind creation, resolved by the parent map's marker LWW. +//! +//! When two peers create different kinds under the same key, each writes a different binary marker +//! into the parent map's slot. The parent map's regular LWW deterministically resolves to one +//! marker, so every peer surfaces the same kind; the loser's container stays reachable only by an +//! explicit `ensure_mergeable_` lookup (which rewrites the marker). +//! See loro-dev/loro#759. + +#[path = "common.rs"] +mod common; +use common::{doc, sync}; + +use loro_internal::{cursor::PosType, event::Index, loro::ExportMode, HandlerTrait, ToJson}; +use serde_json::json; + +/// Requesting a different kind under a key that already holds another kind's marker is a +/// deliberate, local kind change: it rewrites the marker. The new kind becomes active and +/// the new handler is usable. (This is the building block of the List -> Map -> List cycle.) +#[test] +fn local_different_kind_request_rewrites_the_marker() { + let doc = doc(1); + let root = doc.get_map("state"); + + let text = root.ensure_mergeable_text("field").unwrap(); + text.insert(0, "hello", PosType::Unicode).unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "field": "hello" } }) + ); + + // Switch the kind: this rewrites the marker to Map. + let map = root.ensure_mergeable_map("field").unwrap(); + map.insert("k", 1).unwrap(); + doc.commit_then_renew(); + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ "state": { "field": { "k": 1 } } }), + "requesting a different kind rewrites the marker to that kind" + ); +} + +/// When two competing-kind markers land in the same parent's slot under the same key during +/// import, the parent map's regular LWW deterministically picks one. The visible kind is the same +/// on every peer; the loser's container is still reachable by name but not surfaced by default. +#[test] +#[cfg(feature = "counter")] +fn different_kind_collision_resolves_by_map_lww_at_import() { + // Peer A: text under "k", low lamport. + let a = doc(1); + let a_text = a.get_map("state").ensure_mergeable_text("k").unwrap(); + a_text.insert(0, "from_a", PosType::Unicode).unwrap(); + let a_text_id = a_text.id(); + a.commit_then_renew(); + + // Peer B: map under "k", with a later lamport (advance B's clock first). + let b = doc(2); + let b_state = b.get_map("state"); + for i in 0..5 { + b_state.insert(&format!("filler_{i}"), i).unwrap(); + b.commit_then_renew(); + } + let b_map = b_state.ensure_mergeable_map("k").unwrap(); + b_map.insert("from_b", true).unwrap(); + let b_map_id = b_map.id(); + b.commit_then_renew(); + + // B imports A's snapshot. The Text and Map markers compete in B's map slot for "k"; B's Map + // marker has the higher lamport, so Map wins. + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + b.import(&snapshot).unwrap(); + + // Check the direct cids before calling any getter. The losing Text cid must not have a logical + // path while the Map marker wins; the winning Map cid resolves from the marker. + assert!( + b.get_path_to_container(&a_text_id).is_none(), + "old Text cid must be inactive while the Map marker wins" + ); + let map_path = b + .get_path_to_container(&b_map_id) + .expect("winning Map cid must resolve from the parent marker"); + let indexes = map_path + .iter() + .map(|(_, index)| index.clone()) + .collect::>(); + assert_eq!( + indexes, + vec![Index::Key("state".into()), Index::Key("k".into())] + ); + + let value = b.get_deep_value().to_json_value(); + let k_value = &value["state"]["k"]; + assert!( + k_value.is_object(), + "Map marker won the LWW; got {k_value:?}" + ); + assert_eq!(k_value["from_b"], json!(true)); + + // The loser's Text container is still reachable by an explicit ensure_mergeable_text — which + // rewrites the marker back to Text (a local kind change), so it now resurfaces. + let b_text = b.get_map("state").ensure_mergeable_text("k").unwrap(); + assert_eq!(b_text.id(), a_text_id); + assert_eq!( + b_text.get_value().to_json_value(), + json!("from_a"), + "loser's container is preserved and reachable by name" + ); +} + +/// Three peers each create a different kind under the same key and mutate it once. After a full +/// round-robin sync, every peer's parent-map LWW resolves to the same marker, so all agree +/// on exactly one visible kind. +#[test] +#[cfg(feature = "counter")] +fn three_peer_different_kind_conflict_converges() { + let a = doc(1); + let b = doc(2); + let c = doc(3); + + let a_text = a.get_map("state").ensure_mergeable_text("k").unwrap(); + a_text.insert(0, "from_a", PosType::Unicode).unwrap(); + a.commit_then_renew(); + + let b_map = b.get_map("state").ensure_mergeable_map("k").unwrap(); + b_map.insert("from_b", true).unwrap(); + b.commit_then_renew(); + + let c_list = c.get_map("state").ensure_mergeable_list("k").unwrap(); + c_list.insert(0, "from_c").unwrap(); + c.commit_then_renew(); + + sync(&a, &b); + sync(&b, &c); + sync(&a, &c); + sync(&a, &b); + + let va = a.get_deep_value().to_json_value(); + let vb = b.get_deep_value().to_json_value(); + let vc = c.get_deep_value().to_json_value(); + assert_eq!(va, vb, "A and B must agree"); + assert_eq!(vb, vc, "B and C must agree"); + + // Exactly one kind is visible under "k": Text -> string, Map -> object, List -> array. + let k = &va["state"]["k"]; + let survivors = [k.is_string(), k.is_object(), k.is_array()]; + let count: usize = survivors.iter().filter(|x| **x).count(); + assert_eq!( + count, 1, + "exactly one kind must be visible; got {survivors:?} for value {k:?}" + ); +} + +/// Concurrent SAME-kind creation: both peers write the identical marker, so the Map LWW +/// merge is a no-op and both contributions land in the one deterministic cid. +#[test] +fn concurrent_same_kind_creation_converges() { + let a = doc(1); + let b = doc(2); + + let a_list = a.get_map("state").ensure_mergeable_list("k").unwrap(); + a_list.insert(0, "from_a").unwrap(); + a.commit_then_renew(); + + let b_list = b.get_map("state").ensure_mergeable_list("k").unwrap(); + b_list.insert(0, "from_b").unwrap(); + b.commit_then_renew(); + + sync(&a, &b); + + let va = a.get_deep_value().to_json_value(); + assert_eq!(va, b.get_deep_value().to_json_value(), "peers converge"); + let items: Vec<&str> = va["state"]["k"] + .as_array() + .expect("k is a list") + .iter() + .map(|v| v.as_str().unwrap()) + .collect(); + assert!( + items.contains(&"from_a") && items.contains(&"from_b"), + "both contributions preserved; got {items:?}" + ); +} diff --git a/vendor/loro-internal/tests/richtext.md b/vendor/loro-internal/tests/richtext.md new file mode 100644 index 00000000..49b575d6 --- /dev/null +++ b/vendor/loro-internal/tests/richtext.md @@ -0,0 +1,98 @@ +# Richtext Test Cases + +#### 0. Insert into bold span + +| Name | Text | +|:----------------|:-------------------------| +| Origin | `Hello World` | +| Concurrent A | `Hello World` | +| Concurrent B | `Hello New World` | +| Expected Result | `Hello New World` | + +#### 1. Merge Concurrent Styles + +| Name | Text | +|:----------------|:---------------------| +| Origin | Hello World | +| Concurrent A | `Hello World` | +| Concurrent B | `Hello World` | +| Expected Result | `Hello World` | + +#### 2. Concurrent insert text & remove style + +| Name | Text | +|:----------------|:-----------------------| +| Origin | `Hello World` | +| Concurrent A | `Hello World` | +| Concurrent B | `Hello a World` | +| Expected Result | `Hello a World` | + +#### 3. Concurrent insert text & style + +| Name | Text | +|:----------------|:-----------------------| +| Origin | `Hello World` | +| Concurrent A | `Hello World` | +| Concurrent B | `Hello a World` | +| Expected Result | `Hello a World` | + +#### 4. Concurrent text edit & style that shrink + +| Name | Text | +|:----------------|:---------------------------| +| Origin | `Hello World` | +| Concurrent A | `Hello World` | +| Concurrent B | `Hey World` | +| Expected Result | `Hey World` | + +#### 5. Local insertion expand rules + +> [**Hello**](https://www.google.com) World + +When insert a new character after "Hello", the new char should be bold but not link + +> [**Hello**](https://www.google.com)**t** World + + +| Name | Text | +|:----------------|:----------------------------------| +| Origin | `Hello World` | +| Expected Result | `Hellot World` | + + +#### 6. Concurrent unbold + +In Peritext paper 2.3.2 + +| Name | Text | +|:----------------|:---------------------------------------------| +| Origin | `The fox jumped over the dog.` | +| Concurrent A | `The fox jumped over the dog.` | +| Concurrent B | `The fox jumped over the dog.` | +| Expected Result | `The fox jumped over the dog.` | + +#### 7. Bold & Unbold + +In Peritext paper 2.3.3 + +| Name | Text | +|:----------------|:---------------------------------------------| +| Origin | `The fox jumped over the dog.` | +| Concurrent A | `The fox jumped over the dog.` | +| Concurrent B | `The fox jumped over the dog.` | +| Expected Result | `The fox jumped over the dog.` | + +#### 8. Overlapped formatting + +In Peritext paper 3.2, example 3 + +| Name | Text | +|:----------------|:-----------------------------| +| Origin | The fox jumped. | +| Concurrent A | **The fox** jumped. | +| Concurrent B | The *fox jumped*. | +| Expected Result | **The _fox_** jumped. | + +#### 9. Multiple instances of the same mark + +![](https://i.postimg.cc/MTNGq8cH/Clean-Shot-2023-10-09-at-12-16-29-2x.png) diff --git a/vendor/loro-internal/tests/richtext.rs b/vendor/loro-internal/tests/richtext.rs new file mode 100644 index 00000000..553c8e59 --- /dev/null +++ b/vendor/loro-internal/tests/richtext.rs @@ -0,0 +1,363 @@ +//! All the tests in this file are based on [richtext.md] + +use std::ops::Range; + +use loro_common::LoroValue; +use loro_internal::{cursor::PosType, loro::ExportMode, LoroDoc, ToJson}; +use serde_json::json; + +fn init(s: &str) -> LoroDoc { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let richtext = doc.get_text("r"); + richtext.insert(0, s, PosType::Unicode).unwrap(); + doc +} + +fn clone(doc: &LoroDoc, peer_id: u64) -> LoroDoc { + let doc2 = LoroDoc::new_auto_commit(); + doc2.set_peer_id(peer_id).unwrap(); + doc2.import(&doc.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + doc2 +} + +#[derive(Debug)] +enum Kind { + Bold, + Italic, + Link, +} + +impl Kind { + fn key(&self) -> &str { + match self { + Kind::Bold => "bold", + Kind::Link => "link", + Kind::Italic => "italic", + } + } +} + +fn insert(doc: &LoroDoc, pos: usize, s: &str) { + let richtext = doc.get_text("r"); + richtext.insert(pos, s, PosType::Unicode).unwrap(); +} + +fn delete(doc: &LoroDoc, pos: usize, len: usize) { + let richtext = doc.get_text("r"); + richtext.delete(pos, len, PosType::Unicode).unwrap(); +} + +fn mark(doc: &LoroDoc, range: Range, kind: Kind) { + let richtext = doc.get_text("r"); + richtext + .mark( + range.start, + range.end, + kind.key(), + true.into(), + PosType::Event, + ) + .unwrap(); +} + +fn unmark(doc: &LoroDoc, range: Range, kind: Kind) { + let richtext = doc.get_text("r"); + richtext + .mark( + range.start, + range.end, + kind.key(), + false.into(), + PosType::Event, + ) + .unwrap(); +} + +fn mark_kv(doc: &LoroDoc, range: Range, key: &str, value: impl Into) { + let richtext = doc.get_text("r"); + richtext + .mark(range.start, range.end, key, value.into(), PosType::Event) + .unwrap(); +} + +fn merge(a: &LoroDoc, b: &LoroDoc) { + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + b.import(&a.export(ExportMode::updates(&b.oplog_vv())).unwrap()) + .unwrap(); +} + +fn expect_result(doc: &LoroDoc, json: serde_json::Value) { + let richtext = doc.get_text("r"); + let s = richtext.get_richtext_value().to_json_value(); + assert_eq!( + &s, + &json, + "expect: {}, got: {}", + serde_json::to_string_pretty(&json).unwrap(), + serde_json::to_string_pretty(&s).unwrap() + ); +} + +#[test] +fn case0() { + let doc_a = init("Hello World"); + let doc_b = clone(&doc_a, 2); + mark(&doc_a, 0..11, Kind::Bold); + insert(&doc_b, 6, "New "); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + json!([{"insert":"Hello New World","attributes":{"bold":true}}]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +#[test] +fn case1() { + let doc_a = init("Hello World"); + let doc_b = clone(&doc_a, 2); + mark(&doc_a, 0..5, Kind::Bold); + mark(&doc_b, 3..11, Kind::Bold); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + json!([{"insert":"Hello World","attributes":{"bold":true}}]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +#[test] +fn case2() { + let doc_a = init("Hello World"); + mark(&doc_a, 0..11, Kind::Bold); + let doc_b = clone(&doc_a, 2); + unmark(&doc_a, 0..6, Kind::Bold); + insert(&doc_b, 5, " a"); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + json!([{"insert":"Hello a ","attributes":{"bold":false}},{"insert":"World","attributes":{"bold":true}}]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +/// | Name | Text | +/// |:----------------|:-----------------------| +/// | Origin | `Hello World` | +/// | Concurrent A | `Hello World` | +/// | Concurrent B | `Hello a World` | +/// | Expected Result | `Hello a World` | +/// +#[test] +fn case3() { + let doc_a = init("Hello World"); + let doc_b = clone(&doc_a, 2); + mark(&doc_a, 6..11, Kind::Bold); + insert(&doc_b, 5, " a"); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + json!([{"insert":"Hello a "},{"insert":"World","attributes":{"bold":true}}]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +/// | Name | Text | +/// |:----------------|:---------------------------| +/// | Origin | `Hello World` | +/// | Concurrent A | `Hello World` | +/// | Concurrent B | `Hey World` | +/// | Expected Result | `Hey World` | +#[test] +fn case4() { + let doc_a = init("Hello World"); + let doc_b = clone(&doc_a, 2); + mark(&doc_a, 0..5, Kind::Link); + delete(&doc_b, 2, 3); + expect_result(&doc_b, json!([{"insert":"He World"}])); + insert(&doc_b, 2, "y"); + expect_result(&doc_b, json!([{"insert":"Hey World"}])); + merge(&doc_a, &doc_b); + expect_result( + &doc_b, + json!([{"insert":"Hey","attributes":{"link":true}},{"insert":" World"}]), + ); + expect_result( + &doc_a, + json!([{"insert":"Hey","attributes":{"link":true}},{"insert":" World"}]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +/// When insert a new character after "Hello", the new char should be bold but not link +/// +/// | Name | Text | +/// |:----------------|:----------------------------------| +/// | Origin | `Hello World` | +/// | Expected Result | `Hellot World` | +#[test] +fn case5() { + let doc = init("Hello World"); + mark(&doc, 0..5, Kind::Bold); + expect_result( + &doc, + serde_json::json!([ + {"insert": "Hello", "attributes": {"bold": true}}, + {"insert": " World"} + ]), + ); + mark(&doc, 0..5, Kind::Link); + expect_result( + &doc, + serde_json::json!([ + {"insert": "Hello", "attributes": {"bold": true, "link": true}}, + {"insert": " World"} + ]), + ); + insert(&doc, 5, "t"); + expect_result( + &doc, + serde_json::json!([ + {"insert": "Hello", "attributes": {"bold": true, "link": true}}, + {"insert": "t", "attributes": {"bold": true}}, + {"insert": " World"} + ]), + ); + doc.check_state_diff_calc_consistency_slow(); +} + +/// +/// | Name | Text | +/// |:----------------|:---------------------------------------------| +/// | Origin | `The fox jumped over the dog.` | +/// | Concurrent A | `The fox jumped over the dog.` | +/// | Concurrent B | `The fox jumped over the dog.` | +/// | Expected Result | `The fox jumped over the dog.` | +#[test] +fn case6() { + let doc_a = init("The fox jumped over the dog."); + mark(&doc_a, 0..3, Kind::Bold); + let doc_b = clone(&doc_a, 2); + unmark(&doc_a, 0..3, Kind::Bold); + unmark(&doc_b, 4..7, Kind::Bold); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + json!([ + {"insert":"The", "attributes": {"bold": false}}, + {"insert":" ",}, + {"insert":"fox", "attributes": {"bold": false}}, + {"insert":" jumped over the dog."} + ]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +/// | Name | Text | +/// |:----------------|:---------------------------------------------| +/// | Origin | `The fox jumped over the dog.` | +/// | Concurrent A | `The fox jumped over the dog.` | +/// | Concurrent B | `The fox jumped over the dog.` | +/// | Expected Result | `The fox jumped over the dog.` | +#[test] +fn case7() { + let doc_a = init("The fox jumped over the dog."); + mark(&doc_a, 0..14, Kind::Bold); + let doc_b = clone(&doc_a, 2); + unmark(&doc_a, 7..14, Kind::Bold); + unmark(&doc_b, 3..14, Kind::Bold); + mark(&doc_b, 24..27, Kind::Bold); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + serde_json::json!([ + {"insert": "The", "attributes": {"bold": true}}, + {"insert": " fox jumped", "attributes": {"bold": false}}, + {"insert": " over the "}, + {"insert": "dog", "attributes": {"bold": true}}, + {"insert": "."} + ]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +/// | Name | Text | +/// |:----------------|:-----------------------------| +/// | Origin | The fox jumped. | +/// | Concurrent A | **The fox** jumped. | +/// | Concurrent B | The *fox jumped*. | +/// | Expected Result | **The _fox_** jumped. | +#[test] +fn case8() { + let doc_a = init("The fox jumped."); + let doc_b = clone(&doc_a, 2); + mark(&doc_a, 0..7, Kind::Bold); + mark(&doc_a, 4..14, Kind::Italic); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + serde_json::json!([ + {"insert": "The ", "attributes": {"bold": true}}, + {"insert": "fox", "attributes": {"bold": true, "italic": true}}, + {"insert": " jumped", "attributes": {"italic": true}}, + {"insert": "."} + ]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} + +/// ![](https://i.postimg.cc/MTNGq8cH/Clean-Shot-2023-10-09-at-12-16-29-2x.png) +#[test] +fn case9() { + let doc_a = init("The fox jumped."); + let doc_b = clone(&doc_a, 2); + mark_kv(&doc_a, 0..7, "comment:alice", "alice comment"); + mark_kv(&doc_a, 4..14, "comment:bob", "bob comment"); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + serde_json::json!([ + {"insert": "The ", "attributes": {"comment:alice": "alice comment"}}, + {"insert": "fox", "attributes": {"comment:alice": "alice comment", "comment:bob": "bob comment"}}, + {"insert": " jumped", "attributes": {"comment:bob": "bob comment"}}, + {"insert": "."} + ]), + ); +} + +#[test] +fn insert_after_link() { + let doc_a = init("The fox jumped."); + let doc_b = clone(&doc_a, 2); + mark(&doc_a, 0..3, Kind::Link); + merge(&doc_a, &doc_b); + insert(&doc_a, 3, "a"); + merge(&doc_a, &doc_b); + expect_result( + &doc_a, + serde_json::json!([ + {"insert": "The", "attributes": {"link": true}}, + {"insert": "a fox jumped."}, + ]), + ); + expect_result( + &doc_b, + serde_json::json!([ + {"insert": "The", "attributes": {"link": true}}, + {"insert": "a fox jumped."}, + ]), + ); + doc_a.check_state_diff_calc_consistency_slow(); + doc_b.check_state_diff_calc_consistency_slow(); +} diff --git a/vendor/loro-internal/tests/test.rs b/vendor/loro-internal/tests/test.rs new file mode 100644 index 00000000..8cd8809b --- /dev/null +++ b/vendor/loro-internal/tests/test.rs @@ -0,0 +1,1542 @@ +use loro_common::{ + loro_value, ContainerID, ContainerType, IdSpan, LoroError, LoroResult, LoroValue, PeerID, ID, +}; +use loro_internal::awareness::EphemeralStore; +use loro_internal::sync::{AtomicBool, Mutex}; +use loro_internal::{ + cursor::PosType, + delta::ResolvedMapValue, + encoding::ImportStatus, + event::{Diff, EventTriggerKind}, + fx_map, + handler::{Handler, TextDelta, ValueOrHandler}, + loro::{CommitOptions, ExportMode}, + version::{Frontiers, VersionRange}, + ApplyDiff, HandlerTrait, ListHandler, LoroDoc, MapHandler, TextHandler, ToJson, TreeHandler, + TreeParentId, +}; +use rustc_hash::FxHashMap; +use serde_json::json; +use std::sync::Arc; + +#[test] +fn issue_502() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + doc.get_map("map").insert("stringA", "Original data")?; + doc.commit_then_renew(); + doc.get_map("map").insert("stringA", "Updated data")?; + doc.attach(); + doc.get_map("map").insert("stringB", "Something else")?; + doc.commit_then_renew(); + Ok(()) +} + +#[test] +fn issue_225() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "123")?; + text.mark(0, 3, "bold", true.into(), PosType::Event)?; + // when apply_delta, the attributes of insert should override the current styles + text.apply_delta(&[ + TextDelta::Retain { + retain: 3, + attributes: None, + }, + TextDelta::Insert { + insert: "new".into(), + attributes: None, + }, + ])?; + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{ "insert": "123", "attributes": { "bold": true } }, { "insert": "new" }]) + ); + + Ok(()) +} + +#[test] +fn issue_211() -> LoroResult<()> { + let doc1 = LoroDoc::new_auto_commit(); + let doc2 = LoroDoc::new_auto_commit(); + doc1.get_text("text").insert_unicode(0, "T")?; + doc2.merge(&doc1)?; + let v0 = doc1.oplog_frontiers(); + doc1.get_text("text").insert_unicode(1, "A")?; + doc2.get_text("text").insert_unicode(1, "B")?; + doc1.checkout(&v0)?; + doc2.checkout(&v0)?; + doc1.checkout_to_latest(); + doc2.checkout_to_latest(); + // let v1_of_doc1 = doc1.oplog_frontiers(); + let v1_of_doc2 = doc2.oplog_frontiers(); + doc2.get_text("text").insert_unicode(2, "B")?; + doc2.checkout(&v1_of_doc2)?; + doc2.checkout(&v0)?; + assert_eq!( + doc2.get_deep_value().to_json_value(), + json!({ + "text": "T" + }) + ); + Ok(()) +} + +#[test] +fn mark_with_the_same_key_value_should_be_skipped() { + let a = LoroDoc::new_auto_commit(); + let text = a.get_text("text"); + text.insert_unicode(0, "Hello world!").unwrap(); + text.mark(0, 11, "bold", "value".into(), PosType::Event) + .unwrap(); + a.commit_then_renew(); + let v = a.oplog_vv(); + text.mark(0, 5, "bold", "value".into(), PosType::Event) + .unwrap(); + a.commit_then_renew(); + let new_v = a.oplog_vv(); + // new mark should be ignored, so vv should be the same + assert_eq!(v, new_v); +} + +#[test] +fn event_from_checkout() { + let a = LoroDoc::new_auto_commit(); + let sub = a.subscribe_root(Arc::new(|event| { + assert!(matches!( + event.event_meta.by, + EventTriggerKind::Checkout | EventTriggerKind::Local + )); + })); + a.get_text("text").insert_unicode(0, "hello").unwrap(); + a.commit_then_renew(); + let version = a.oplog_frontiers(); + a.get_text("text").insert_unicode(0, "hello").unwrap(); + a.commit_then_renew(); + sub.unsubscribe(); + let ran = Arc::new(AtomicBool::new(false)); + let ran_cloned = ran.clone(); + let _g = a.subscribe_root(Arc::new(move |event| { + assert!(event.event_meta.by.is_checkout()); + ran.store(true, std::sync::atomic::Ordering::Relaxed); + })); + a.checkout(&version).unwrap(); + assert!(ran_cloned.load(std::sync::atomic::Ordering::Relaxed)); +} + +#[test] +fn handler_in_event() { + let doc = LoroDoc::new_auto_commit(); + let _g = doc.subscribe_root(Arc::new(|e| { + dbg!(&e); + let value = e.events[0] + .diff + .as_list() + .unwrap() + .iter() + .next() + .unwrap() + .as_replace() + .unwrap() + .0 + .iter() + .next() + .unwrap(); + assert!(matches!(value, ValueOrHandler::Handler(Handler::Text(_)))); + })); + let list = doc.get_list("list"); + list.insert_container(0, TextHandler::new_detached()) + .unwrap(); + doc.commit_then_renew(); +} + +#[test] +fn out_of_bound_test() { + let a = LoroDoc::new_auto_commit(); + a.get_text("text").insert_unicode(0, "Hello").unwrap(); + a.get_list("list").insert(0, "Hello").unwrap(); + a.get_list("list").insert(1, "Hello").unwrap(); + // expect out of bound err + let err = a.get_text("text").insert_unicode(6, "Hello").unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })); + let err = a.get_text("text").delete_unicode(3, 5).unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })); + let err = a + .get_text("text") + .mark(0, 8, "h", 5.into(), PosType::Event) + .unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })); + let _err = a + .get_text("text") + .mark(3, 0, "h", 5.into(), PosType::Event) + .unwrap_err(); + let err = a.get_list("list").insert(6, "Hello").unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })); + let err = a.get_list("list").delete(3, 2).unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })); + let err = a + .get_list("list") + .insert_container(3, MapHandler::new_detached()) + .unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })); +} + +#[test] +fn list() { + let a = LoroDoc::new_auto_commit(); + a.get_list("list").insert(0, "Hello").unwrap(); + assert_eq!(a.get_list("list").get(0).unwrap(), LoroValue::from("Hello")); + let map = a + .get_list("list") + .insert_container(1, MapHandler::new_detached()) + .unwrap(); + map.insert("Hello", LoroValue::from("u")).unwrap(); + let pos = map + .insert_container("pos", MapHandler::new_detached()) + .unwrap(); + pos.insert("x", 0).unwrap(); + pos.insert("y", 100).unwrap(); + + let cid = map.id(); + let id = a.get_list("list").get(1); + assert_eq!(id.as_ref().unwrap().as_container().unwrap(), &cid); + let map = a.get_map(id.unwrap().into_container().unwrap()); + let new_pos = a.get_map(map.get("pos").unwrap().into_container().unwrap()); + assert_eq!( + new_pos.get_deep_value().to_json_value(), + json!({ + "x": 0, + "y": 100, + }) + ); +} + +#[test] +fn richtext_mark_event() { + let a = LoroDoc::new_auto_commit(); + let _g = a.subscribe( + &a.get_text("text").id(), + Arc::new(|e| { + let delta = e.events[0].diff.as_text().unwrap(); + assert_eq!( + delta.to_json_value(), + json!([ + {"insert": "He", "attributes": {"bold": true}}, + {"insert": "ll", "attributes": {"bold": null}}, + {"insert": "o", "attributes": {"bold": true}} + ]) + ) + }), + ); + a.get_text("text").insert_unicode(0, "Hello").unwrap(); + a.get_text("text") + .mark(0, 5, "bold", true.into(), PosType::Event) + .unwrap(); + a.get_text("text") + .mark(2, 4, "bold", LoroValue::Null, PosType::Event) + .unwrap(); + let _ = a.implicit_commit_then_stop(); + let b = LoroDoc::new_auto_commit(); + let _g = b.subscribe( + &a.get_text("text").id(), + Arc::new(|e| { + let delta = e.events[0].diff.as_text().unwrap(); + assert_eq!( + delta.to_json_value(), + json!([ + {"insert": "He", "attributes": {"bold": true}}, + {"insert": "ll", "attributes": {"bold": null}}, + {"insert": "o", "attributes": {"bold": true}} + ]) + ) + }), + ); + b.merge(&a).unwrap(); +} + +#[test] +fn concurrent_richtext_mark_event() { + let a = LoroDoc::new_auto_commit(); + let b = LoroDoc::new_auto_commit(); + let c = LoroDoc::new_auto_commit(); + a.get_text("text").insert_unicode(0, "Hello").unwrap(); + b.merge(&a).unwrap(); + c.merge(&a).unwrap(); + b.get_text("text") + .mark(0, 3, "bold", true.into(), PosType::Event) + .unwrap(); + c.get_text("text") + .mark(1, 4, "link", true.into(), PosType::Event) + .unwrap(); + b.merge(&c).unwrap(); + let sub_id = a.subscribe( + &a.get_text("text").id(), + Arc::new(|e| { + let delta = e.events[0].diff.as_text().unwrap(); + assert_eq!( + delta.to_json_value(), + json!([ + {"retain": 1, "attributes": {"bold": true, }}, + {"retain": 2, "attributes": {"bold": true, "link": true}}, + {"retain": 1, "attributes": {"link": true}}, + ]) + ) + }), + ); + + a.merge(&b).unwrap(); + sub_id.unsubscribe(); + + let sub_id = a.subscribe( + &a.get_text("text").id(), + Arc::new(|e| { + let delta = e.events[0].diff.as_text().unwrap(); + assert_eq!( + delta.to_json_value(), + json!([ + { + "retain": 2, + }, + { + "retain": 1, + "attributes": {"bold": null, "link": true} + } + ]) + ) + }), + ); + + b.get_text("text") + .mark(2, 3, "bold", LoroValue::Null, PosType::Event) + .unwrap(); + a.merge(&b).unwrap(); + sub_id.unsubscribe(); + let _g = a.subscribe( + &a.get_text("text").id(), + Arc::new(|e| { + for container_diff in e.events { + let delta = container_diff.diff.as_text().unwrap(); + assert_eq!( + delta.to_json_value(), + json!([ + { + "retain": 2, + }, + { + "insert": "A", + "attributes": {"bold": true, "link": true} + } + ]) + ) + } + }), + ); + a.get_text("text").insert_unicode(2, "A").unwrap(); + let _ = a.implicit_commit_then_stop(); +} + +#[test] +fn insert_richtext_event() { + let a = LoroDoc::new_auto_commit(); + a.get_text("text").insert_unicode(0, "Hello").unwrap(); + a.get_text("text") + .mark(0, 5, "bold", true.into(), PosType::Event) + .unwrap(); + a.commit_then_renew(); + let text = a.get_text("text"); + let _g = a.subscribe( + &text.id(), + Arc::new(|e| { + let delta = e.events[0].diff.as_text().unwrap(); + assert_eq!( + delta.to_json_value(), + json!([ + {"retain": 5,}, + {"insert": " World!", "attributes": {"bold": true}} + ]) + ) + }), + ); + + text.insert_unicode(5, " World!").unwrap(); +} + +#[test] +fn import_after_init_handlers() { + let a = LoroDoc::new_auto_commit(); + let _g = a.subscribe( + &ContainerID::new_root("text", ContainerType::Text), + Arc::new(|event| { + assert!(matches!( + event.events[0].diff, + loro_internal::event::Diff::Text(_) + )) + }), + ); + let _g = a.subscribe( + &ContainerID::new_root("map", ContainerType::Map), + Arc::new(|event| { + assert!(matches!( + event.events[0].diff, + loro_internal::event::Diff::Map(_) + )) + }), + ); + let _g = a.subscribe( + &ContainerID::new_root("list", ContainerType::List), + Arc::new(|event| { + assert!(matches!( + event.events[0].diff, + loro_internal::event::Diff::List(_) + )) + }), + ); + + let b = LoroDoc::new_auto_commit(); + b.get_list("list").insert(0, "list").unwrap(); + b.get_list("list_a").insert(0, "list_a").unwrap(); + b.get_text("text").insert_unicode(0, "text").unwrap(); + b.get_map("map").insert("m", "map").unwrap(); + a.import(&b.export(ExportMode::Snapshot).unwrap()).unwrap(); + a.commit_then_renew(); +} + +#[test] +fn test_from_snapshot() { + let a = LoroDoc::new_auto_commit(); + a.get_text("text").insert_unicode(0, "0").unwrap(); + let snapshot = a.export(ExportMode::Snapshot).unwrap(); + let c = LoroDoc::from_snapshot(&snapshot).unwrap(); + assert_eq!(a.get_deep_value(), c.get_deep_value()); + assert_eq!(a.oplog_frontiers(), c.oplog_frontiers()); + assert_eq!(a.state_frontiers(), c.state_frontiers()); + let updates = a.export(ExportMode::all_updates()).unwrap(); + let d = match LoroDoc::from_snapshot(&updates) { + Ok(_) => panic!(), + Err(e) => e, + }; + assert!(matches!(d, loro_common::LoroError::DecodeError(..))); +} + +#[test] +fn test_pending() { + let a = LoroDoc::new_auto_commit(); + a.set_peer_id(0).unwrap(); + a.get_text("text").insert_unicode(0, "0").unwrap(); + let b = LoroDoc::new_auto_commit(); + b.set_peer_id(1).unwrap(); + b.import(&a.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + b.get_text("text").insert_unicode(0, "1").unwrap(); + let c = LoroDoc::new_auto_commit(); + b.set_peer_id(2).unwrap(); + c.import(&b.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + c.get_text("text").insert_unicode(0, "2").unwrap(); + + // c creates a pending change for a, insert "2" cannot be merged into a yet + a.import(&c.export(ExportMode::updates(&b.oplog_vv())).unwrap()) + .unwrap(); + assert_eq!(a.get_deep_value().to_json_value(), json!({"text": "0"})); + + // b does not has c's change + a.import(&b.export(ExportMode::updates(&a.oplog_vv())).unwrap()) + .unwrap(); + dbg!(&a.oplog().lock()); + assert_eq!(a.get_deep_value().to_json_value(), json!({"text": "210"})); +} + +#[test] +fn test_checkout() { + let doc_0 = LoroDoc::new_auto_commit(); + doc_0.set_peer_id(0).unwrap(); + let doc_1 = LoroDoc::new_auto_commit(); + doc_1.set_peer_id(1).unwrap(); + + let value: Arc> = Arc::new(Mutex::new(LoroValue::Map(Default::default()))); + let root_value = value.clone(); + let _g = doc_0.subscribe_root(Arc::new(move |event| { + dbg!(&event); + let mut root_value = root_value.lock(); + for container_diff in event.events { + root_value.apply( + &container_diff.path.iter().map(|x| x.1.clone()).collect(), + std::slice::from_ref(&container_diff.diff), + ); + } + })); + + let map = doc_0.get_map("map"); + let handler = map + .insert_container("text", TextHandler::new_detached()) + .unwrap(); + let text = handler; + text.insert_unicode(0, "123").unwrap(); + + let map = doc_1.get_map("map"); + + let handler = map + .insert_container("text", TextHandler::new_detached()) + .unwrap(); + let text = handler; + text.insert_unicode(0, "123").unwrap(); + + doc_0 + .import(&doc_1.export(ExportMode::all_updates()).unwrap()) + .unwrap(); + + doc_0 + .checkout(&Frontiers::from(vec![ID::new(0, 2)])) + .unwrap(); + + assert_eq!(&doc_0.get_deep_value(), &*value.lock()); + assert_eq!( + value.lock().to_json_value(), + json!({ + "map": { + "text": "12" + } + }) + ); +} + +#[test] +fn test_timestamp() { + let doc = LoroDoc::new(); + doc.set_record_timestamp(true); + let text = doc.get_text("text"); + let mut txn = doc.txn().unwrap(); + text.insert_with_txn(&mut txn, 0, "123", PosType::Unicode) + .unwrap(); + txn.commit().unwrap(); + let op_log = &doc.oplog().lock(); + let change = op_log.get_change_at(ID::new(doc.peer_id(), 0)).unwrap(); + assert!(change.timestamp() > 1690966970); +} + +#[test] +fn test_text_checkout() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(1).unwrap(); + let text = doc.get_text("text"); + text.insert_unicode(0, "你界").unwrap(); + text.insert_unicode(1, "好世").unwrap(); + doc.commit_then_renew(); + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 0)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你"); + } + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 1)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你界"); + } + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 2)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好界"); + } + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 3)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好世界"); + } + assert_eq!(text.len_unicode(), 4); + assert_eq!(text.len_utf8(), 12); + assert_eq!(text.len_unicode(), 4); + + doc.checkout_to_latest(); + text.delete_unicode(3, 1).unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好世"); + text.delete_unicode(2, 1).unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好"); + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 3)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好世界"); + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 4)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好世"); + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 5)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好"); + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 0)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你"); + } + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 1)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你界"); + } + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 2)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好界"); + } + { + doc.checkout(&Frontiers::from([ID::new(doc.peer_id(), 3)].as_slice())) + .unwrap(); + assert_eq!(text.get_value().as_string().unwrap().as_str(), "你好世界"); + } +} + +#[test] +fn map_checkout() { + let doc = LoroDoc::new_auto_commit(); + let meta = doc.get_map("meta"); + let v_empty = doc.oplog_frontiers(); + meta.insert("key", 0).unwrap(); + let v0 = doc.oplog_frontiers(); + meta.insert("key", 1).unwrap(); + let v1 = doc.oplog_frontiers(); + assert_eq!(meta.get_deep_value().to_json(), r#"{"key":1}"#); + doc.checkout(&v0).unwrap(); + assert_eq!(meta.get_deep_value().to_json(), r#"{"key":0}"#); + doc.checkout(&v_empty).unwrap(); + assert_eq!(meta.get_deep_value().to_json(), r#"{}"#); + doc.checkout(&v1).unwrap(); + assert_eq!(meta.get_deep_value().to_json(), r#"{"key":1}"#); +} + +#[test] +fn a_list_of_map_checkout() { + let doc = LoroDoc::new_auto_commit(); + let entry = doc.get_map("entry"); + let (list, sub) = { + let list = entry + .insert_container("list", ListHandler::new_detached()) + .unwrap(); + let sub_map = list + .insert_container(0, MapHandler::new_detached()) + .unwrap(); + sub_map.insert("x", 100).unwrap(); + sub_map.insert("y", 1000).unwrap(); + (list, sub_map) + }; + let v0 = doc.oplog_frontiers(); + let d0 = doc.get_deep_value().to_json(); + + list.insert(0, 3).unwrap(); + list.push(4).unwrap(); + list.insert_container(2, MapHandler::new_detached()) + .unwrap(); + list.insert_container(3, TextHandler::new_detached()) + .unwrap(); + + list.delete(2, 1).unwrap(); + sub.insert("x", 9).unwrap(); + sub.insert("y", 9).unwrap(); + sub.insert("z", 9).unwrap(); + let v1 = doc.oplog_frontiers(); + let d1 = doc.get_deep_value().to_json(); + sub.insert("x", 77).unwrap(); + sub.insert("y", 88).unwrap(); + list.delete(0, 1).unwrap(); + list.insert(0, 123).unwrap(); + list.push(99).unwrap(); + let v2 = doc.oplog_frontiers(); + let d2 = doc.get_deep_value().to_json(); + + doc.checkout(&v0).unwrap(); + assert_eq!(doc.get_deep_value().to_json(), d0); + doc.checkout(&v1).unwrap(); + assert_eq!(doc.get_deep_value().to_json(), d1); + doc.checkout(&v2).unwrap(); + println!("{}", doc.get_deep_value_with_id().to_json_pretty()); + assert_eq!(doc.get_deep_value().to_json(), d2); + doc.checkout(&v1).unwrap(); + + println!("{}", doc.get_deep_value_with_id().to_json_pretty()); + assert_eq!(doc.get_deep_value().to_json(), d1); + doc.checkout(&v0).unwrap(); + assert_eq!(doc.get_deep_value().to_json(), d0); +} + +#[test] +fn map_concurrent_checkout() { + let doc_a = LoroDoc::new_auto_commit(); + let meta_a = doc_a.get_map("meta"); + let doc_b = LoroDoc::new_auto_commit(); + let meta_b = doc_b.get_map("meta"); + + meta_a.insert("key", 0).unwrap(); + let va = doc_a.oplog_frontiers(); + meta_b.insert("s", 1).unwrap(); + let vb_0 = doc_b.oplog_frontiers(); + meta_b.insert("key", 1).unwrap(); + let vb_1 = doc_b.oplog_frontiers(); + doc_a + .import(&doc_b.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + meta_a.insert("key", 2).unwrap(); + + let v_merged = doc_a.oplog_frontiers(); + + doc_a.checkout(&va).unwrap(); + assert_eq!(meta_a.get_deep_value().to_json_value(), json!({"key":0})); + doc_a.checkout(&vb_0).unwrap(); + assert_eq!(meta_a.get_deep_value().to_json_value(), json!({"s":1})); + doc_a.checkout(&vb_1).unwrap(); + assert_eq!( + meta_a.get_deep_value().to_json_value(), + json!({"s":1,"key":1}) + ); + doc_a.checkout(&v_merged).unwrap(); + assert_eq!( + meta_a.get_deep_value().to_json_value(), + json!({"s":1,"key":2}) + ); +} + +#[test] +fn tree_checkout() { + let doc_a = LoroDoc::new_auto_commit(); + let _g = doc_a.subscribe_root(Arc::new(|_e| {})); + doc_a.set_peer_id(1).unwrap(); + let tree = doc_a.get_tree("root"); + let id1 = tree.create(TreeParentId::Root).unwrap(); + let id2 = tree.create(TreeParentId::Node(id1)).unwrap(); + let v1_state = tree.get_deep_value(); + let v1 = doc_a.oplog_frontiers(); + let _id3 = tree.create(TreeParentId::Node(id2)).unwrap(); + let v2_state = tree.get_deep_value(); + let v2 = doc_a.oplog_frontiers(); + tree.delete(id2).unwrap(); + let v3_state = tree.get_deep_value(); + let v3 = doc_a.oplog_frontiers(); + doc_a.checkout(&v1).unwrap(); + assert_eq!( + serde_json::to_value(tree.get_deep_value()) + .unwrap() + .get("roots"), + serde_json::to_value(v1_state).unwrap().get("roots") + ); + doc_a.checkout(&v2).unwrap(); + assert_eq!( + serde_json::to_value(tree.get_deep_value()) + .unwrap() + .get("roots"), + serde_json::to_value(v2_state).unwrap().get("roots") + ); + doc_a.checkout(&v3).unwrap(); + assert_eq!( + serde_json::to_value(tree.get_deep_value()) + .unwrap() + .get("roots"), + serde_json::to_value(v3_state).unwrap().get("roots") + ); + + doc_a.attach(); + tree.create(TreeParentId::Root).unwrap(); +} + +#[test] +fn issue_batch_import_snapshot() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(123).unwrap(); + let doc2 = LoroDoc::new_auto_commit(); + doc2.set_peer_id(456).unwrap(); + doc.get_map("map").insert("s", "hello world!").unwrap(); + doc2.get_map("map").insert("s", "hello?").unwrap(); + + let data1 = doc.export(ExportMode::Snapshot).unwrap(); + let data2 = doc2.export(ExportMode::Snapshot).unwrap(); + let doc3 = LoroDoc::new(); + doc3.import_batch(&[data1, data2]).unwrap(); +} + +#[test] +fn state_may_deadlock_when_import() { + // helper function ref: https://github.com/rust-lang/rfcs/issues/2798#issuecomment-552949300 + use std::time::Duration; + use std::{sync::mpsc, thread}; + fn panic_after(d: Duration, f: F) -> T + where + T: Send + 'static, + F: FnOnce() -> T, + F: Send + 'static, + { + let (done_tx, done_rx) = mpsc::channel(); + let handle = thread::spawn(move || { + let val = f(); + done_tx.send(()).expect("Unable to send completion signal"); + val + }); + + match done_rx.recv_timeout(d) { + Ok(_) => handle.join().expect("Thread panicked"), + Err(_) => panic!("Thread took too long"), + } + } + + panic_after(Duration::from_millis(100), || { + let doc = LoroDoc::new_auto_commit(); + let map = doc.get_map("map"); + let _g = doc.subscribe_root(Arc::new(move |_e| { + map.id(); + })); + + let doc2 = LoroDoc::new_auto_commit(); + doc2.get_map("map").insert("foo", 123).unwrap(); + doc.import(&doc.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + }) +} + +#[ctor::ctor] +fn init() { + dev_utils::setup_test_log(); +} + +#[test] +fn missing_event_when_checkout() { + let doc = LoroDoc::new_auto_commit(); + doc.checkout(&doc.oplog_frontiers()).unwrap(); + let value = Arc::new(Mutex::new(FxHashMap::default())); + let map = value.clone(); + let _g = doc.subscribe( + &ContainerID::new_root("tree", ContainerType::Tree), + Arc::new(move |e| { + let mut v = map.lock(); + for container_diff in e.events.iter() { + let from_children = + container_diff.id != ContainerID::new_root("tree", ContainerType::Tree); + if from_children { + if let Diff::Map(map) = &container_diff.diff { + for (k, ResolvedMapValue { value, .. }) in map.updated.iter() { + match value { + Some(value) => { + v.insert( + k.to_string(), + *value.as_value().unwrap().as_i64().unwrap(), + ); + } + None => { + v.remove(&k.to_string()); + } + } + } + } + } + } + }), + ); + + let doc2 = LoroDoc::new_auto_commit(); + let tree = doc2.get_tree("tree"); + let node = tree.create_at(TreeParentId::Root, 0).unwrap(); + let _ = tree.create_at(TreeParentId::Root, 0).unwrap(); + let meta = tree.get_meta(node).unwrap(); + meta.insert("a", 0).unwrap(); + doc.import(&doc2.export(ExportMode::updates(&doc.oplog_vv())).unwrap()) + .unwrap(); + doc.attach(); + meta.insert("b", 1).unwrap(); + doc.checkout(&doc.oplog_frontiers()).unwrap(); + doc.import(&doc2.export(ExportMode::updates(&doc.oplog_vv())).unwrap()) + .unwrap(); + // checkout use the same diff_calculator, the depth of calculator is not updated + doc.attach(); + assert!(value.lock().contains_key("b")); +} + +#[test] +fn empty_event() { + let doc = LoroDoc::new_auto_commit(); + doc.get_map("map").insert("key", 123).unwrap(); + doc.commit_then_renew(); + let fire = Arc::new(AtomicBool::new(false)); + let fire_clone = Arc::clone(&fire); + let _g = doc.subscribe_root(Arc::new(move |_e| { + fire_clone.store(true, std::sync::atomic::Ordering::Relaxed); + })); + doc.import(&doc.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + assert!(!fire.load(std::sync::atomic::Ordering::Relaxed)); +} + +#[test] +fn insert_attach_container() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + let list = doc.get_list("list"); + list.insert_container(0, MapHandler::new_detached())? + .insert("key", 1)?; + list.insert_container(1, MapHandler::new_detached())? + .insert("key", 2)?; + let elem = list.insert_container(2, MapHandler::new_detached())?; + elem.insert("key", 3)?; + let new_map = list.insert_container(0, elem)?; + assert_eq!(new_map.get_value().to_json_value(), json!({"key": 3})); + list.delete(3, 1)?; + + let elem = list.insert_container(0, TextHandler::new_detached())?; + elem.insert(0, "abc", PosType::Unicode)?; + elem.mark(0, 2, "bold", true.into(), PosType::Event)?; + let new_text = list.insert_container(0, elem)?; + assert_eq!( + new_text.get_richtext_value().to_json_value(), + json!([{"insert":"ab", "attributes": {"bold": true}}, {"insert":"c"}]) + ); + + let elem = list.insert_container(0, ListHandler::new_detached())?; + elem.insert(0, "list")?; + let new_list = list.insert_container(0, elem)?; + new_list.insert(0, "new_list")?; + assert_eq!( + new_list.get_value().to_json_value(), + json!(["new_list", "list"]) + ); + + // let elem = list.insert_container(2, TreeHandler::new_detached())?; + // let p = elem.create(None)?; + // elem.create(p)?; + // list.insert_container(0, elem)?; + + assert_eq!( + doc.get_deep_value().to_json_value(), + json!({ + "list": [["new_list", "list"], ["list"],"abc", "abc", {"key": 3}, {"key": 1}, {"key": 2}] + }) + ); + Ok(()) +} + +#[test] +fn tree_attach() { + let tree = TreeHandler::new_detached(); + let id = tree.create(TreeParentId::Root).unwrap(); + tree.get_meta(id).unwrap().insert("key", "value").unwrap(); + let doc = LoroDoc::new_auto_commit(); + doc.get_list("list").insert_container(0, tree).unwrap(); + let v = doc.get_deep_value(); + assert_eq!( + v.as_map().unwrap().get("list").unwrap().as_list().unwrap()[0] + .as_list() + .unwrap()[0] + .as_map() + .unwrap() + .get("meta") + .unwrap() + .to_json_value(), + json!({"key":"value"}) + ) +} + +#[test] +#[cfg(feature = "counter")] +fn counter() { + let doc = LoroDoc::new_auto_commit(); + let counter = doc.get_counter("counter"); + counter.increment(1.).unwrap(); + counter.increment(2.).unwrap(); + counter.decrement(1.).unwrap(); + let json = doc.export_json_updates(&Default::default(), &doc.oplog_vv(), true); + let doc2 = LoroDoc::new_auto_commit(); + doc2.import_json_updates(json).unwrap(); +} + +#[test] +fn test_insert_utf8() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "Hello ").unwrap(); + text.insert_utf8(6, "World").unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"Hello World"}]) + ) +} + +#[test] +fn test_insert_utf8_cross_unicode_1() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "你好").unwrap(); + text.insert_utf8(3, "World").unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"你World好"}]) + ) +} + +#[test] +fn test_insert_utf8_cross_unicode_2() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "你好").unwrap(); + text.insert_utf8(6, "World").unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"你好World"}]) + ) +} + +#[test] +fn test_insert_utf8_detached() { + let text = TextHandler::new_detached(); + text.insert_utf8(0, "Hello ").unwrap(); + text.insert_utf8(6, "World").unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"Hello World"}]) + ) +} + +#[test] +#[should_panic] +#[ignore = "fix me later after gbtree support Result for query"] +fn test_insert_utf8_panic_cross_unicode() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "你好").unwrap(); + text.insert_utf8(1, "World").unwrap(); +} + +#[test] +#[should_panic] +fn test_insert_utf8_panic_out_bound() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "Hello ").unwrap(); + text.insert_utf8(7, "World").unwrap(); +} + +// println!("{}", text.get_richtext_value().to_json_value().to_string()); + +#[test] +fn test_delete_utf8() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "Hello").unwrap(); + text.delete_utf8(1, 3).unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"Ho"}]) + ) +} + +#[test] +fn test_delete_utf8_with_zero_len() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "Hello").unwrap(); + text.delete_utf8(1, 0).unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"Hello"}]) + ) +} + +#[test] +fn test_delete_utf8_cross_unicode() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_utf8(0, "你好").unwrap(); + text.delete_utf8(0, 3).unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"好"}]) + ) +} + +#[test] +fn test_delete_utf8_detached() { + let text = TextHandler::new_detached(); + text.insert_utf8(0, "Hello").unwrap(); + text.delete_utf8(1, 3).unwrap(); + assert_eq!( + text.get_richtext_value().to_json_value(), + json!([{"insert":"Ho"}]) + ) +} + +// WARNING: +// Due to the current inability to report an error on +// get_offset_and_found on BTree, this test won't be ok. +// #[test] +// #[should_panic] +// fn test_delete_utf8_panic_cross_unicode() { +// let doc = LoroDoc::new_auto_commit(); +// let text = doc.get_text("text"); +// text.insert_utf8(0, "你好").unwrap(); +// text.delete_utf8(0, 2).unwrap(); +// } + +#[test] +#[should_panic] +fn test_delete_utf8_panic_out_bound_pos() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "Hello").unwrap(); + text.delete_utf8(10, 1).unwrap(); +} + +#[test] +#[should_panic] +fn test_delete_utf8_panic_out_bound_len() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "Hello").unwrap(); + text.delete_utf8(1, 10).unwrap(); +} + +#[test] +fn test_char_at() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "Herld").unwrap(); + text.insert_unicode(2, "llo Wo").unwrap(); + assert_eq!(text.char_at(0, PosType::Unicode).unwrap(), 'H'); + assert_eq!(text.char_at(1, PosType::Unicode).unwrap(), 'e'); + assert_eq!(text.char_at(2, PosType::Unicode).unwrap(), 'l'); + assert_eq!(text.char_at(3, PosType::Unicode).unwrap(), 'l'); + let err = text.char_at(15, PosType::Unicode).unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })) +} + +#[test] +fn test_char_at_detached() { + let text = TextHandler::new_detached(); + text.insert_unicode(0, "Herld").unwrap(); + text.insert_unicode(2, "llo Wo").unwrap(); + assert_eq!(text.char_at(0, PosType::Unicode).unwrap(), 'H'); + assert_eq!(text.char_at(1, PosType::Unicode).unwrap(), 'e'); + assert_eq!(text.char_at(2, PosType::Unicode).unwrap(), 'l'); + assert_eq!(text.char_at(3, PosType::Unicode).unwrap(), 'l'); + let err = text.char_at(15, PosType::Unicode).unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })) +} + +#[test] +fn test_char_at_wchar() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "你好").unwrap(); + text.insert_unicode(1, "世界").unwrap(); + assert_eq!(text.char_at(0, PosType::Unicode).unwrap(), '你'); + assert_eq!(text.char_at(1, PosType::Unicode).unwrap(), '世'); + assert_eq!(text.char_at(2, PosType::Unicode).unwrap(), '界'); + assert_eq!(text.char_at(3, PosType::Unicode).unwrap(), '好'); + let err = text.char_at(5, PosType::Unicode).unwrap_err(); + assert!(matches!(err, loro_common::LoroError::OutOfBound { .. })) +} + +#[test] +fn test_text_slice() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "Hello").unwrap(); + text.insert_unicode(1, "World").unwrap(); + assert_eq!(text.slice(0, 4, PosType::Unicode).unwrap(), "HWor"); + assert_eq!(text.slice(0, 1, PosType::Unicode).unwrap(), "H"); +} + +#[test] +fn test_text_slice_detached() { + let text = TextHandler::new_detached(); + text.insert_unicode(0, "Herld").unwrap(); + text.insert_unicode(2, "llo Wo").unwrap(); + assert_eq!(text.slice(0, 4, PosType::Unicode).unwrap(), "Hell"); + assert_eq!(text.slice(0, 1, PosType::Unicode).unwrap(), "H"); +} + +#[test] +fn test_text_slice_wchar() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "你好").unwrap(); + text.insert_unicode(1, "世界").unwrap(); + assert_eq!(text.slice(0, 3, PosType::Unicode).unwrap(), "你世界"); +} + +#[test] +#[should_panic] +fn test_text_slice_end_index_less_than_start() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "你好").unwrap(); + text.insert_unicode(1, "世界").unwrap(); + text.slice(2, 1, PosType::Unicode).unwrap(); +} + +#[test] +#[should_panic] +fn test_text_slice_out_of_bound() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "你好").unwrap(); + text.insert_unicode(1, "世界").unwrap(); + text.slice(1, 10, PosType::Unicode).unwrap(); +} + +#[test] +fn test_text_splice() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "你好").unwrap(); + assert_eq!(text.splice(1, 1, "世界", PosType::Unicode).unwrap(), "好"); + assert_eq!(text.to_string(), "你世界"); +} + +#[test] +fn test_text_iter() { + let mut str = String::new(); + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "Hello").unwrap(); + text.insert_unicode(1, "Hello").unwrap(); + text.iter(|s| { + str.push_str(s); + true + }); + assert_eq!(str, "HHelloello"); + str = String::new(); + let mut i = 0; + text.iter(|s| { + if i == 1 { + return false; + } + str.push_str(s); + i += 1; + true + }); + assert_eq!(str, "H"); +} + +#[test] +fn test_text_iter_detached() { + let mut str = String::new(); + let text = TextHandler::new_detached(); + text.insert_unicode(0, "Hello").unwrap(); + text.insert_unicode(1, "Hello").unwrap(); + text.iter(|s| { + str.push_str(s); + true + }); + assert_eq!(str, "HHelloello"); +} + +#[test] +fn test_text_update() { + let doc = LoroDoc::new_auto_commit(); + let text = doc.get_text("text"); + text.insert_unicode(0, "Hello 😊Bro").unwrap(); + text.update("Hello World Bro😊", Default::default()) + .unwrap(); + assert_eq!(text.to_string(), "Hello World Bro😊"); +} + +#[test] +fn test_map_contains_key() { + let doc = LoroDoc::new_auto_commit(); + let map = doc.get_map("m"); + assert!(!map.contains_key("bro")); + map.insert("bro", 114514).unwrap(); + assert!(map.contains_key("bro")); + map.delete("bro").unwrap(); + assert!(!map.contains_key("bro")); +} + +#[test] +fn set_max_peer_id() { + let doc = LoroDoc::new_auto_commit(); + assert_eq!( + doc.set_peer_id(PeerID::MAX), + Result::Err(LoroError::InvalidPeerID) + ); +} + +#[test] +fn import_status() -> LoroResult<()> { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0)?; + let t = doc.get_text("text"); + t.insert_unicode(0, "a")?; + + let doc2 = LoroDoc::new_auto_commit(); + doc2.set_peer_id(1)?; + let t2 = doc2.get_text("text"); + t2.insert_unicode(0, "b")?; + doc2.commit_then_renew(); + let update1 = doc2.export(ExportMode::Snapshot).unwrap(); + let vv1 = doc2.oplog_vv(); + t2.insert_unicode(1, "c")?; + let update2 = doc2.export(ExportMode::updates(&vv1)).unwrap(); + + let status1 = doc.import(&update2)?; + let status2 = doc.import(&update1)?; + assert_eq!( + status1, + ImportStatus { + success: Default::default(), + pending: Some(VersionRange::from_map(fx_map!(1=>(1, 2)))) + } + ); + assert_eq!( + status2, + ImportStatus { + success: VersionRange::from_map(fx_map!(1=>(0, 2))), + pending: None + } + ); + + Ok(()) +} + +#[test] +fn test_on_first_commit_from_peer() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let p = Arc::new(Mutex::new(vec![])); + let p2 = Arc::clone(&p); + let sub = doc.subscribe_first_commit_from_peer(Box::new(move |e| { + p2.lock().push(e.peer); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.commit_then_renew(); + doc.get_text("text").insert_unicode(0, "b").unwrap(); + doc.commit_then_renew(); + doc.set_peer_id(1).unwrap(); + doc.get_text("text").insert_unicode(0, "c").unwrap(); + doc.commit_then_renew(); + sub.unsubscribe(); + assert_eq!(p.lock().as_slice(), &[0, 1]); +} + +#[test] +fn test_on_first_commit_from_peer_when_drop_doc() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let p = Arc::new(Mutex::new(vec![])); + let p2 = Arc::clone(&p); + let _sub = doc.subscribe_first_commit_from_peer(Box::new(move |e| { + p2.lock().push(e.peer); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + drop(doc); + assert_eq!(p.lock().as_slice(), &[0]); +} + +#[test] +fn test_on_first_commit_from_peer_and_set_peer_id() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let f = Arc::new(AtomicBool::new(false)); + let f2 = Arc::clone(&f); + let sub = doc.subscribe_first_commit_from_peer(Box::new(move |_e| { + f2.store(true, std::sync::atomic::Ordering::Relaxed); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.set_peer_id(1).unwrap(); + sub.unsubscribe(); + assert!(f.load(std::sync::atomic::Ordering::Relaxed)); +} + +#[test] +fn test_on_first_commit_from_peer_with_lock() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let doc_clone = doc.clone(); + let sub = doc.subscribe_first_commit_from_peer(Box::new(move |_e| { + doc_clone.get_text("text").insert_unicode(0, "b").unwrap(); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.commit_then_renew(); + sub.unsubscribe(); + assert_eq!(doc.get_text("text").to_string(), "ba"); + assert_eq!( + doc.export_json_updates(&Default::default(), &doc.oplog_vv(), false) + .changes + .len(), + 1 + ); +} + +#[test] +fn test_on_first_peer_commit_attach_user_id() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let doc_clone = doc.clone(); + let sub = doc.subscribe_first_commit_from_peer(Box::new(move |e| { + doc_clone + .get_map("::loro::user_id") + .insert(e.peer.to_string().as_str(), "user_bob") + .unwrap(); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.commit_then_renew(); + sub.unsubscribe(); + assert_eq!( + doc.get_map("::loro::user_id").get_value(), + loro_value!({ + "0": "user_bob" + }) + ); +} + +#[test] +fn test_pre_commit_with_lock() { + let doc = LoroDoc::new_auto_commit(); + let doc_clone = doc.clone(); + let sub = doc.subscribe_pre_commit(Box::new(move |_e| { + // state lock + doc_clone.get_deep_value(); + // oplog lock + doc_clone.oplog_vv(); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.commit_then_renew(); + sub.unsubscribe(); +} + +#[test] +fn test_pre_commit_with_hash() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let doc_clone = doc.clone(); + let sub = doc.subscribe_pre_commit(Box::new(move |e| { + let change_json = + doc_clone.export_json_in_id_span(e.change_meta.id.to_span(e.change_meta.len)); + assert!(change_json.len() == 1); + let mut deps = vec![]; + for dep in e.change_meta.deps.iter() { + let dep_msg = doc_clone + .oplog() + .lock() + .get_change_at(dep) + .unwrap() + .message() + .cloned() + .unwrap_or(Arc::from("")); + deps.push(dep_msg); + } + e.modifier.set_timestamp(0).set_message(&format!( + "{:08x}\n{}", + // just for test example, should use sha256 or blake3 for hash + xxhash_rust::xxh32::xxh32( + serde_json::to_string(&(&change_json, &deps)) + .unwrap() + .as_bytes(), + 0 + ), + e.change_meta.message() + )); + true + })); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.commit_with( + CommitOptions::default() + .commit_msg("add a") + .immediate_renew(true), + ); + doc.get_text("text").insert_unicode(0, "b").unwrap(); + doc.commit_with( + CommitOptions::default() + .commit_msg("add b") + .immediate_renew(true), + ); + sub.unsubscribe(); + let changes = doc + .export_json_updates(&Default::default(), &doc.oplog_vv(), false) + .changes; + assert_eq!(changes.len(), 2); + for c in changes { + let mut msg = c.msg.as_ref().unwrap().lines(); + assert_eq!(msg.next().unwrap().len(), 8); + assert!(msg.next().is_some()); + } +} + +#[test] +fn test_change_to_json_schema_include_uncommit() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + doc.get_text("text").insert_unicode(0, "a").unwrap(); + doc.commit_then_renew(); + let doc_clone = doc.clone(); + let _sub = doc.subscribe_pre_commit(Box::new(move |e| { + let changes = doc_clone.export_json_in_id_span(IdSpan::new( + 0, + 0, + e.change_meta.id.counter + e.change_meta.len as i32, + )); + assert_eq!(changes.len(), 2); + true + })); + doc.get_text("text").insert_unicode(0, "b").unwrap(); + let changes = doc.export_json_in_id_span(IdSpan::new(0, 0, 2)); + assert_eq!(changes.len(), 1); + doc.commit_then_renew(); + // change merged + assert_eq!(changes.len(), 1); +} + +#[test] +fn test_ephemeral_store() { + let store = EphemeralStore::new(1000); + let store_clone = store.clone(); + let _sub = store.subscribe(Box::new(move |_| { + store_clone.get_all_states(); + true + })); + store.set("a", 1); + store.set("b", 2); + store.set("c", 3); +} + +#[test] +fn test_origin() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let remote = LoroDoc::new_auto_commit(); + remote.set_peer_id(1).unwrap(); + doc.get_map("map").insert("a", 1).unwrap(); + doc.commit_then_renew(); + let snapshot = doc.export(ExportMode::Snapshot).unwrap(); + let expected_origin_string = "expectedOriginString"; + + let sub = remote.subscribe_root(Arc::new(move |e| { + assert_eq!(e.event_meta.origin, expected_origin_string.into()); + })); + remote + .import_with(&snapshot, expected_origin_string.into()) + .unwrap(); + sub.unsubscribe(); +} + +#[test] +fn apply_diff_with_same_peer_id_should_not_hang() { + let doc1 = LoroDoc::new_auto_commit(); + doc1.set_peer_id(1).unwrap(); + + let doc2 = doc1.fork(); + doc2.set_peer_id(1).unwrap(); // Same peer ID + + // The key issue: setContainer creates a nested container + let map = doc2.get_map("root"); + let nested = map + .insert_container("nested", MapHandler::new_detached()) + .unwrap(); // This is what triggers the hang + nested.insert("key", "value").unwrap(); + doc2.commit_then_renew(); + + let frontiers_before = doc1.oplog_frontiers(); + let frontiers_after = doc2.oplog_frontiers(); + let diff = doc2.diff(&frontiers_before, &frontiers_after).unwrap(); + + // This should NOT hang + doc1.apply_diff(diff).unwrap(); + assert_eq!( + doc1.get_deep_value().to_json_value(), + doc2.get_deep_value().to_json_value() + ); +} diff --git a/vendor/loro-internal/tests/tree.rs b/vendor/loro-internal/tests/tree.rs new file mode 100644 index 00000000..8cdae51f --- /dev/null +++ b/vendor/loro-internal/tests/tree.rs @@ -0,0 +1,35 @@ +use loro_internal::{LoroDoc, TreeParentId}; + +#[test] +fn tree_index() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let tree = doc.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + let child = tree.create(root.into()).unwrap(); + let child2 = tree.create_at(root.into(), 0).unwrap(); + // sort with OpID + assert_eq!(tree.get_index_by_tree_id(&child).unwrap(), 1); + assert_eq!(tree.get_index_by_tree_id(&child2).unwrap(), 0); + + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let tree = doc.get_tree("tree"); + tree.enable_fractional_index(0); + let root = tree.create(TreeParentId::Root).unwrap(); + let child = tree.create(root.into()).unwrap(); + let child2 = tree.create_at(root.into(), 0).unwrap(); + // sort with fractional index + assert_eq!(tree.get_index_by_tree_id(&child).unwrap(), 1); + assert_eq!(tree.get_index_by_tree_id(&child2).unwrap(), 0); +} + +#[test] +fn tree_move_in_parent() { + let doc = LoroDoc::new_auto_commit(); + doc.set_peer_id(0).unwrap(); + let tree = doc.get_tree("tree"); + let root = tree.create(TreeParentId::Root).unwrap(); + let child = tree.create(root.into()).unwrap(); + tree.mov(child, root.into()).unwrap(); +} diff --git a/vendor/loro-internal/tests/undo.rs b/vendor/loro-internal/tests/undo.rs new file mode 100644 index 00000000..c1ad818f --- /dev/null +++ b/vendor/loro-internal/tests/undo.rs @@ -0,0 +1,231 @@ +use std::borrow::Cow; + +use loro_internal::{ + cursor::PosType, handler::UpdateOptions, loro::ExportMode, LoroDoc, UndoManager, +}; + +#[test] +fn test_basic_undo_group_checkpoint() { + let doc = LoroDoc::new(); + let undo_manager = UndoManager::new(&doc); + let text = doc.get_text("text"); + + text.update("0", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + undo_manager.group_start().unwrap(); + text.update("1", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + text.update("12", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + undo_manager.group_end(); + undo_manager.undo().unwrap(); + assert_eq!( + text.to_string(), + "0", + "undo should undo the grouped updates" + ); + undo_manager.redo().unwrap(); + assert_eq!( + text.to_string(), + "12", + "redo should redo the grouped updates" + ); +} + +#[test] +fn test_invalid_nested_group() { + let doc = LoroDoc::new(); + + let undo_manager = UndoManager::new(&doc); + + assert!( + undo_manager.group_start().is_ok(), + "group start should succeed" + ); + assert!( + undo_manager.group_start().is_err(), + "nested group start should fail" + ); + undo_manager.group_end(); + assert!( + undo_manager.group_start().is_ok(), + "nested group end should fail" + ); +} + +#[test] +fn test_simulate_intersecting_remote_undo() { + let doc = LoroDoc::new(); + let undo_manager = UndoManager::new(&doc); + let text = doc.get_text("text"); + undo_manager.group_start().unwrap(); + println!("pushing 1"); + text.update("1", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + println!("pushing 2"); + text.update("12", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + + // At this point, the doc has state of "12" in the "text" container + + let snapshot = doc.export(ExportMode::Snapshot).unwrap(); + let doc2 = LoroDoc::from_snapshot(&snapshot).unwrap(); + let text2 = doc2.get_text("text"); + let vv = doc2.state_vv(); + text2.update("123", UpdateOptions::default()).unwrap(); + doc2.commit_then_renew(); + + let update = doc2 + .export(ExportMode::Updates { + from: Cow::Borrowed(&vv), + }) + .unwrap(); + + println!("importing 3"); + doc.import(&update).unwrap(); + + // At this point, the doc has state of "123" in the "text" container + + text.update("1234", UpdateOptions::default()).unwrap(); + println!("pushing 4"); + doc.commit_then_renew(); + println!("ending group"); + undo_manager.group_end(); + + undo_manager.undo().unwrap(); + + assert_eq!(text.to_string(), "123"); +} + +#[test] +fn test_simulate_non_intersecting_remote_undo() { + let doc = LoroDoc::new(); + let undo_manager = UndoManager::new(&doc); + let text = doc.get_text("text"); + undo_manager.group_start().unwrap(); + text.update("1", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + text.update("12", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + let snapshot = doc.export(ExportMode::Snapshot).unwrap(); + // At the point the first doc has state of "12" in the "text" container + + // Doc 2 makes changes to a separate container + // The doc 2 has state of "123" in the "text2" container + let doc2 = LoroDoc::from_snapshot(&snapshot).unwrap(); + let text2 = doc2.get_text("text2"); + let vv = doc2.state_vv(); + text2.update("123", UpdateOptions::default()).unwrap(); + doc2.commit_then_renew(); + let update = doc2 + .export(ExportMode::Updates { + from: Cow::Borrowed(&vv), + }) + .unwrap(); + doc.import(&update).unwrap(); + + text.update("123", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + undo_manager.group_end(); + + undo_manager.undo().unwrap(); + + assert_eq!(text.to_string(), ""); +} + +#[test] +fn test_undo_group_start_with_remote_ops() { + let doc = LoroDoc::new(); + let doc2 = LoroDoc::new(); + let undo_manager = UndoManager::new(&doc); + doc.get_text("text") + .insert(0, "hi", PosType::Unicode) + .unwrap(); + doc2.import(&doc.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + doc2.get_text("text") + .insert(0, "test", PosType::Unicode) + .unwrap(); + doc.import(&doc2.export(ExportMode::Snapshot).unwrap()) + .unwrap(); + undo_manager.group_start().unwrap(); + doc.get_text("text") + .insert(0, "t", PosType::Unicode) + .unwrap(); + undo_manager.undo().unwrap(); + assert_eq!(doc.get_text("text").to_string(), "testhi"); + undo_manager.undo().unwrap(); + assert_eq!(doc.get_text("text").to_string(), "test"); + undo_manager.redo().unwrap(); + assert_eq!(doc.get_text("text").to_string(), "testhi"); + undo_manager.redo().unwrap(); + assert_eq!(doc.get_text("text").to_string(), "ttesthi"); + undo_manager.undo().unwrap(); + assert_eq!(doc.get_text("text").to_string(), "testhi"); + undo_manager.undo().unwrap(); + assert_eq!(doc.get_text("text").to_string(), "test"); +} + +#[test] +fn test_clear_redo() { + let doc = LoroDoc::new(); + let undo_manager = UndoManager::new(&doc); + let text = doc.get_text("text"); + + // Make some edits + text.update("hello", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + text.update("hello world", UpdateOptions::default()) + .unwrap(); + doc.commit_then_renew(); + + // Undo to create redo stack + undo_manager.undo().unwrap(); + assert_eq!(text.to_string(), "hello"); + assert!(undo_manager.can_redo(), "should be able to redo"); + assert!(undo_manager.can_undo(), "should be able to undo"); + + // Clear only redo stack + undo_manager.clear_redo(); + assert!(!undo_manager.can_redo(), "redo stack should be empty"); + assert!( + undo_manager.can_undo(), + "undo stack should still have items" + ); + + // Verify undo still works + undo_manager.undo().unwrap(); + assert_eq!(text.to_string(), ""); +} + +#[test] +fn test_clear_undo() { + let doc = LoroDoc::new(); + let undo_manager = UndoManager::new(&doc); + let text = doc.get_text("text"); + + // Make some edits + text.update("hello", UpdateOptions::default()).unwrap(); + doc.commit_then_renew(); + text.update("hello world", UpdateOptions::default()) + .unwrap(); + doc.commit_then_renew(); + + // Undo to create redo stack + undo_manager.undo().unwrap(); + assert_eq!(text.to_string(), "hello"); + assert!(undo_manager.can_redo(), "should be able to redo"); + assert!(undo_manager.can_undo(), "should be able to undo"); + + // Clear only undo stack + undo_manager.clear_undo(); + assert!( + undo_manager.can_redo(), + "redo stack should still have items" + ); + assert!(!undo_manager.can_undo(), "undo stack should be empty"); + + // Verify redo still works + undo_manager.redo().unwrap(); + assert_eq!(text.to_string(), "hello world"); +} diff --git a/vendor/rdocx/Cargo.toml b/vendor/rdocx/Cargo.toml new file mode 100644 index 00000000..88873b55 --- /dev/null +++ b/vendor/rdocx/Cargo.toml @@ -0,0 +1,99 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.93" +name = "rdocx" +version = "0.1.2" +authors = ["Atul Sharma"] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "High-level API for reading, writing, and converting DOCX documents" +homepage = "https://github.com/tensorbee/rdocx" +readme = "README.md" +keywords = [ + "docx", + "word", + "pdf", + "ooxml", + "document", +] +categories = [ + "parser-implementations", + "text-processing", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/tensorbee/rdocx" +resolver = "2" + +[lib] +name = "rdocx" +path = "src/lib.rs" + +[[example]] +name = "convert_html_md" +path = "examples/convert_html_md.rs" + +[[example]] +name = "generate_all_samples" +path = "examples/generate_all_samples.rs" + +[[example]] +name = "generate_pdf" +path = "examples/generate_pdf.rs" + +[[example]] +name = "generate_samples" +path = "examples/generate_samples.rs" + +[[example]] +name = "header_banner" +path = "examples/header_banner.rs" + +[[example]] +name = "styled_tables" +path = "examples/styled_tables.rs" + +[[example]] +name = "template_replace" +path = "examples/template_replace.rs" + +[[test]] +name = "integration_test" +path = "tests/integration_test.rs" + +[dependencies.rdocx-html] +version = "0.1.2" + +[dependencies.rdocx-layout] +version = "0.1.2" + +[dependencies.rdocx-opc] +version = "0.1.2" + +[dependencies.rdocx-oxml] +version = "0.1.2" + +[dependencies.rdocx-pdf] +version = "0.1.2" + +[dependencies.regex] +version = "1" + +[dependencies.thiserror] +version = "2" + +[dev-dependencies] diff --git a/vendor/rdocx/README.md b/vendor/rdocx/README.md new file mode 100644 index 00000000..c9a0822b --- /dev/null +++ b/vendor/rdocx/README.md @@ -0,0 +1,267 @@ +# rdocx + +[![CI](https://github.com/tensorbee/rdocx/actions/workflows/ci.yml/badge.svg)](https://github.com/tensorbee/rdocx/actions/workflows/ci.yml) +[![crates.io](https://img.shields.io/crates/v/rdocx.svg)](https://crates.io/crates/rdocx) +[![docs.rs](https://docs.rs/rdocx/badge.svg)](https://docs.rs/rdocx) +[![License: MIT/Apache-2.0](https://img.shields.io/crates/l/rdocx.svg)](LICENSE) +[![MSRV: 1.93](https://img.shields.io/badge/MSRV-1.93-blue.svg)](https://blog.rust-lang.org/2026/01/09/Rust-1.93.0.html) + +A pure Rust DOCX library — create, read, and modify Word documents programmatically. Additionally, render pixel-identical PDFs and export to HTML and Markdown, all from the same document object. No LibreOffice, no unoconv, no C dependencies. + +## Why rdocx? + +Most DOCX solutions in the ecosystem shell out to LibreOffice or wrap C/C++ libraries. rdocx is written entirely in Rust, so it compiles to a single binary with zero runtime dependencies. It works everywhere Rust does — including WASM. + +The core focus is **DOCX**: a high-level, python-docx-inspired API for building and editing Word documents with paragraphs, tables, images, headers/footers, styles, and lists. On top of that, rdocx includes a built-in layout engine that paginates your document and can render it to **PDF** (with font subsetting, bookmarks, and selectable text) or export to **HTML** and **Markdown** — so you get faithful output in every format without leaving Rust. + +## DOCX Features + +- **Read & write** DOCX files with a high-level API +- **Tables** with merged cells, borders, shading, and content-based column sizing +- **Images** — inline and anchored, with header/footer background images +- **Headers & footers** with first-page support and per-section overrides +- **Styles** — paragraph and character styles, theme color resolution +- **Lists** with automatic numbering ID management +- **Template engine** with placeholder replacement (plain text and regex) +- **TOC generation** with internal hyperlinks and dot-leader tabs +- **Document merging** with style deduplication and numbering remapping + +## Output Formats + +- **PDF** — built-in layout engine with text shaping (rustybuzz), Unicode line breaking, multi-section pagination, font subsetting, ToUnicode CMap, bookmarks, and images +- **HTML** — semantic mapping from OOXML with CSS styling and base64-embedded images +- **Markdown** — GFM-compatible output with pipe tables and formatting +- **PNG** — page-to-image rendering via tiny-skia rasterizer + +## Extras + +- **WASM support** via standalone `rdocx-wasm` crate +- **CLI tool** (`rdocx-cli`) — inspect, convert, diff, replace, validate, render + +## Installation + +```toml +[dependencies] +rdocx = "0.1" +``` + +To include bundled metric-compatible fonts (Carlito, Caladea, Liberation family): + +```toml +[dependencies] +rdocx-layout = { version = "0.1", features = ["bundled-fonts"] } +``` + +## Quick Start + +### Create a document + +```rust +use rdocx::{Document, Length}; + +let mut doc = Document::new(); + +doc.add_paragraph("Hello, World!"); + +let mut para = doc.add_paragraph(""); +para.add_run("Bold text").bold(true); +para.add_run(" and "); +para.add_run("italic text").italic(true); + +doc.add_table(3, 4); + +doc.save("output.docx").unwrap(); +``` + +### Read a document + +```rust +use rdocx::Document; + +let doc = Document::open("report.docx").unwrap(); + +for para in doc.paragraphs() { + println!("{}", para.text()); +} + +for table in doc.tables() { + for row in table.rows() { + for cell in row.cells() { + print!("{}\t", cell.text()); + } + println!(); + } +} +``` + +### Convert to PDF + +```rust +use rdocx::Document; + +let doc = Document::open("report.docx").unwrap(); +doc.save_pdf("report.pdf").unwrap(); + +// Or get bytes directly +let pdf_bytes = doc.to_pdf().unwrap(); +``` + +### Convert to HTML / Markdown + +```rust +use rdocx::Document; + +let doc = Document::open("report.docx").unwrap(); + +let html = doc.to_html(); +let markdown = doc.to_markdown(); +``` + +### Template replacement + +```rust +use rdocx::Document; +use std::collections::HashMap; + +let mut doc = Document::open("template.docx").unwrap(); + +let mut replacements = HashMap::new(); +replacements.insert("{{name}}", "Jane Doe"); +replacements.insert("{{date}}", "2025-01-15"); +doc.replace_all(&replacements); + +doc.save("filled.docx").unwrap(); +``` + +### Merge documents + +```rust +use rdocx::{Document, SectionBreak}; + +let mut doc = Document::open("part1.docx").unwrap(); +let part2 = Document::open("part2.docx").unwrap(); + +doc.append_with_break(&part2, SectionBreak::NextPage); +doc.save("combined.docx").unwrap(); +``` + +## CLI + +Install the CLI: + +```sh +cargo install rdocx-cli +``` + +```sh +# Inspect document structure +rdocx inspect report.docx + +# Extract plain text +rdocx text report.docx + +# Convert to PDF +rdocx convert report.docx -o report.pdf + +# Convert to HTML or Markdown +rdocx convert report.docx -o report.html +rdocx convert report.docx -o report.md + +# Find and replace text +rdocx replace report.docx --find "Draft" --replace "Final" -o final.docx + +# Diff two documents +rdocx diff v1.docx v2.docx +``` + +## How rdocx Compares + +### vs. Python Libraries + +| | rdocx | python-docx | docx2pdf | pypandoc | +|---|---|---|---|---| +| Create DOCX | Yes | Yes | -- | -- | +| Read DOCX | Yes | Yes | -- | -- | +| DOCX to PDF | Yes (built-in) | No | Via MS Word | Via Pandoc + LaTeX | +| DOCX to HTML | Yes (built-in) | No | No | Yes (lossy) | +| DOCX to Markdown | Yes (built-in) | No | No | Yes (lossy) | +| Layout engine | Yes | None | Delegates to Word | Delegates to LaTeX | +| External runtime | **None** | None (but no PDF) | **MS Word required** | **Pandoc + LaTeX** | +| Install size | **4 MB binary** | ~5 MB | ~31 KB + Word | 300-650 MB | +| Runs in Docker / CI | Yes | Yes (no PDF) | No | Yes (huge image) | +| WASM / browser | Yes | No | No | No | + +**python-docx** is the most popular DOCX library in any language (~14M PyPI downloads/month), but it has **zero conversion capabilities** — no PDF, no HTML, no Markdown. Users who need PDF must bolt on a separate tool like LibreOffice (~500 MB) or a commercial API. rdocx gives you the same read/write API *plus* built-in conversion in a single 4 MB binary. + +### vs. Java Libraries + +| | rdocx | Apache POI | docx4j | Aspose.Words | +|---|---|---|---|---| +| Create DOCX | Yes | Yes | Yes | Yes | +| Read DOCX | Yes | Yes | Yes | Yes | +| PDF (built-in) | Yes | No | Via FOP (limited) | Yes (high fidelity) | +| HTML (built-in) | Yes | No | Yes | Yes | +| License | MIT / Apache-2.0 | Apache-2.0 | Apache-2.0 | **$1,199+** | +| Total dependency size | **4 MB** | 18-28 MB + JRE | 50-80 MB + JRE | 14 MB + JRE | +| Typical memory (moderate doc) | **10-50 MB** | 256 MB - 1 GB | 256 MB - 2 GB | 50-300 MB | +| Cold start | **< 10 ms** | 2-5 sec | 2-5 sec | 2-5 sec | +| Runtime required | None | JVM (~200 MB) | JVM (~200 MB) | JVM (~200 MB) | + +Java solutions carry the JVM's baseline overhead: 50-100 MB of RAM before a single document is loaded, and 2-5 second cold starts from class loading. Apache POI has **no built-in PDF** at all. docx4j's FOP pipeline is acknowledged by its own maintainer as limited in fidelity. Aspose has excellent PDF output but costs $1,199+ per developer. rdocx delivers comparable capabilities as a zero-dependency native binary. + +### vs. Other Rust Crates + +| | rdocx | docx-rs | docx-rust | ooxmlsdk | +|---|---|---|---|---| +| Create DOCX | Yes | Yes | Yes | Low-level | +| Read DOCX | Yes | Yes | Yes | Low-level | +| Round-trip preservation | Yes | Limited | Limited | N/A | +| Tables, images, headers | Yes | Yes | Basic | Raw XML | +| PDF conversion | **Yes** | No | No | No | +| HTML / Markdown export | **Yes** | No | No | No | +| Layout engine | **Yes** | No | No | No | +| Page-to-image rendering | **Yes** | No | No | No | +| Template engine | **Yes** | No | No | No | +| Document merging | **Yes** | No | No | No | +| Regex find/replace | **Yes** | No | No | No | +| CLI tool | **Yes** | No | No | No | +| WASM | Yes | Yes | No | No | + +**docx-rs** (1M+ downloads, 500+ stars) is the most popular Rust DOCX crate, but it is a read/write library only — no conversion, no layout engine, no PDF. The same is true for every other Rust DOCX crate. rdocx is the only Rust crate that combines DOCX read/write with a built-in layout engine and multi-format output (PDF, HTML, Markdown, PNG). + +### Resource Footprint + +| Metric | rdocx (native) | Python + LibreOffice | Java (POI + FOP) | +|---|---|---|---| +| Binary / install size | **4 MB** | ~500 MB | ~250 MB (JARs + JRE) | +| Memory (moderate document) | **10-50 MB** | ~200-500 MB | ~300 MB - 1.5 GB | +| Cold start | **< 10 ms** | ~2-4 sec (LibreOffice) | ~2-5 sec (JVM) | +| Serverless / Lambda friendly | Yes | Difficult | Difficult | +| Docker image overhead | **~10 MB** (musl static) | ~500 MB+ | ~250 MB+ | +| WASM compatible | Yes | No | No | + +## Crate Architecture + +| Crate | Purpose | +|---|---| +| `rdocx` | High-level Document API | +| `rdocx-opc` | OPC/ZIP package I/O | +| `rdocx-oxml` | OOXML types (CT_Document, CT_PPr, CT_RPr, CT_Tbl, ...) | +| `rdocx-layout` | Layout engine (text shaping, line breaking, pagination) | +| `rdocx-pdf` | PDF rendering with font subsetting | +| `rdocx-html` | HTML and Markdown conversion | +| `rdocx-cli` | CLI binary | +| `rdocx-wasm` | WASM bindings (standalone, excluded from workspace) | + +## Minimum Supported Rust Version + +1.93 (edition 2024) + +## License + +Licensed under either of + +- MIT license ([LICENSE](LICENSE) or http://opensource.org/licenses/MIT) +- Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +at your option. diff --git a/vendor/rdocx/examples/convert_html_md.rs b/vendor/rdocx/examples/convert_html_md.rs new file mode 100644 index 00000000..3768e470 --- /dev/null +++ b/vendor/rdocx/examples/convert_html_md.rs @@ -0,0 +1,22 @@ +use rdocx::Document; + +fn main() { + let doc = Document::open("samples/feature_showcase.docx").expect("Failed to open document"); + + let html = doc.to_html(); + std::fs::write("/tmp/feature_showcase.html", &html).expect("Failed to write HTML"); + println!("HTML: {} bytes -> /tmp/feature_showcase.html", html.len()); + + let md = doc.to_markdown(); + std::fs::write("/tmp/feature_showcase.md", &md).expect("Failed to write Markdown"); + println!("Markdown: {} bytes -> /tmp/feature_showcase.md", md.len()); + + // Simple document + let mut simple = Document::new(); + simple.add_paragraph("Hello, World!"); + let html = simple.to_html(); + println!("\n--- Simple HTML ---\n{html}"); + + let md = simple.to_markdown(); + println!("--- Simple Markdown ---\n{md}"); +} diff --git a/vendor/rdocx/examples/generate_all_samples.rs b/vendor/rdocx/examples/generate_all_samples.rs new file mode 100644 index 00000000..5d25dc06 --- /dev/null +++ b/vendor/rdocx/examples/generate_all_samples.rs @@ -0,0 +1,2453 @@ +//! Generates all sample documents, each exported to DOCX, PDF, HTML, and Markdown. +//! +//! Documents: +//! 1. feature_showcase — Updated to cover ALL library features +//! 2. proposal — Professional project proposal (Tensorbee) +//! 3. quote — Quote with bill of materials +//! 4. invoice — Professional invoice +//! 5. report — Hierarchical report with images +//! 6. letter — Formal business letter +//! 7. contract — Employment contract +//! +//! Run with: cargo run --example generate_all_samples + +use std::collections::HashMap; +use std::fmt::Write as FmtWrite; +use std::path::Path; + +use rdocx::{ + Alignment, BorderStyle, Document, Length, SectionBreak, StyleBuilder, TabAlignment, TabLeader, + VerticalAlignment, +}; + +fn main() { + let samples_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("samples"); + std::fs::create_dir_all(&samples_dir).unwrap(); + + println!( + "Generating all sample documents in {}\n", + samples_dir.display() + ); + + let generators: Vec<(&str, fn(&Path) -> Document)> = vec![ + ("feature_showcase", generate_feature_showcase), + ("proposal", generate_proposal), + ("quote", generate_quote), + ("invoice", generate_invoice), + ("report", generate_report), + ("letter", generate_letter), + ("contract", generate_contract), + ]; + + for (name, generator) in &generators { + println!("--- {} ---", name); + let doc = generator(&samples_dir); + export_all(&samples_dir, name, doc); + println!(); + } + + println!("All done!"); +} + +/// Export a document to DOCX, PDF, HTML, and Markdown. +fn export_all(dir: &Path, name: &str, mut doc: Document) { + let docx_path = dir.join(format!("{name}.docx")); + doc.save(&docx_path).unwrap(); + println!(" {name}.docx"); + + match doc.to_pdf() { + Ok(pdf) => { + let pdf_path = dir.join(format!("{name}.pdf")); + std::fs::write(&pdf_path, &pdf).unwrap(); + println!(" {name}.pdf ({} bytes)", pdf.len()); + } + Err(e) => println!(" {name}.pdf — skipped: {e}"), + } + + let html = doc.to_html(); + let html_path = dir.join(format!("{name}.html")); + std::fs::write(&html_path, &html).unwrap(); + println!(" {name}.html ({} bytes)", html.len()); + + let md = doc.to_markdown(); + let md_path = dir.join(format!("{name}.md")); + std::fs::write(&md_path, &md).unwrap(); + println!(" {name}.md ({} bytes)", md.len()); +} + +// ============================================================================= +// 1. FEATURE SHOWCASE — Updated to cover ALL library features +// ============================================================================= +fn generate_feature_showcase(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // ── Page Setup & Metadata ── + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + ); + doc.set_header_footer_distance(Length::twips(720), Length::twips(432)); + doc.set_gutter(Length::twips(0)); + doc.set_title("rdocx Feature Showcase"); + doc.set_author("rdocx Sample Generator"); + doc.set_subject("Comprehensive feature demonstration"); + doc.set_keywords("rdocx, docx, rust, sample, showcase"); + + // Headers & Footers with different first page + doc.set_different_first_page(true); + doc.set_first_page_header("rdocx"); + doc.set_first_page_footer("Feature Showcase — Cover Page"); + doc.set_header("rdocx Feature Showcase"); + doc.set_footer("Generated by rdocx"); + + // ── COVER PAGE ── + let bg = create_sample_png(612, 792, [20, 45, 90]); + doc.add_background_image(&bg, "cover_bg.png"); + + for _ in 0..3 { + doc.add_paragraph(""); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("rdocx").bold(true).size(72.0).color("FFFFFF"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Complete Feature Showcase") + .size(28.0) + .color("FFFFFF") + .italic(true); + } + doc.add_paragraph(""); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Every feature of the rdocx Rust library") + .size(13.0) + .color("B0C4DE"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("demonstrated in a single document.") + .size(13.0) + .color("B0C4DE"); + } + + // ── TABLE OF CONTENTS ── + doc.add_paragraph("").page_break_before(true); + doc.insert_toc(doc.content_count(), 3); + + // ── SECTION 1: TEXT FORMATTING ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("1. Text Formatting").style("Heading1"); + + // Paragraph Alignment + doc.add_paragraph("1.1 Paragraph Alignment") + .style("Heading2"); + doc.add_paragraph("Left-aligned paragraph (default).") + .alignment(Alignment::Left); + doc.add_paragraph("Center-aligned paragraph.") + .alignment(Alignment::Center); + doc.add_paragraph("Right-aligned paragraph.") + .alignment(Alignment::Right); + doc.add_paragraph( + "Justified paragraph — this text is long enough to demonstrate how justified alignment \ + distributes extra space across word gaps so lines fill the full width of the text area.", + ) + .alignment(Alignment::Justify); + + doc.add_paragraph(""); + + // Run Formatting + doc.add_paragraph("1.2 Run Formatting").style("Heading2"); + { + let mut p = doc.add_paragraph(""); + p.add_run("Bold").bold(true); + p.add_run(" | "); + p.add_run("Italic").italic(true); + p.add_run(" | "); + p.add_run("Bold Italic").bold(true).italic(true); + p.add_run(" | "); + p.add_run("Underline").underline(true); + p.add_run(" | "); + p.add_run("Strikethrough").strike(true); + p.add_run(" | "); + p.add_run("Double Strike").double_strike(true); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Red").color("FF0000"); + p.add_run(" | "); + p.add_run("Blue").color("0000FF"); + p.add_run(" | "); + p.add_run("Green").color("00AA00"); + p.add_run(" | "); + p.add_run("Highlighted").highlight("FFFF00"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("8pt").size(8.0); + p.add_run(" | "); + p.add_run("11pt").size(11.0); + p.add_run(" | "); + p.add_run("16pt").size(16.0); + p.add_run(" | "); + p.add_run("24pt").size(24.0); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Arial").font("Arial"); + p.add_run(" | "); + p.add_run("Times New Roman").font("Times New Roman"); + p.add_run(" | "); + p.add_run("Courier New").font("Courier New"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("H"); + p.add_run("2").subscript(); + p.add_run("O (subscript) | E=mc"); + p.add_run("2").superscript(); + p.add_run(" (superscript)"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("ALL CAPS").all_caps(true); + p.add_run(" | "); + p.add_run("Small Caps").small_caps(true); + p.add_run(" | "); + p.add_run("Expanded").character_spacing(Length::twips(40)); + p.add_run(" | "); + p.add_run("Hidden text (not visible)").hidden(true); + } + + doc.add_paragraph(""); + + // Underline Styles + doc.add_paragraph("1.3 Underline Styles").style("Heading2"); + { + let mut p = doc.add_paragraph(""); + p.add_run("Single ") + .underline_style(rdocx::UnderlineStyle::Single); + p.add_run("Double ") + .underline_style(rdocx::UnderlineStyle::Double); + p.add_run("Thick ") + .underline_style(rdocx::UnderlineStyle::Thick); + p.add_run("Dotted ") + .underline_style(rdocx::UnderlineStyle::Dotted); + p.add_run("Dash ") + .underline_style(rdocx::UnderlineStyle::Dash); + p.add_run("Wave ") + .underline_style(rdocx::UnderlineStyle::Wave); + } + + // ── SECTION 2: PARAGRAPH FORMATTING ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("2. Paragraph Formatting") + .style("Heading1"); + + doc.add_paragraph("2.1 Shading & Borders").style("Heading2"); + doc.add_paragraph("Paragraph with light green shading") + .shading("E2EFDA"); + doc.add_paragraph("Paragraph with bottom border") + .border_bottom(BorderStyle::Single, 6, "2E75B6"); + doc.add_paragraph("Paragraph with all borders (red)") + .border_all(BorderStyle::Single, 4, "FF0000"); + + doc.add_paragraph("2.2 Indentation").style("Heading2"); + doc.add_paragraph("1-inch left indent + 0.5-inch hanging indent") + .indent_left(Length::inches(1.0)) + .hanging_indent(Length::inches(0.5)); + doc.add_paragraph("0.5-inch first-line indent on this paragraph") + .first_line_indent(Length::inches(0.5)); + doc.add_paragraph("Both left and right indent (0.75 inches each)") + .indent_left(Length::inches(0.75)) + .indent_right(Length::inches(0.75)); + + doc.add_paragraph("2.3 Spacing & Line Height") + .style("Heading2"); + doc.add_paragraph("Extra space before (24pt) and after (12pt)") + .space_before(Length::pt(24.0)) + .space_after(Length::pt(12.0)); + doc.add_paragraph( + "Double line spacing paragraph. This text should have extra vertical space between \ + lines to demonstrate line_spacing_multiple(2.0).", + ) + .line_spacing_multiple(2.0); + doc.add_paragraph("Exact 20pt line spacing (fixed height).") + .line_spacing(20.0); + + doc.add_paragraph("2.4 Pagination Controls") + .style("Heading2"); + doc.add_paragraph("keep_with_next — this paragraph stays with the next") + .keep_with_next(true); + doc.add_paragraph("(This paragraph was kept with the one above.)"); + doc.add_paragraph("keep_together — all lines of this paragraph stay on the same page") + .keep_together(true); + doc.add_paragraph("widow_control — prevents widow/orphan lines") + .widow_control(true); + + // ── SECTION 3: LISTS ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("3. Lists").style("Heading1"); + + doc.add_paragraph("3.1 Bullet Lists").style("Heading2"); + doc.add_bullet_list_item("First item", 0); + doc.add_bullet_list_item("Second item", 0); + doc.add_bullet_list_item("Nested level 1", 1); + doc.add_bullet_list_item("Nested level 2", 2); + doc.add_bullet_list_item("Back to level 1", 1); + doc.add_bullet_list_item("Third item", 0); + + doc.add_paragraph(""); + + doc.add_paragraph("3.2 Numbered Lists").style("Heading2"); + doc.add_numbered_list_item("First numbered item", 0); + doc.add_numbered_list_item("Second numbered item", 0); + doc.add_numbered_list_item("Sub-item A", 1); + doc.add_numbered_list_item("Sub-item B", 1); + doc.add_numbered_list_item("Third numbered item", 0); + + // ── SECTION 4: TAB STOPS ── + doc.add_paragraph(""); + doc.add_paragraph("4. Tab Stops").style("Heading1"); + + doc.add_paragraph("4.1 Alignment Tabs").style("Heading2"); + doc.add_paragraph("Left\tCenter\tRight\tDecimal") + .add_tab_stop(TabAlignment::Left, Length::inches(0.0)) + .add_tab_stop(TabAlignment::Center, Length::inches(2.5)) + .add_tab_stop(TabAlignment::Right, Length::inches(5.0)) + .add_tab_stop(TabAlignment::Decimal, Length::inches(6.5)); + + doc.add_paragraph("4.2 Tab Leaders").style("Heading2"); + doc.add_paragraph("Item\t\tPrice") + .add_tab_stop_with_leader(TabAlignment::Left, Length::inches(0.0), TabLeader::None) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(4.0), TabLeader::Dot) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(5.0), TabLeader::None); + doc.add_paragraph("Widget\t\t$19.99") + .add_tab_stop_with_leader(TabAlignment::Left, Length::inches(0.0), TabLeader::None) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(4.0), TabLeader::Dot) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(5.0), TabLeader::None); + doc.add_paragraph("Gadget\t\t$249.50") + .add_tab_stop_with_leader(TabAlignment::Left, Length::inches(0.0), TabLeader::None) + .add_tab_stop_with_leader( + TabAlignment::Right, + Length::inches(4.0), + TabLeader::Underscore, + ) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(5.0), TabLeader::None); + + // ── SECTION 5: TABLES ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("5. Tables").style("Heading1"); + + doc.add_paragraph("5.1 Basic Table").style("Heading2"); + { + let mut tbl = doc + .add_table(4, 3) + .borders(BorderStyle::Single, 4, "000000"); + for col in 0..3 { + tbl.cell(0, col).unwrap().shading("2E75B6"); + } + tbl.cell(0, 0).unwrap().set_text("Name"); + tbl.cell(0, 1).unwrap().set_text("Role"); + tbl.cell(0, 2).unwrap().set_text("Location"); + tbl.cell(1, 0).unwrap().set_text("Walter White"); + tbl.cell(1, 1).unwrap().set_text("CEO"); + tbl.cell(1, 2).unwrap().set_text("Albuquerque"); + tbl.cell(2, 0).unwrap().set_text("Jesse Pinkman"); + tbl.cell(2, 1).unwrap().set_text("CTO"); + tbl.cell(2, 2).unwrap().set_text("Remote"); + tbl.cell(3, 0).unwrap().set_text("Hank Schrader"); + tbl.cell(3, 1).unwrap().set_text("Security"); + tbl.cell(3, 2).unwrap().set_text("Washington"); + } + + doc.add_paragraph(""); + + doc.add_paragraph("5.2 Column Span & Cell Shading") + .style("Heading2"); + { + let mut tbl = doc + .add_table(3, 4) + .borders(BorderStyle::Single, 4, "000000") + .width_pct(100.0); + tbl.cell(0, 0).unwrap().set_text("Quarterly Report"); + tbl.cell(0, 0).unwrap().shading("1F4E79").grid_span(4); + tbl.cell(1, 0).unwrap().set_text("Region"); + tbl.cell(1, 0).unwrap().shading("D6E4F0"); + tbl.cell(1, 1).unwrap().set_text("Q1"); + tbl.cell(1, 1).unwrap().shading("D6E4F0"); + tbl.cell(1, 2).unwrap().set_text("Q2"); + tbl.cell(1, 2).unwrap().shading("D6E4F0"); + tbl.cell(1, 3).unwrap().set_text("Total"); + tbl.cell(1, 3).unwrap().shading("D6E4F0"); + tbl.cell(2, 0).unwrap().set_text("Americas"); + tbl.cell(2, 1).unwrap().set_text("$2.4M"); + tbl.cell(2, 2).unwrap().set_text("$2.7M"); + tbl.cell(2, 3).unwrap().set_text("$5.1M"); + } + + doc.add_paragraph(""); + + doc.add_paragraph("5.3 Vertical Merge").style("Heading2"); + { + let mut tbl = doc + .add_table(4, 3) + .borders(BorderStyle::Single, 4, "000000"); + tbl.cell(0, 0).unwrap().set_text("Category"); + tbl.cell(0, 0).unwrap().shading("E2EFDA"); + tbl.cell(0, 1).unwrap().set_text("Item"); + tbl.cell(0, 1).unwrap().shading("E2EFDA"); + tbl.cell(0, 2).unwrap().set_text("Price"); + tbl.cell(0, 2).unwrap().shading("E2EFDA"); + tbl.cell(1, 0).unwrap().set_text("Hardware"); + tbl.cell(1, 0).unwrap().v_merge_restart(); + tbl.cell(1, 1).unwrap().set_text("Laptop"); + tbl.cell(1, 2).unwrap().set_text("$1,200"); + tbl.cell(2, 0).unwrap().v_merge_continue(); + tbl.cell(2, 1).unwrap().set_text("Monitor"); + tbl.cell(2, 2).unwrap().set_text("$450"); + tbl.cell(3, 0).unwrap().set_text("Software"); + tbl.cell(3, 1).unwrap().set_text("IDE License"); + tbl.cell(3, 2).unwrap().set_text("$200/yr"); + } + + doc.add_paragraph(""); + + doc.add_paragraph("5.4 Nested Table").style("Heading2"); + { + let mut tbl = doc + .add_table(2, 2) + .borders(BorderStyle::Single, 6, "2E75B6"); + tbl.cell(0, 0).unwrap().set_text("Outer (0,0)"); + tbl.cell(0, 1).unwrap().set_text("Outer (0,1)"); + tbl.cell(1, 0).unwrap().set_text("Outer (1,0)"); + { + let mut cell = tbl.cell(1, 1).unwrap(); + cell.set_text("Nested table below:"); + let mut nested = cell + .add_table(2, 2) + .borders(BorderStyle::Single, 2, "FF6600"); + nested.cell(0, 0).unwrap().set_text("A"); + nested.cell(0, 1).unwrap().set_text("B"); + nested.cell(1, 0).unwrap().set_text("C"); + nested.cell(1, 1).unwrap().set_text("D"); + } + } + + doc.add_paragraph(""); + + doc.add_paragraph("5.5 Vertical Alignment & Row Properties") + .style("Heading2"); + { + let mut tbl = doc + .add_table(2, 3) + .borders(BorderStyle::Single, 4, "666666"); + tbl.row(0).unwrap().height(Length::pt(50.0)).header(); + tbl.cell(0, 0).unwrap().set_text("Top"); + tbl.cell(0, 0) + .unwrap() + .vertical_alignment(VerticalAlignment::Top) + .shading("FFF2CC"); + tbl.cell(0, 1).unwrap().set_text("Center"); + tbl.cell(0, 1) + .unwrap() + .vertical_alignment(VerticalAlignment::Center) + .shading("D9E2F3"); + tbl.cell(0, 2).unwrap().set_text("Bottom"); + tbl.cell(0, 2) + .unwrap() + .vertical_alignment(VerticalAlignment::Bottom) + .shading("E2EFDA"); + tbl.row(1).unwrap().cant_split(); + tbl.cell(1, 0).unwrap().set_text("No-wrap cell"); + tbl.cell(1, 0).unwrap().no_wrap(); + tbl.cell(1, 1).unwrap().set_text("Fixed width"); + tbl.cell(1, 1).unwrap().width(Length::inches(2.0)); + tbl.cell(1, 2).unwrap().set_text("Normal"); + } + + // ── SECTION 6: IMAGES ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("6. Images").style("Heading1"); + + doc.add_paragraph("6.1 Inline Image").style("Heading2"); + doc.add_paragraph("A blue gradient image below:"); + let img = create_sample_png(200, 50, [0, 80, 200]); + doc.add_picture(&img, "chart.png", Length::inches(3.0), Length::inches(0.75)); + + doc.add_paragraph(""); + doc.add_paragraph("6.2 Header Image").style("Heading2"); + let hdr_img = create_sample_png(400, 40, [40, 40, 40]); + doc.set_header_image( + &hdr_img, + "header_logo.png", + Length::inches(2.0), + Length::inches(0.2), + ); + doc.add_paragraph("The header now contains an inline image (check the top of this page)."); + + doc.add_paragraph(""); + doc.add_paragraph("Note: Page 1 uses a full-page background image (add_background_image)."); + + // ── SECTION 7: CONTENT MANIPULATION ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("7. Content Manipulation") + .style("Heading1"); + + doc.add_paragraph("7.1 Placeholder Replacement") + .style("Heading2"); + doc.add_paragraph("Customer: {{customer}}"); + doc.add_paragraph("Date: {{date}}"); + doc.add_paragraph("Reference: {{ref_number}}"); + { + let mut tbl = doc + .add_table(2, 2) + .borders(BorderStyle::Single, 4, "000000"); + tbl.cell(0, 0).unwrap().set_text("Project"); + tbl.cell(0, 0).unwrap().shading("D6E4F0"); + tbl.cell(0, 1).unwrap().set_text("{{project}}"); + tbl.cell(1, 0).unwrap().set_text("Status"); + tbl.cell(1, 0).unwrap().shading("D6E4F0"); + tbl.cell(1, 1).unwrap().set_text("{{status}}"); + } + + let mut replacements = HashMap::new(); + replacements.insert("{{customer}}", "Tensorbee Inc."); + replacements.insert("{{date}}", "February 22, 2026"); + replacements.insert("{{ref_number}}", "TB-2026-001"); + replacements.insert("{{project}}", "Infrastructure Upgrade"); + replacements.insert("{{status}}", "In Progress"); + let n = doc.replace_all(&replacements); + doc.add_paragraph(&format!("({n} placeholders replaced above)")); + + doc.add_paragraph(""); + + doc.add_paragraph("7.2 Regex Replacement").style("Heading2"); + doc.add_paragraph("Emails: user1@example.com and admin@tensorbee.com"); + let _ = doc.replace_regex(r"\b\w+@\w+\.\w+\b", "[REDACTED]"); + doc.add_paragraph("(Email addresses above were redacted using regex replacement)"); + + doc.add_paragraph(""); + + doc.add_paragraph("7.3 Content Insertion").style("Heading2"); + doc.add_paragraph("Section A: First content."); + doc.add_paragraph("Section C: Third content."); + if let Some(idx) = doc.find_content_index("Section C") { + doc.insert_paragraph( + idx, + "Section B: Inserted between A and C via find_content_index().", + ); + } + + // ── SECTION 8: SECTION BREAKS & ORIENTATION ── + doc.add_paragraph("").section_break(SectionBreak::NextPage); + doc.add_paragraph("8. Section Breaks & Orientation") + .style("Heading1"); + doc.add_paragraph("This page is LANDSCAPE. Useful for wide tables and charts."); + + { + let mut tbl = doc + .add_table(3, 7) + .borders(BorderStyle::Single, 4, "2E75B6"); + let headers = ["Region", "Jan", "Feb", "Mar", "Apr", "May", "Total"]; + for (c, h) in headers.iter().enumerate() { + tbl.cell(0, c).unwrap().set_text(h); + tbl.cell(0, c).unwrap().shading("2E75B6"); + } + let data = [ + [ + "Americas", "$1.2M", "$1.3M", "$1.4M", "$1.5M", "$1.6M", "$7.0M", + ], + [ + "Europe", "$0.8M", "$0.9M", "$0.9M", "$1.0M", "$1.1M", "$4.7M", + ], + ]; + for (r, row) in data.iter().enumerate() { + for (c, v) in row.iter().enumerate() { + tbl.cell(r + 1, c).unwrap().set_text(v); + } + } + } + + doc.add_paragraph("") + .section_break(SectionBreak::NextPage) + .section_landscape(); + + // ── SECTION 9: CUSTOM STYLES ── + doc.add_paragraph("9. Custom Styles").style("Heading1"); + doc.add_style( + StyleBuilder::paragraph("CustomHighlight", "Custom Highlight") + .based_on("Normal") + .paragraph_properties({ + let mut ppr = rdocx_oxml::properties::CT_PPr::default(); + ppr.shading = Some(rdocx_oxml::properties::CT_Shd { + val: "clear".to_string(), + color: None, + fill: Some("FFF2CC".to_string()), + }); + ppr + }) + .run_properties({ + let mut rpr = rdocx_oxml::properties::CT_RPr::default(); + rpr.bold = Some(true); + rpr.color = Some("C45911".to_string()); + rpr + }), + ); + doc.add_paragraph("This paragraph uses a custom style: bold orange text on yellow background.") + .style("CustomHighlight"); + + doc.add_paragraph(""); + + // ── SECTION 10: DOCUMENT INTELLIGENCE ── + doc.add_paragraph("10. Document Intelligence API") + .style("Heading1"); + + let wc = doc.word_count(); + let hc = doc.headings().len(); + let ic = doc.images().len(); + let lc = doc.links().len(); + doc.add_paragraph(&format!("Word count: {wc}")); + doc.add_paragraph(&format!("Heading count: {hc}")); + doc.add_paragraph(&format!("Image count: {ic}")); + doc.add_paragraph(&format!("Link count: {lc}")); + + let outline = doc.document_outline(); + doc.add_paragraph(&format!("Top-level outline nodes: {}", outline.len())); + + let issues = doc.audit_accessibility(); + doc.add_paragraph(&format!("Accessibility issues: {}", issues.len())); + for issue in &issues { + doc.add_bullet_list_item(&format!("{:?}: {}", issue.severity, issue.message), 0); + } + + // ── SECTION 11: DOCUMENT MERGING ── + doc.add_paragraph(""); + doc.add_paragraph("11. Document Merging").style("Heading1"); + { + let mut other = Document::new(); + other.add_paragraph("This paragraph was merged from another document using append()."); + doc.append(&other); + } + + doc.add_paragraph(""); + + // ── SUMMARY ── + doc.add_paragraph("Summary of Demonstrated Features") + .style("Heading1"); + let features = [ + "Page setup: size, margins, header/footer distance, gutter", + "Document metadata: title, author, subject, keywords", + "Headers and footers: text, images, different first page", + "Background images (full-page behind text)", + "Table of Contents generation with bookmarks", + "Text formatting: bold, italic, underline styles, strike, color, size, font", + "Advanced run: superscript, subscript, caps, small caps, spacing, hidden", + "Paragraph formatting: alignment, borders, shading, spacing, indentation", + "Pagination controls: keep-with-next, keep-together, widow control", + "Bullet and numbered lists with nesting", + "Tab stops with dot/underscore/hyphen leaders", + "Tables: borders, shading, column spans, row spans, vertical alignment, nested tables", + "Row properties: header rows, exact height, cant-split", + "Cell properties: width, no-wrap, vertical alignment", + "Inline images and header images", + "Placeholder replacement (single and batch)", + "Regex-based find and replace", + "Content insertion at specific positions", + "Section breaks with mixed portrait/landscape", + "Custom paragraph and character styles", + "Document intelligence: word count, headings, outline, images, links", + "Accessibility audit", + "Document merging (append)", + "Export: DOCX, PDF, HTML, Markdown", + ]; + for f in &features { + doc.add_bullet_list_item(f, 0); + } + doc.add_paragraph(""); + doc.add_paragraph("All features built entirely with the rdocx Rust crate.") + .alignment(Alignment::Center) + .shading("E2EFDA") + .border_all(BorderStyle::Single, 2, "00AA00"); + + doc +} + +// ============================================================================= +// 2. PROPOSAL DOCUMENT — Deep navy + gold scheme +// ============================================================================= +fn generate_proposal(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // Colors: Navy #1B2A4A, Gold #C5922E, Light #F4F1EB + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + ); + doc.set_title("AI Platform Modernization Proposal"); + doc.set_author("Walter White"); + doc.set_subject("Technology Proposal"); + doc.set_keywords("proposal, AI, modernization, Tensorbee"); + + // Banner header + let logo_img = create_logo_png(220, 48); + let banner = build_header_banner_xml( + "rId1", + &BannerOpts { + bg_color: "1B2A4A", + banner_width: 7772400, + banner_height: 914400, + logo_width: 2011680, + logo_height: 438912, + logo_x_offset: 295125, + logo_y_offset: 237744, + }, + ); + doc.set_raw_header_with_images( + banner, + &[("rId1", &logo_img, "logo.png")], + rdocx_oxml::header_footer::HdrFtrType::Default, + ); + doc.set_different_first_page(true); + let cover_banner = build_header_banner_xml( + "rId1", + &BannerOpts { + bg_color: "C5922E", + banner_width: 7772400, + banner_height: 914400, + logo_width: 2011680, + logo_height: 438912, + logo_x_offset: 295125, + logo_y_offset: 237744, + }, + ); + doc.set_raw_header_with_images( + cover_banner, + &[("rId1", &logo_img, "logo.png")], + rdocx_oxml::header_footer::HdrFtrType::First, + ); + doc.set_margins( + Length::twips(2292), + Length::twips(1440), + Length::twips(1440), + Length::twips(1440), + ); + doc.set_header_footer_distance(Length::twips(720), Length::twips(432)); + doc.set_footer("Tensorbee — Confidential"); + + // Custom styles + doc.add_style( + StyleBuilder::paragraph("ProposalTitle", "Proposal Title") + .based_on("Normal") + .run_properties({ + let mut rpr = rdocx_oxml::properties::CT_RPr::default(); + rpr.bold = Some(true); + rpr.font_ascii = Some("Georgia".to_string()); + rpr.font_hansi = Some("Georgia".to_string()); + rpr.sz = Some(rdocx_oxml::HalfPoint(56)); // half-points → 28pt + rpr.color = Some("1B2A4A".to_string()); + rpr + }), + ); + + // ── Cover Page ── + for _ in 0..4 { + doc.add_paragraph(""); + } + doc.add_paragraph("AI Platform Modernization") + .style("ProposalTitle") + .alignment(Alignment::Center); + doc.add_paragraph("") + .alignment(Alignment::Center) + .border_bottom(BorderStyle::Single, 8, "C5922E"); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Prepared for Global Financial Corp.") + .size(14.0) + .color("666666") + .italic(true); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Prepared by: Walter White, CEO — Tensorbee") + .size(12.0) + .color("888888"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("February 22, 2026").size(12.0).color("888888"); + } + + // ── TOC ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Table of Contents").style("Heading1"); + // We'll add a manual-style TOC since the insert_toc goes at a specific position + doc.insert_toc(doc.content_count(), 2); + + // ── Executive Summary ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Executive Summary").style("Heading1"); + doc.add_paragraph( + "Tensorbee proposes a comprehensive modernization of Global Financial Corp's AI platform, \ + transitioning from legacy batch-processing systems to a real-time inference architecture. \ + This transformation will reduce model deployment time from weeks to hours, improve \ + prediction accuracy by an estimated 23%, and deliver $4.2M in annual operational savings.", + ) + .first_line_indent(Length::inches(0.3)); + doc.add_paragraph( + "The project spans three phases over 18 months, with a total investment of $2.8M. \ + Tensorbee brings deep expertise in ML infrastructure, having successfully completed \ + similar transformations for three Fortune 500 financial institutions.", + ) + .first_line_indent(Length::inches(0.3)); + + // Key metrics table + doc.add_paragraph(""); + { + let mut tbl = doc + .add_table(5, 2) + .borders(BorderStyle::Single, 4, "1B2A4A") + .width_pct(80.0); + tbl.cell(0, 0).unwrap().set_text("Key Metric"); + tbl.cell(0, 0).unwrap().shading("1B2A4A"); + tbl.cell(0, 1).unwrap().set_text("Value"); + tbl.cell(0, 1).unwrap().shading("1B2A4A"); + let rows = [ + ("Total Investment", "$2.8M"), + ("Annual Savings", "$4.2M"), + ("ROI (Year 1)", "150%"), + ("Timeline", "18 months"), + ]; + for (i, (k, v)) in rows.iter().enumerate() { + tbl.cell(i + 1, 0).unwrap().set_text(k); + if i % 2 == 0 { + tbl.cell(i + 1, 0).unwrap().shading("F4F1EB"); + } + tbl.cell(i + 1, 1).unwrap().set_text(v); + if i % 2 == 0 { + tbl.cell(i + 1, 1).unwrap().shading("F4F1EB"); + } + } + } + + // ── Problem Statement ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Problem Statement").style("Heading1"); + doc.add_paragraph( + "Global Financial Corp's current AI infrastructure faces three critical challenges:", + ); + doc.add_numbered_list_item("Deployment Latency — New models take 3-4 weeks to deploy to production, compared to the industry benchmark of 2-3 days.", 0); + doc.add_numbered_list_item("Scalability Constraints — The batch processing architecture cannot handle real-time inference demands during peak trading hours.", 0); + doc.add_numbered_list_item("Technical Debt — Legacy Python 2.7 codebases and manual deployment scripts create reliability risks and slow iteration speed.", 0); + + // ── Proposed Solution ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Proposed Solution").style("Heading1"); + + doc.add_paragraph("Phase 1: Foundation (Months 1-6)") + .style("Heading2"); + doc.add_bullet_list_item("Deploy Kubernetes-based ML serving infrastructure", 0); + doc.add_bullet_list_item("Implement CI/CD pipeline for model deployment", 0); + doc.add_bullet_list_item("Migrate top 5 production models to new platform", 0); + + doc.add_paragraph("Phase 2: Expansion (Months 7-12)") + .style("Heading2"); + doc.add_bullet_list_item( + "Real-time feature engineering pipeline (Apache Kafka + Flink)", + 0, + ); + doc.add_bullet_list_item("A/B testing framework for model variants", 0); + doc.add_bullet_list_item("Automated model monitoring and drift detection", 0); + + doc.add_paragraph("Phase 3: Optimization (Months 13-18)") + .style("Heading2"); + doc.add_bullet_list_item("GPU inference optimization (TensorRT/ONNX Runtime)", 0); + doc.add_bullet_list_item("Multi-region deployment for latency reduction", 0); + doc.add_bullet_list_item( + "Self-service model deployment portal for data science teams", + 0, + ); + + // ── Budget ── + doc.add_paragraph("Budget Breakdown").style("Heading1"); + { + let mut tbl = doc + .add_table(6, 3) + .borders(BorderStyle::Single, 4, "1B2A4A") + .width_pct(100.0); + tbl.cell(0, 0).unwrap().set_text("Category"); + tbl.cell(0, 0).unwrap().shading("1B2A4A"); + tbl.cell(0, 1).unwrap().set_text("Cost"); + tbl.cell(0, 1).unwrap().shading("1B2A4A"); + tbl.cell(0, 2).unwrap().set_text("Notes"); + tbl.cell(0, 2).unwrap().shading("1B2A4A"); + let items = [ + ( + "Infrastructure", + "$450,000", + "Cloud compute + GPU instances", + ), + ("Engineering Services", "$1,600,000", "12 FTE x 18 months"), + ( + "Software Licenses", + "$350,000", + "Kafka, monitoring, CI/CD tools", + ), + ( + "Training & Enablement", + "$200,000", + "Team training, documentation", + ), + ("Contingency (10%)", "$200,000", "Risk buffer"), + ]; + for (i, (cat, cost, note)) in items.iter().enumerate() { + tbl.cell(i + 1, 0).unwrap().set_text(cat); + tbl.cell(i + 1, 1).unwrap().set_text(cost); + tbl.cell(i + 1, 2).unwrap().set_text(note); + if i % 2 == 0 { + tbl.cell(i + 1, 0).unwrap().shading("F4F1EB"); + tbl.cell(i + 1, 1).unwrap().shading("F4F1EB"); + tbl.cell(i + 1, 2).unwrap().shading("F4F1EB"); + } + } + } + + // ── Closing ── + doc.add_paragraph(""); + doc.add_paragraph("Next Steps").style("Heading1"); + doc.add_numbered_list_item( + "Schedule technical deep-dive with GFC engineering team (Week 1)", + 0, + ); + doc.add_numbered_list_item("Finalize scope and sign SOW (Week 2-3)", 0); + doc.add_numbered_list_item("Kick off Phase 1 with joint planning session (Week 4)", 0); + + doc.add_paragraph(""); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Walter White") + .bold(true) + .size(14.0) + .color("1B2A4A"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("CEO, Tensorbee | walter@tensorbee.com") + .size(10.0) + .color("888888"); + } + + doc +} + +// ============================================================================= +// 3. QUOTE / BILL OF MATERIALS — Teal + orange scheme +// ============================================================================= +fn generate_quote(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // Colors: Teal #008B8B, Dark #1A3C3C, Orange #E07020, Light #F0F8F8 + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(0.75), + Length::inches(0.75), + Length::inches(0.75), + Length::inches(0.75), + ); + doc.set_title("Quotation QT-2026-0147"); + doc.set_author("Walter White"); + doc.set_keywords("quote, BOM, Tensorbee"); + + doc.set_header("Tensorbee — Quotation"); + doc.set_footer("QT-2026-0147 | Page"); + + // ── Header Block ── + { + let mut p = doc.add_paragraph("").alignment(Alignment::Left); + p.add_run("TENSORBEE") + .bold(true) + .size(28.0) + .color("008B8B") + .font("Helvetica"); + } + doc.add_paragraph("123 Innovation Drive, Suite 400") + .alignment(Alignment::Left); + doc.add_paragraph("San Francisco, CA 94105 | +1 (415) 555-0199") + .alignment(Alignment::Left); + doc.add_paragraph("") + .border_bottom(BorderStyle::Single, 8, "008B8B"); + + // Quote meta + doc.add_paragraph(""); + { + let mut tbl = doc.add_table(4, 4).width_pct(100.0); + tbl.cell(0, 0).unwrap().set_text("QUOTATION"); + tbl.cell(0, 0).unwrap().shading("008B8B").grid_span(2); + tbl.cell(0, 2).unwrap().set_text("DATE"); + tbl.cell(0, 2).unwrap().shading("008B8B"); + tbl.cell(0, 3).unwrap().set_text("VALID UNTIL"); + tbl.cell(0, 3).unwrap().shading("008B8B"); + tbl.cell(1, 0).unwrap().set_text("Quote #:"); + tbl.cell(1, 1).unwrap().set_text("QT-2026-0147"); + tbl.cell(1, 2).unwrap().set_text("Feb 22, 2026"); + tbl.cell(1, 3).unwrap().set_text("Mar 22, 2026"); + tbl.cell(2, 0).unwrap().set_text("Prepared by:"); + tbl.cell(2, 1).unwrap().set_text("Walter White"); + tbl.cell(2, 2).unwrap().set_text("Payment:"); + tbl.cell(2, 3).unwrap().set_text("Net 30"); + tbl.cell(3, 0).unwrap().set_text("Customer:"); + tbl.cell(3, 1).unwrap().set_text("Meridian Dynamics LLC"); + tbl.cell(3, 2).unwrap().set_text("Currency:"); + tbl.cell(3, 3).unwrap().set_text("USD"); + } + + doc.add_paragraph(""); + + // ── Bill of Materials ── + { + let mut p = doc.add_paragraph(""); + p.add_run("Bill of Materials") + .bold(true) + .size(16.0) + .color("1A3C3C"); + } + doc.add_paragraph(""); + + { + let mut tbl = doc + .add_table(10, 6) + .borders(BorderStyle::Single, 2, "008B8B") + .width_pct(100.0) + .layout_fixed(); + // Headers + let hdrs = [ + "#", + "Part Number", + "Description", + "Qty", + "Unit Price", + "Total", + ]; + for (c, h) in hdrs.iter().enumerate() { + tbl.cell(0, c).unwrap().set_text(h); + tbl.cell(0, c).unwrap().shading("008B8B"); + } + + let items: Vec<(&str, &str, &str, &str, &str)> = vec![ + ( + "TB-GPU-A100", + "NVIDIA A100 80GB GPU", + "4", + "$12,500.00", + "$50,000.00", + ), + ( + "TB-SRV-R750", + "Dell R750xa Server Chassis", + "2", + "$8,200.00", + "$16,400.00", + ), + ( + "TB-RAM-256G", + "256GB DDR5 ECC Memory Module", + "8", + "$890.00", + "$7,120.00", + ), + ( + "TB-SSD-3840", + "3.84TB NVMe U.2 SSD", + "8", + "$1,150.00", + "$9,200.00", + ), + ( + "TB-NET-CX7", + "ConnectX-7 200GbE NIC", + "4", + "$1,800.00", + "$7,200.00", + ), + ( + "TB-SW-48P", + "48-Port 100GbE Switch", + "1", + "$22,000.00", + "$22,000.00", + ), + ( + "TB-CAB-RACK", + "42U Server Rack + PDU", + "1", + "$4,500.00", + "$4,500.00", + ), + ( + "TB-SVC-INST", + "Installation & Configuration", + "1", + "$8,500.00", + "$8,500.00", + ), + ( + "TB-SVC-SUPP", + "3-Year Premium Support", + "1", + "$15,000.00", + "$15,000.00", + ), + ]; + + for (i, (pn, desc, qty, unit, total)) in items.iter().enumerate() { + let row = i + 1; + tbl.cell(row, 0).unwrap().set_text(&format!("{}", i + 1)); + tbl.cell(row, 1).unwrap().set_text(pn); + tbl.cell(row, 2).unwrap().set_text(desc); + tbl.cell(row, 3).unwrap().set_text(qty); + tbl.cell(row, 4).unwrap().set_text(unit); + tbl.cell(row, 5).unwrap().set_text(total); + if i % 2 == 0 { + for c in 0..6 { + tbl.cell(row, c).unwrap().shading("F0F8F8"); + } + } + } + } + + doc.add_paragraph(""); + + // ── Totals ── + { + let mut tbl = doc + .add_table(4, 2) + .borders(BorderStyle::Single, 2, "008B8B") + .width(Length::inches(3.5)) + .alignment(Alignment::Right); + tbl.cell(0, 0).unwrap().set_text("Subtotal"); + tbl.cell(0, 1).unwrap().set_text("$139,920.00"); + tbl.cell(1, 0).unwrap().set_text("Shipping & Handling"); + tbl.cell(1, 1).unwrap().set_text("$2,500.00"); + tbl.cell(2, 0).unwrap().set_text("Tax (8.625%)"); + tbl.cell(2, 1).unwrap().set_text("$12,068.10"); + tbl.cell(3, 0).unwrap().set_text("TOTAL"); + tbl.cell(3, 0).unwrap().shading("E07020"); + tbl.cell(3, 1).unwrap().set_text("$154,488.10"); + tbl.cell(3, 1).unwrap().shading("E07020"); + } + + doc.add_paragraph(""); + + // ── Terms & Conditions ── + { + let mut p = doc.add_paragraph(""); + p.add_run("Terms & Conditions") + .bold(true) + .size(14.0) + .color("1A3C3C"); + } + doc.add_numbered_list_item( + "This quotation is valid for 30 calendar days from the date of issue.", + 0, + ); + doc.add_numbered_list_item( + "All prices are in USD and exclusive of applicable taxes unless stated.", + 0, + ); + doc.add_numbered_list_item("Standard lead time is 4-6 weeks from PO receipt.", 0); + doc.add_numbered_list_item( + "Payment terms: Net 30 from invoice date. 2% discount for payment within 10 days.", + 0, + ); + doc.add_numbered_list_item( + "Warranty: 3-year manufacturer warranty on all hardware components.", + 0, + ); + doc.add_numbered_list_item( + "Returns subject to 15% restocking fee if initiated after 14 days.", + 0, + ); + + doc.add_paragraph(""); + doc.add_paragraph("") + .border_bottom(BorderStyle::Single, 4, "008B8B"); + doc.add_paragraph(""); + { + let mut p = doc.add_paragraph(""); + p.add_run("Accepted by: ").bold(true); + p.add_run("___________________________ Date: ___________"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Print Name: ").bold(true); + p.add_run("___________________________ Title: ___________"); + } + + doc +} + +// ============================================================================= +// 4. INVOICE — Crimson + charcoal scheme +// ============================================================================= +fn generate_invoice(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // Colors: Crimson #B22222, Charcoal #333333, Light #FAF0F0 + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(0.75), + Length::inches(0.75), + Length::inches(0.75), + Length::inches(0.75), + ); + doc.set_title("Invoice INV-2026-0392"); + doc.set_author("Walter White"); + doc.set_keywords("invoice, Tensorbee"); + + doc.set_footer("Tensorbee — Thank you for your business!"); + + // ── Company & Invoice Header ── + { + let mut tbl = doc.add_table(4, 2).width_pct(100.0); + // Company name left, INVOICE right + { + let mut cell = tbl.cell(0, 0).unwrap(); + let mut p = cell.add_paragraph(""); + p.add_run("TENSORBEE") + .bold(true) + .size(32.0) + .color("B22222") + .font("Helvetica"); + } + { + let mut cell = tbl.cell(0, 1).unwrap(); + let mut p = cell.add_paragraph(""); + p.add_run("INVOICE").bold(true).size(32.0).color("333333"); + } + tbl.cell(1, 0) + .unwrap() + .set_text("123 Innovation Drive, Suite 400"); + tbl.cell(1, 1).unwrap().set_text("Invoice #: INV-2026-0392"); + tbl.cell(2, 0).unwrap().set_text("San Francisco, CA 94105"); + tbl.cell(2, 1).unwrap().set_text("Date: February 22, 2026"); + tbl.cell(3, 0).unwrap().set_text("walter@tensorbee.com"); + tbl.cell(3, 1).unwrap().set_text("Due Date: March 24, 2026"); + } + + doc.add_paragraph("") + .border_bottom(BorderStyle::Thick, 8, "B22222"); + doc.add_paragraph(""); + + // ── Bill To / Ship To ── + { + let mut tbl = doc.add_table(4, 2).width_pct(100.0); + tbl.cell(0, 0).unwrap().set_text("BILL TO"); + tbl.cell(0, 0).unwrap().shading("B22222"); + tbl.cell(0, 1).unwrap().set_text("SHIP TO"); + tbl.cell(0, 1).unwrap().shading("B22222"); + tbl.cell(1, 0).unwrap().set_text("Meridian Dynamics LLC"); + tbl.cell(1, 1).unwrap().set_text("Meridian Dynamics LLC"); + tbl.cell(2, 0).unwrap().set_text("456 Enterprise Blvd"); + tbl.cell(2, 1).unwrap().set_text("Attn: Server Room B"); + tbl.cell(3, 0).unwrap().set_text("Austin, TX 78701"); + tbl.cell(3, 1) + .unwrap() + .set_text("456 Enterprise Blvd, Austin, TX 78701"); + } + + doc.add_paragraph(""); + + // ── Line Items ── + { + let mut tbl = doc + .add_table(8, 5) + .borders(BorderStyle::Single, 2, "B22222") + .width_pct(100.0); + let hdrs = ["Description", "Qty", "Unit Price", "Tax", "Amount"]; + for (c, h) in hdrs.iter().enumerate() { + tbl.cell(0, c).unwrap().set_text(h); + tbl.cell(0, c).unwrap().shading("B22222"); + } + let items = [ + ( + "ML Infrastructure Setup — Phase 1", + "1", + "$45,000.00", + "$3,881.25", + "$48,881.25", + ), + ( + "Data Pipeline Development (160 hrs)", + "160", + "$225.00", + "$3,105.00", + "$39,105.00", + ), + ( + "GPU Cluster Configuration", + "1", + "$12,000.00", + "$1,035.00", + "$13,035.00", + ), + ( + "API Gateway Implementation", + "1", + "$18,500.00", + "$1,595.63", + "$20,095.63", + ), + ( + "Load Testing & QA (80 hrs)", + "80", + "$195.00", + "$1,345.50", + "$16,945.50", + ), + ( + "Documentation & Training", + "1", + "$8,000.00", + "$690.00", + "$8,690.00", + ), + ( + "Project Management (3 months)", + "3", + "$6,500.00", + "$1,679.63", + "$21,179.63", + ), + ]; + for (i, (desc, qty, unit, tax, amt)) in items.iter().enumerate() { + let r = i + 1; + tbl.cell(r, 0).unwrap().set_text(desc); + tbl.cell(r, 1).unwrap().set_text(qty); + tbl.cell(r, 2).unwrap().set_text(unit); + tbl.cell(r, 3).unwrap().set_text(tax); + tbl.cell(r, 4).unwrap().set_text(amt); + if i % 2 == 0 { + for c in 0..5 { + tbl.cell(r, c).unwrap().shading("FAF0F0"); + } + } + } + } + + doc.add_paragraph(""); + + // ── Totals ── + { + let mut tbl = doc + .add_table(4, 2) + .borders(BorderStyle::Single, 2, "B22222") + .width(Length::inches(3.0)) + .alignment(Alignment::Right); + tbl.cell(0, 0).unwrap().set_text("Subtotal"); + tbl.cell(0, 1).unwrap().set_text("$154,600.00"); + tbl.cell(1, 0).unwrap().set_text("Tax (8.625%)"); + tbl.cell(1, 1).unwrap().set_text("$13,334.25"); + tbl.cell(2, 0).unwrap().set_text("Discount (5%)"); + tbl.cell(2, 1).unwrap().set_text("-$7,730.00"); + tbl.cell(3, 0).unwrap().set_text("AMOUNT DUE"); + tbl.cell(3, 0).unwrap().shading("B22222"); + tbl.cell(3, 1).unwrap().set_text("$160,204.25"); + tbl.cell(3, 1).unwrap().shading("B22222"); + } + + doc.add_paragraph(""); + doc.add_paragraph(""); + + // ── Payment Details ── + { + let mut p = doc.add_paragraph(""); + p.add_run("Payment Details") + .bold(true) + .size(14.0) + .color("333333"); + } + doc.add_paragraph("Bank: Silicon Valley Bank") + .indent_left(Length::inches(0.3)); + doc.add_paragraph("Account: Tensorbee Inc. — 0847-2953-1120") + .indent_left(Length::inches(0.3)); + doc.add_paragraph("Routing: 121140399") + .indent_left(Length::inches(0.3)); + doc.add_paragraph("Swift: SVBKUS6S") + .indent_left(Length::inches(0.3)); + + doc.add_paragraph(""); + + doc.add_paragraph("Please include invoice number INV-2026-0392 in the payment reference.") + .shading("FAF0F0") + .border_all(BorderStyle::Single, 2, "B22222"); + + doc +} + +// ============================================================================= +// 5. REPORT — Forest green + earth tones, with images & hierarchical sections +// ============================================================================= +fn generate_report(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // Colors: Forest #2D5016, Sage #6B8E23, Earth #8B7355, Cream #FFFAF0 + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + ); + doc.set_title("Q4 2025 Environmental Impact Report"); + doc.set_author("Walter White"); + doc.set_subject("Quarterly Environmental Report"); + doc.set_keywords("environment, sustainability, report, Tensorbee"); + + doc.set_different_first_page(true); + doc.set_first_page_header(""); + doc.set_header("Tensorbee — Q4 2025 Environmental Impact Report"); + doc.set_footer("CONFIDENTIAL — Page"); + + // ── Cover ── + let cover_bg = create_sample_png(612, 792, [20, 50, 15]); + doc.add_background_image(&cover_bg, "report_cover.png"); + + for _ in 0..5 { + doc.add_paragraph(""); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Q4 2025") + .bold(true) + .size(48.0) + .color("FFFFFF") + .font("Georgia"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Environmental Impact Report") + .size(24.0) + .color("90EE90") + .font("Georgia") + .italic(true); + } + doc.add_paragraph(""); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Tensorbee — Sustainability Division") + .size(14.0) + .color("C0C0C0"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Prepared by Walter White, Chief Sustainability Officer") + .size(11.0) + .color("AAAAAA"); + } + + // ── TOC ── + doc.add_paragraph("").page_break_before(true); + doc.insert_toc(doc.content_count(), 3); + + // ── Section 1: Executive Overview ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Executive Overview").style("Heading1"); + doc.add_paragraph( + "This report presents Tensorbee's environmental performance for Q4 2025. Our \ + sustainability initiatives have yielded a 34% reduction in carbon emissions compared \ + to Q4 2024, exceeding our target of 25%. Key achievements include the transition to \ + 100% renewable energy in our primary data centers and the launch of our carbon offset \ + marketplace.", + ) + .first_line_indent(Length::inches(0.3)); + + // Image: performance chart + let chart_img = create_chart_png(400, 200); + doc.add_paragraph(""); + doc.add_picture( + &chart_img, + "performance_chart.png", + Length::inches(5.0), + Length::inches(2.5), + ) + .alignment(Alignment::Center); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Figure 1: Quarterly Carbon Emissions (tonnes CO2e)") + .italic(true) + .size(9.0) + .color("666666"); + } + + // ── Section 2: Energy Consumption ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Energy Consumption").style("Heading1"); + + doc.add_paragraph("Data Center Operations") + .style("Heading2"); + doc.add_paragraph( + "Our three primary data centers consumed a combined 4.2 GWh in Q4 2025, \ + a 12% reduction from Q3 2025 driven by improved cooling efficiency and \ + server consolidation.", + ); + + // Data table + { + let mut tbl = doc + .add_table(5, 4) + .borders(BorderStyle::Single, 4, "2D5016") + .width_pct(100.0); + let hdrs = ["Data Center", "Capacity (MW)", "Usage (GWh)", "PUE"]; + for (c, h) in hdrs.iter().enumerate() { + tbl.cell(0, c).unwrap().set_text(h); + tbl.cell(0, c).unwrap().shading("2D5016"); + } + let rows = [ + ("San Francisco (Primary)", "3.2", "1.8", "1.12"), + ("Dublin (EU)", "2.1", "1.2", "1.18"), + ("Singapore (APAC)", "1.8", "1.2", "1.24"), + ("TOTAL", "7.1", "4.2", "1.17 avg"), + ]; + for (i, (dc, cap, use_, pue)) in rows.iter().enumerate() { + tbl.cell(i + 1, 0).unwrap().set_text(dc); + tbl.cell(i + 1, 1).unwrap().set_text(cap); + tbl.cell(i + 1, 2).unwrap().set_text(use_); + tbl.cell(i + 1, 3).unwrap().set_text(pue); + if i == 3 { + for c in 0..4 { + tbl.cell(i + 1, c).unwrap().shading("E2EFDA"); + } + } + } + } + + doc.add_paragraph(""); + + doc.add_paragraph("Renewable Energy Mix").style("Heading2"); + doc.add_paragraph("Breakdown of energy sources across all facilities:"); + + doc.add_bullet_list_item("Solar PV: 42% (1.76 GWh)", 0); + doc.add_bullet_list_item("Wind Power Purchase Agreements: 38% (1.60 GWh)", 0); + doc.add_bullet_list_item("Hydroelectric: 12% (0.50 GWh)", 0); + doc.add_bullet_list_item("Grid (non-renewable): 8% (0.34 GWh)", 0); + + // ── Section 3: Water & Waste ── + doc.add_paragraph("Water & Waste Management") + .style("Heading1"); + + doc.add_paragraph("Water Usage").style("Heading2"); + doc.add_paragraph( + "Total water consumption was 12.4 million gallons, a 15% reduction from Q3. \ + Our closed-loop cooling systems now recycle 78% of water used in cooling operations.", + ); + + doc.add_paragraph("Waste Reduction").style("Heading2"); + doc.add_paragraph("E-waste management results for Q4:") + .keep_with_next(true); + { + let mut tbl = doc + .add_table(4, 3) + .borders(BorderStyle::Single, 2, "6B8E23") + .width_pct(80.0); + tbl.cell(0, 0).unwrap().set_text("Category"); + tbl.cell(0, 0).unwrap().shading("6B8E23"); + tbl.cell(0, 1).unwrap().set_text("Weight (kg)"); + tbl.cell(0, 1).unwrap().shading("6B8E23"); + tbl.cell(0, 2).unwrap().set_text("Recycled %"); + tbl.cell(0, 2).unwrap().shading("6B8E23"); + let rows = [ + ("Server Hardware", "2,450", "94%"), + ("Networking Equipment", "820", "91%"), + ("Storage Media", "340", "99%"), + ]; + for (i, (cat, wt, pct)) in rows.iter().enumerate() { + tbl.cell(i + 1, 0).unwrap().set_text(cat); + tbl.cell(i + 1, 1).unwrap().set_text(wt); + tbl.cell(i + 1, 2).unwrap().set_text(pct); + } + } + + // ── Section 4: Initiatives ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Q1 2026 Initiatives").style("Heading1"); + + doc.add_paragraph("Planned Programs").style("Heading2"); + doc.add_numbered_list_item( + "Deploy on-site battery storage at SF data center (2 MWh capacity)", + 0, + ); + doc.add_numbered_list_item("Pilot immersion cooling in Dublin facility", 0); + doc.add_numbered_list_item("Launch employee commute carbon offset program", 0); + doc.add_numbered_list_item("Achieve ISO 14001 certification for Singapore facility", 0); + + doc.add_paragraph("Investment Targets").style("Heading2"); + doc.add_paragraph("Sustainability CapEx allocation for FY2026:") + .keep_with_next(true); + { + let mut tbl = doc + .add_table(5, 2) + .borders(BorderStyle::Single, 4, "2D5016"); + tbl.cell(0, 0).unwrap().set_text("Initiative"); + tbl.cell(0, 0).unwrap().shading("2D5016"); + tbl.cell(0, 1).unwrap().set_text("Budget"); + tbl.cell(0, 1).unwrap().shading("2D5016"); + let rows = [ + ("Battery Storage", "$1.2M"), + ("Immersion Cooling Pilot", "$800K"), + ("Solar Panel Expansion", "$2.1M"), + ("Carbon Credits", "$500K"), + ]; + for (i, (init, budget)) in rows.iter().enumerate() { + tbl.cell(i + 1, 0).unwrap().set_text(init); + tbl.cell(i + 1, 1).unwrap().set_text(budget); + if i % 2 == 0 { + tbl.cell(i + 1, 0).unwrap().shading("FFFAF0"); + tbl.cell(i + 1, 1).unwrap().shading("FFFAF0"); + } + } + } + + // Image: sustainability roadmap + let roadmap_img = create_sample_png(500, 100, [40, 80, 30]); + doc.add_paragraph(""); + doc.add_picture( + &roadmap_img, + "roadmap.png", + Length::inches(6.0), + Length::inches(1.2), + ) + .alignment(Alignment::Center); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Figure 2: 2026 Sustainability Roadmap") + .italic(true) + .size(9.0) + .color("666666"); + } + + doc.add_paragraph(""); + doc.add_paragraph( + "For questions about this report, contact Walter White at sustainability@tensorbee.com.", + ) + .shading("FFFAF0") + .border_all(BorderStyle::Single, 2, "2D5016"); + + doc +} + +// ============================================================================= +// 6. LETTER — Slate blue + silver scheme +// ============================================================================= +fn generate_letter(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // Colors: Slate #4A5568, Blue accent #3182CE + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.25), + Length::inches(1.25), + Length::inches(1.25), + Length::inches(1.25), + ); + doc.set_title("Business Letter"); + doc.set_author("Walter White"); + + // Banner header using raw XML + let logo_img = create_logo_png(220, 48); + let banner = build_header_banner_xml( + "rId1", + &BannerOpts { + bg_color: "4A5568", + banner_width: 7772400, + banner_height: 731520, // ~0.8" + logo_width: 2011680, + logo_height: 438912, + logo_x_offset: 295125, + logo_y_offset: 146304, + }, + ); + doc.set_raw_header_with_images( + banner, + &[("rId1", &logo_img, "logo.png")], + rdocx_oxml::header_footer::HdrFtrType::Default, + ); + doc.set_margins( + Length::twips(2000), + Length::twips(1800), + Length::twips(1440), + Length::twips(1800), + ); + doc.set_header_footer_distance(Length::twips(720), Length::twips(432)); + + // Footer with contact + doc.set_footer( + "Tensorbee | 123 Innovation Drive, Suite 400 | San Francisco, CA 94105 | tensorbee.com", + ); + + // ── Sender Address ── + doc.add_paragraph("Walter White"); + doc.add_paragraph("Chief Executive Officer"); + doc.add_paragraph("Tensorbee"); + doc.add_paragraph("123 Innovation Drive, Suite 400"); + doc.add_paragraph("San Francisco, CA 94105"); + doc.add_paragraph(""); + doc.add_paragraph("February 22, 2026"); + + doc.add_paragraph(""); + + // ── Recipient ── + doc.add_paragraph("Ms. Sarah Chen"); + doc.add_paragraph("VP of Engineering"); + doc.add_paragraph("Meridian Dynamics LLC"); + doc.add_paragraph("456 Enterprise Boulevard"); + doc.add_paragraph("Austin, TX 78701"); + + doc.add_paragraph(""); + + // ── Subject ── + { + let mut p = doc.add_paragraph(""); + p.add_run("Re: Strategic Technology Partnership — Phase 2 Expansion") + .bold(true); + } + + doc.add_paragraph(""); + + // ── Body ── + doc.add_paragraph("Dear Ms. Chen,"); + doc.add_paragraph(""); + + doc.add_paragraph( + "I am writing to express Tensorbee's enthusiasm for expanding our strategic technology \ + partnership with Meridian Dynamics. Following the successful completion of Phase 1, \ + which delivered a 40% improvement in your ML inference pipeline throughput, we believe \ + the foundation is firmly established for an ambitious Phase 2 engagement.", + ) + .first_line_indent(Length::inches(0.5)); + + doc.add_paragraph( + "During our recent executive review, your team highlighted three priority areas for \ + the next phase of collaboration:", + ) + .first_line_indent(Length::inches(0.5)); + + doc.add_numbered_list_item( + "Real-time anomaly detection for your financial transaction monitoring system, \ + targeting sub-100ms latency at 50,000 transactions per second.", + 0, + ); + doc.add_numbered_list_item( + "Federated learning infrastructure to enable model training across your distributed \ + data centers without centralizing sensitive financial data.", + 0, + ); + doc.add_numbered_list_item( + "MLOps automation to reduce your model deployment cycle from the current 5 days \ + to under 4 hours.", + 0, + ); + + doc.add_paragraph( + "Our engineering team has prepared a detailed technical proposal addressing each of \ + these areas. We have allocated a dedicated team of eight senior engineers, led by \ + our CTO, to ensure continuity with the Phase 1 team your organization has already \ + built a productive working relationship with.", + ) + .first_line_indent(Length::inches(0.5)); + + doc.add_paragraph( + "I would welcome the opportunity to present our Phase 2 proposal in person. My \ + assistant will reach out to coordinate a meeting at your convenience during the \ + first week of March.", + ) + .first_line_indent(Length::inches(0.5)); + + doc.add_paragraph(""); + + doc.add_paragraph("Warm regards,"); + doc.add_paragraph(""); + doc.add_paragraph(""); + + // Signature line + doc.add_paragraph("") + .border_bottom(BorderStyle::Single, 4, "4A5568"); + { + let mut p = doc.add_paragraph(""); + p.add_run("Walter White").bold(true).size(12.0); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Chief Executive Officer, Tensorbee") + .italic(true) + .size(10.0) + .color("666666"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("walter@tensorbee.com | +1 (415) 555-0199") + .size(10.0) + .color("888888"); + } + + doc +} + +// ============================================================================= +// 7. EMPLOYMENT CONTRACT — Purple + charcoal scheme +// ============================================================================= +fn generate_contract(_samples_dir: &Path) -> Document { + let mut doc = Document::new(); + + // Colors: Purple #5B2C6F, Plum #8E44AD, Gray #2C3E50 + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.25), + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + ); + doc.set_title("Employment Agreement"); + doc.set_author("Tensorbee HR Department"); + doc.set_subject("Employment Contract — Walter White"); + doc.set_keywords("employment, contract, agreement, Tensorbee"); + + doc.set_header("TENSORBEE — Employment Agreement"); + doc.set_footer("Employment Agreement — Walter White — Page"); + + // ── Title Block ── + doc.add_paragraph("") + .border_bottom(BorderStyle::Thick, 12, "5B2C6F"); + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("EMPLOYMENT AGREEMENT") + .bold(true) + .size(24.0) + .color("5B2C6F") + .font("Georgia"); + } + doc.add_paragraph("") + .border_bottom(BorderStyle::Thick, 12, "5B2C6F"); + doc.add_paragraph(""); + + // ── Parties ── + doc.add_paragraph( + "This Employment Agreement (\"Agreement\") is entered into as of February 22, 2026 \ + (\"Effective Date\"), by and between:", + ); + + doc.add_paragraph(""); + + { + let mut p = doc.add_paragraph("").indent_left(Length::inches(0.5)); + p.add_run("EMPLOYER: ").bold(true); + p.add_run( + "Tensorbee, Inc., a Delaware corporation with its principal place of business at \ + 123 Innovation Drive, Suite 400, San Francisco, CA 94105 (\"Company\")", + ); + } + doc.add_paragraph(""); + { + let mut p = doc.add_paragraph("").indent_left(Length::inches(0.5)); + p.add_run("EMPLOYEE: ").bold(true); + p.add_run( + "Walter White, residing at 308 Negra Arroyo Lane, Albuquerque, NM 87104 (\"Employee\")", + ); + } + + doc.add_paragraph(""); + doc.add_paragraph("The Company and Employee are collectively referred to as the \"Parties.\""); + doc.add_paragraph(""); + + // ── Article 1: Position and Duties ── + doc.add_paragraph("Article 1 — Position and Duties") + .style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("1.1 ").bold(true); + p.add_run("The Company hereby employs the Employee as "); + p.add_run("Chief Executive Officer (CEO)") + .bold(true) + .italic(true); + p.add_run(", reporting directly to the Board of Directors."); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("1.2 ").bold(true); + p.add_run("The Employee shall devote full working time, attention, and best efforts to \ + the performance of duties as reasonably assigned by the Board, including but not limited to:"); + } + doc.add_bullet_list_item( + "Setting and executing the Company's strategic vision and business plan", + 0, + ); + doc.add_bullet_list_item( + "Overseeing all operations, engineering, and commercial activities", + 0, + ); + doc.add_bullet_list_item( + "Representing the Company to investors, customers, and the public", + 0, + ); + doc.add_bullet_list_item("Recruiting, developing, and retaining key talent", 0); + + { + let mut p = doc.add_paragraph(""); + p.add_run("1.3 ").bold(true); + p.add_run( + "The Employee's primary work location shall be the Company's San Francisco \ + headquarters, with reasonable travel as required by business needs.", + ); + } + + // ── Article 2: Compensation ── + doc.add_paragraph("Article 2 — Compensation and Benefits") + .style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("2.1 Base Salary. ").bold(true); + p.add_run("The Company shall pay the Employee an annual base salary of "); + p.add_run("$375,000.00").bold(true); + p.add_run(" (Three Hundred Seventy-Five Thousand Dollars), payable in accordance with the \ + Company's standard payroll schedule, less applicable withholdings and deductions."); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("2.2 Annual Bonus. ").bold(true); + p.add_run("The Employee shall be eligible for an annual performance bonus of up to "); + p.add_run("40%").bold(true); + p.add_run( + " of base salary, based on achievement of mutually agreed performance objectives.", + ); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("2.3 Equity. ").bold(true); + p.add_run("Subject to Board approval, the Employee shall receive a stock option grant of "); + p.add_run("500,000 shares").bold(true); + p.add_run( + " of the Company's common stock, vesting over four (4) years with a one-year cliff, \ + at an exercise price equal to the fair market value on the date of grant.", + ); + } + + // Compensation summary table + doc.add_paragraph(""); + { + let mut tbl = doc + .add_table(5, 2) + .borders(BorderStyle::Single, 4, "5B2C6F") + .width_pct(70.0) + .alignment(Alignment::Center); + tbl.cell(0, 0).unwrap().set_text("Compensation Element"); + tbl.cell(0, 0).unwrap().shading("5B2C6F"); + tbl.cell(0, 1).unwrap().set_text("Value"); + tbl.cell(0, 1).unwrap().shading("5B2C6F"); + let items = [ + ("Base Salary", "$375,000/year"), + ("Target Bonus", "Up to 40% ($150,000)"), + ("Equity Grant", "500,000 shares (4yr vest)"), + ("Total Target Comp", "$525,000 + equity"), + ]; + for (i, (elem, val)) in items.iter().enumerate() { + tbl.cell(i + 1, 0).unwrap().set_text(elem); + tbl.cell(i + 1, 1).unwrap().set_text(val); + if i == 3 { + tbl.cell(i + 1, 0).unwrap().shading("E8DAEF"); + tbl.cell(i + 1, 1).unwrap().shading("E8DAEF"); + } + } + } + + // ── Article 3: Benefits ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("Article 3 — Benefits").style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("3.1 ").bold(true); + p.add_run( + "The Employee shall be entitled to participate in all benefit programs \ + generally available to senior executives, including:", + ); + } + doc.add_bullet_list_item( + "Medical, dental, and vision insurance (100% premium coverage for employee and dependents)", + 0, + ); + doc.add_bullet_list_item("401(k) retirement plan with 6% company match", 0); + doc.add_bullet_list_item("Life insurance and long-term disability coverage", 0); + doc.add_bullet_list_item("Annual professional development allowance of $10,000", 0); + + { + let mut p = doc.add_paragraph(""); + p.add_run("3.2 Paid Time Off. ").bold(true); + p.add_run( + "The Employee shall receive 25 days of paid vacation per year, plus Company holidays, \ + accruing on a monthly basis.", + ); + } + + // ── Article 4: Term and Termination ── + doc.add_paragraph("Article 4 — Term and Termination") + .style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("4.1 At-Will Employment. ").bold(true); + p.add_run( + "This Agreement is for at-will employment and may be terminated by either Party \ + at any time, with or without cause, subject to the notice provisions herein.", + ); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("4.2 Notice Period. ").bold(true); + p.add_run("Either Party shall provide "); + p.add_run("ninety (90) days").bold(true); + p.add_run(" written notice of termination, or pay in lieu thereof."); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("4.3 Severance. ").bold(true); + p.add_run("In the event of termination by the Company without Cause, the Employee shall \ + receive (a) twelve (12) months of base salary continuation, (b) pro-rata bonus \ + for the year of termination, and (c) twelve (12) months of COBRA premium coverage."); + } + + // ── Article 5: Confidentiality ── + doc.add_paragraph("Article 5 — Confidentiality and IP") + .style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("5.1 ").bold(true); + p.add_run( + "The Employee agrees to maintain strict confidentiality of all proprietary \ + information, trade secrets, and business plans of the Company during and after \ + employment.", + ); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("5.2 ").bold(true); + p.add_run( + "All intellectual property, inventions, and works of authorship created by the \ + Employee during the term of employment and within the scope of duties shall be \ + the sole and exclusive property of the Company.", + ); + } + + // ── Article 6: Non-Compete ── + doc.add_paragraph("Article 6 — Non-Competition") + .style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("6.1 ").bold(true); + p.add_run("For a period of twelve (12) months following termination, the Employee agrees \ + not to engage in any business that directly competes with the Company's core \ + business of AI infrastructure and ML operations services, within the United States."); + } + + // ── Article 7: General Provisions ── + doc.add_paragraph("Article 7 — General Provisions") + .style("Heading1"); + { + let mut p = doc.add_paragraph(""); + p.add_run("7.1 Governing Law. ").bold(true); + p.add_run( + "This Agreement shall be governed by and construed in accordance with the laws of \ + the State of California.", + ); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("7.2 Entire Agreement. ").bold(true); + p.add_run( + "This Agreement constitutes the entire agreement between the Parties and supersedes \ + all prior negotiations, representations, and agreements.", + ); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("7.3 Amendment. ").bold(true); + p.add_run( + "This Agreement may only be amended by a written instrument signed by both Parties.", + ); + } + + // ── Signature Block ── + doc.add_paragraph("").page_break_before(true); + doc.add_paragraph("IN WITNESS WHEREOF, the Parties have executed this Employment Agreement as of the Effective Date.") + .space_after(Length::pt(24.0)); + + // Signature table + { + let mut tbl = doc.add_table(6, 2).width_pct(100.0); + tbl.cell(0, 0).unwrap().set_text("FOR THE COMPANY:"); + tbl.cell(0, 0).unwrap().shading("5B2C6F"); + tbl.cell(0, 1).unwrap().set_text("EMPLOYEE:"); + tbl.cell(0, 1).unwrap().shading("5B2C6F"); + + tbl.cell(1, 0).unwrap().set_text(""); + tbl.cell(1, 1).unwrap().set_text(""); + tbl.row(1).unwrap().height(Length::pt(40.0)); + + tbl.cell(2, 0) + .unwrap() + .set_text("Signature: ___________________________"); + tbl.cell(2, 1) + .unwrap() + .set_text("Signature: ___________________________"); + tbl.cell(3, 0) + .unwrap() + .set_text("Name: Board Representative"); + tbl.cell(3, 1).unwrap().set_text("Name: Walter White"); + tbl.cell(4, 0) + .unwrap() + .set_text("Title: Chair, Board of Directors"); + tbl.cell(4, 1) + .unwrap() + .set_text("Title: Chief Executive Officer"); + tbl.cell(5, 0).unwrap().set_text("Date: _______________"); + tbl.cell(5, 1).unwrap().set_text("Date: _______________"); + } + + doc +} + +// ============================================================================= +// PNG generation helpers +// ============================================================================= + +/// Create a gradient sample PNG (used for backgrounds, charts, etc.) +fn create_sample_png(width: u32, height: u32, base_color: [u8; 3]) -> Vec { + let mut pixels = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + let fy = y as f64 / height as f64; + let fx = x as f64 / width as f64; + let r = (base_color[0] as f64 + (1.0 - fy) * 40.0).min(255.0) as u8; + let g = (base_color[1] as f64 + fx * 30.0).min(255.0) as u8; + let b = (base_color[2] as f64 + fy * 60.0).min(255.0) as u8; + pixels.extend_from_slice(&[r, g, b, 255]); + } + } + encode_png(width, height, &pixels) +} + +/// Create a simple bar chart PNG for the report. +fn create_chart_png(width: u32, height: u32) -> Vec { + let mut pixels = vec![0u8; (width * height * 4) as usize]; + // White background + for i in 0..pixels.len() / 4 { + pixels[i * 4] = 255; + pixels[i * 4 + 1] = 255; + pixels[i * 4 + 2] = 255; + pixels[i * 4 + 3] = 255; + } + // Draw simple bars + let bars: Vec<(u32, u32, [u8; 3])> = vec![ + (30, 150, [45, 80, 22]), // Q1 + (100, 120, [45, 80, 22]), // Q2 + (170, 100, [45, 80, 22]), // Q3 + (240, 70, [45, 80, 22]), // Q4 (best) + ]; + for (bx, bar_height, color) in &bars { + let bar_width = 50; + let y_start = height - bar_height; + for y in y_start..height { + for x in *bx..(*bx + bar_width).min(width) { + let idx = ((y * width + x) * 4) as usize; + if idx + 3 < pixels.len() { + pixels[idx] = color[0]; + pixels[idx + 1] = color[1]; + pixels[idx + 2] = color[2]; + pixels[idx + 3] = 255; + } + } + } + } + encode_png(width, height, &pixels) +} + +/// Create a logo placeholder PNG (white rectangle on transparent). +fn create_logo_png(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + let in_text_area = + x > width / 8 && x < width * 7 / 8 && y > height / 4 && y < height * 3 / 4; + if in_text_area { + pixels.extend_from_slice(&[255, 255, 255, 255]); + } else { + pixels.extend_from_slice(&[255, 255, 255, 40]); + } + } + } + encode_png(width, height, &pixels) +} + +// ============================================================================= +// PNG encoding (no external dependencies) +// ============================================================================= + +fn encode_png(width: u32, height: u32, pixels: &[u8]) -> Vec { + let mut png = Vec::new(); + { + use std::io::Write as _; + png.write_all(&[137, 80, 78, 71, 13, 10, 26, 10]).unwrap(); + + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&height.to_be_bytes()); + ihdr.extend_from_slice(&[8, 6, 0, 0, 0]); // 8-bit RGBA + write_chunk(&mut png, b"IHDR", &ihdr); + + let mut raw = Vec::new(); + for y in 0..height { + raw.push(0); // filter: None + let s = (y * width * 4) as usize; + raw.extend_from_slice(&pixels[s..s + (width * 4) as usize]); + } + write_chunk(&mut png, b"IDAT", &zlib_store(&raw)); + write_chunk(&mut png, b"IEND", &[]); + } + png +} + +fn write_chunk(out: &mut Vec, ct: &[u8; 4], data: &[u8]) { + use std::io::Write as _; + out.write_all(&(data.len() as u32).to_be_bytes()).unwrap(); + out.write_all(ct).unwrap(); + out.write_all(data).unwrap(); + out.write_all(&crc32(ct, data).to_be_bytes()).unwrap(); +} + +fn crc32(ct: &[u8], data: &[u8]) -> u32 { + static T: std::sync::LazyLock<[u32; 256]> = std::sync::LazyLock::new(|| { + let mut t = [0u32; 256]; + for n in 0..256u32 { + let mut c = n; + for _ in 0..8 { + c = if c & 1 != 0 { + 0xEDB88320 ^ (c >> 1) + } else { + c >> 1 + }; + } + t[n as usize] = c; + } + t + }); + let mut c = 0xFFFFFFFF_u32; + for &b in ct.iter().chain(data) { + c = T[((c ^ b as u32) & 0xFF) as usize] ^ (c >> 8); + } + c ^ 0xFFFFFFFF +} + +fn zlib_store(data: &[u8]) -> Vec { + let mut out = vec![0x78, 0x01]; + let chunks: Vec<&[u8]> = data.chunks(65535).collect(); + for (i, chunk) in chunks.iter().enumerate() { + let last = i == chunks.len() - 1; + out.push(if last { 0x01 } else { 0x00 }); + let len = chunk.len() as u16; + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(!len).to_le_bytes()); + out.extend_from_slice(chunk); + } + let (mut a, mut b) = (1u32, 0u32); + for &byte in data { + a = (a + byte as u32) % 65521; + b = (b + a) % 65521; + } + out.extend_from_slice(&((b << 16) | a).to_be_bytes()); + out +} + +// ============================================================================= +// Header banner builder (DrawingML group shape via raw XML) +// ============================================================================= + +struct BannerOpts<'a> { + bg_color: &'a str, + banner_width: i64, + banner_height: i64, + logo_width: i64, + logo_height: i64, + logo_x_offset: i64, + logo_y_offset: i64, +} + +fn build_header_banner_xml(image_rel_id: &str, opts: &BannerOpts) -> Vec { + let mut xml = String::with_capacity(2048); + write!( + xml, + r#""# + ) + .unwrap(); + write!(xml, r#""# + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!( + xml, + r#""# + ) + .unwrap(); + write!( + xml, + r#""# + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + xml.into_bytes() +} diff --git a/vendor/rdocx/examples/generate_pdf.rs b/vendor/rdocx/examples/generate_pdf.rs new file mode 100644 index 00000000..2da27e62 --- /dev/null +++ b/vendor/rdocx/examples/generate_pdf.rs @@ -0,0 +1,73 @@ +//! Generate a PDF from a simple document and from the feature showcase sample. +//! +//! Run with: cargo run --example generate_pdf + +use rdocx::Document; + +fn main() { + // Test 1: Simple document + let doc = Document::new(); + match doc.to_pdf() { + Ok(bytes) => { + std::fs::write("/tmp/rdocx_simple.pdf", &bytes).unwrap(); + println!("Simple PDF: {} bytes -> /tmp/rdocx_simple.pdf", bytes.len()); + } + Err(e) => println!("Simple PDF failed: {e}"), + } + + // Test 2: Document with content + let mut doc = Document::new(); + doc.set_title("Test PDF Document"); + doc.set_author("rdocx-pdf"); + doc.add_paragraph("Chapter 1: Introduction") + .style("Heading1"); + doc.add_paragraph( + "This is a test document generated by rdocx and rendered to PDF. \ + It demonstrates text rendering with proper font shaping and pagination.", + ); + doc.add_paragraph("Section 1.1").style("Heading2"); + doc.add_paragraph("More content in a sub-section."); + + { + let mut table = doc.add_table(2, 3); + for r in 0..2 { + for c in 0..3 { + if let Some(mut cell) = table.cell(r, c) { + cell.set_text(&format!("R{}C{}", r + 1, c + 1)); + } + } + } + } + + doc.add_paragraph("After the table."); + + match doc.to_pdf() { + Ok(bytes) => { + std::fs::write("/tmp/rdocx_content.pdf", &bytes).unwrap(); + println!( + "Content PDF: {} bytes -> /tmp/rdocx_content.pdf", + bytes.len() + ); + } + Err(e) => println!("Content PDF failed: {e}"), + } + + // Test 3: From feature_showcase.docx + let showcase_path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../samples/feature_showcase.docx" + ); + match Document::open(showcase_path) { + Ok(doc) => match doc.to_pdf() { + Ok(bytes) => { + std::fs::write("/tmp/rdocx_showcase.pdf", &bytes).unwrap(); + println!( + "Showcase PDF: {} bytes -> /tmp/rdocx_showcase.pdf", + bytes.len() + ); + } + Err(e) => println!("Showcase PDF failed: {e}"), + }, + Err(e) => println!("Failed to open showcase: {e}"), + } +} diff --git a/vendor/rdocx/examples/generate_samples.rs b/vendor/rdocx/examples/generate_samples.rs new file mode 100644 index 00000000..8cf7b915 --- /dev/null +++ b/vendor/rdocx/examples/generate_samples.rs @@ -0,0 +1,783 @@ +//! Generates a single comprehensive sample document exercising every rdocx feature. +//! +//! Run with: cargo run --example generate_samples + +use std::collections::HashMap; +use std::path::Path; + +use rdocx::{ + Alignment, BorderStyle, Document, Length, SectionBreak, TabAlignment, TabLeader, + VerticalAlignment, +}; + +fn main() { + let samples_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("samples"); + std::fs::create_dir_all(&samples_dir).unwrap(); + + println!( + "Generating comprehensive sample document in {}", + samples_dir.display() + ); + + let out = samples_dir.join("feature_showcase.docx"); + generate_feature_showcase(&out); + println!(" feature_showcase.docx — all rdocx features in one document"); + + println!("\nDone!"); +} + +fn generate_feature_showcase(path: &Path) { + let mut doc = Document::new(); + + // ========================================================================= + // PAGE SETUP & METADATA + // ========================================================================= + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.0), // top + Length::inches(1.0), // right + Length::inches(1.0), // bottom + Length::inches(1.0), // left + ); + doc.set_header_footer_distance(Length::twips(720), Length::twips(432)); + doc.set_gutter(Length::twips(0)); + + doc.set_title("rdocx Feature Showcase"); + doc.set_author("rdocx Sample Generator"); + doc.set_subject("Comprehensive feature demonstration"); + doc.set_keywords("rdocx, docx, rust, sample"); + + // Header & Footer + doc.set_header("rdocx Feature Showcase"); + doc.set_footer("Generated by rdocx — Page"); + + // Different first page header + doc.set_different_first_page(true); + doc.set_first_page_header("rdocx"); + doc.set_first_page_footer("Feature Showcase — Cover Page"); + + // ========================================================================= + // PAGE 1: COVER PAGE — background image, run formatting + // ========================================================================= + let bg_cover = create_sample_png(612, 792, [30, 60, 120]); + doc.add_background_image(&bg_cover, "cover_bg.png"); + + doc.add_paragraph(""); // spacer + doc.add_paragraph(""); // spacer + doc.add_paragraph(""); // spacer + + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("rdocx").bold(true).size(72.0).color("FFFFFF"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("Feature Showcase") + .size(28.0) + .color("FFFFFF") + .italic(true); + } + + doc.add_paragraph(""); // spacer + + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("A comprehensive demonstration of every feature") + .size(14.0) + .color("CCDDFF"); + } + { + let mut p = doc.add_paragraph("").alignment(Alignment::Center); + p.add_run("provided by the rdocx Rust crate for DOCX generation.") + .size(14.0) + .color("CCDDFF"); + } + + // ========================================================================= + // PAGE 2: TEXT FORMATTING + // ========================================================================= + doc.add_paragraph("").page_break_before(true); + + doc.add_paragraph("1. Text Formatting").style("Heading1"); + + doc.add_paragraph("This section demonstrates paragraph and run-level formatting options."); + doc.add_paragraph(""); + + // --- Paragraph alignment --- + doc.add_paragraph("Paragraph Alignment").style("Heading2"); + + doc.add_paragraph("This paragraph is left-aligned (the default).") + .alignment(Alignment::Left); + doc.add_paragraph("This paragraph is center-aligned.") + .alignment(Alignment::Center); + doc.add_paragraph("This paragraph is right-aligned.") + .alignment(Alignment::Right); + doc.add_paragraph( + "This paragraph is justified. To demonstrate justified text properly, it needs \ + to be long enough to span multiple lines so the word spacing adjustment is visible \ + across the full width of the text area on the page.", + ) + .alignment(Alignment::Justify); + + doc.add_paragraph(""); + + // --- Run formatting --- + doc.add_paragraph("Run Formatting").style("Heading2"); + + { + let mut p = doc.add_paragraph(""); + p.add_run("Bold text").bold(true); + p.add_run(" | "); + p.add_run("Italic text").italic(true); + p.add_run(" | "); + p.add_run("Bold + Italic").bold(true).italic(true); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Single underline").underline(true); + p.add_run(" | "); + p.add_run("Strikethrough").strike(true); + p.add_run(" | "); + p.add_run("Double strikethrough").double_strike(true); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Red text").color("FF0000"); + p.add_run(" | "); + p.add_run("Blue text").color("0000FF"); + p.add_run(" | "); + p.add_run("Green text").color("00AA00"); + p.add_run(" | "); + p.add_run("Highlighted").highlight("FFFF00"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("8pt small").size(8.0); + p.add_run(" | "); + p.add_run("11pt normal").size(11.0); + p.add_run(" | "); + p.add_run("16pt large").size(16.0); + p.add_run(" | "); + p.add_run("24pt extra-large").size(24.0); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Arial font").font("Arial"); + p.add_run(" | "); + p.add_run("Times New Roman font").font("Times New Roman"); + p.add_run(" | "); + p.add_run("Courier New font").font("Courier New"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Normal"); + p.add_run(" H").size(11.0); + p.add_run("2").subscript(); + p.add_run("O (subscript)").size(11.0); + p.add_run(" | E = mc").size(11.0); + p.add_run("2").superscript(); + p.add_run(" (superscript)").size(11.0); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("ALL CAPS").all_caps(true); + p.add_run(" | "); + p.add_run("Small Caps").small_caps(true); + p.add_run(" | "); + p.add_run("Expanded spacing") + .character_spacing(Length::twips(40)); + } + + doc.add_paragraph(""); + + // --- Paragraph formatting --- + doc.add_paragraph("Paragraph Formatting").style("Heading2"); + + doc.add_paragraph("Paragraph with shading (light green background)") + .shading("E2EFDA"); + + doc.add_paragraph("Paragraph with bottom border") + .border_bottom(BorderStyle::Single, 6, "2E75B6"); + + doc.add_paragraph("Paragraph with all borders") + .border_all(BorderStyle::Single, 4, "FF0000"); + + doc.add_paragraph("Paragraph with 1-inch left indent and hanging indent") + .indent_left(Length::inches(1.0)) + .hanging_indent(Length::inches(0.5)); + + doc.add_paragraph("Paragraph with first-line indent of 0.5 inches") + .first_line_indent(Length::inches(0.5)); + + doc.add_paragraph("Paragraph with extra space before (24pt) and after (12pt)") + .space_before(Length::pt(24.0)) + .space_after(Length::pt(12.0)); + + doc.add_paragraph( + "Paragraph with double line spacing. This text should have extra vertical \ + space between lines to demonstrate the line_spacing_multiple setting.", + ) + .line_spacing_multiple(2.0); + + doc.add_paragraph("Paragraph with keep-with-next (won't break from the next paragraph)") + .keep_with_next(true); + doc.add_paragraph("(This stays with the paragraph above.)"); + + // ========================================================================= + // PAGE 3: LISTS & TAB STOPS + // ========================================================================= + doc.add_paragraph("").page_break_before(true); + + doc.add_paragraph("2. Lists").style("Heading1"); + + doc.add_paragraph("Bullet List").style("Heading2"); + + doc.add_bullet_list_item("First bullet item", 0); + doc.add_bullet_list_item("Second bullet item", 0); + doc.add_bullet_list_item("Nested level 1", 1); + doc.add_bullet_list_item("Nested level 2", 2); + doc.add_bullet_list_item("Back to level 1", 1); + doc.add_bullet_list_item("Third bullet item", 0); + + doc.add_paragraph(""); + + doc.add_paragraph("Numbered List").style("Heading2"); + + doc.add_numbered_list_item("First numbered item", 0); + doc.add_numbered_list_item("Second numbered item", 0); + doc.add_numbered_list_item("Sub-item A", 1); + doc.add_numbered_list_item("Sub-item B", 1); + doc.add_numbered_list_item("Third numbered item", 0); + + doc.add_paragraph(""); + + // --- Tab stops --- + doc.add_paragraph("Tab Stops").style("Heading2"); + + doc.add_paragraph("Left\tCenter\tRight\tDecimal") + .add_tab_stop(TabAlignment::Left, Length::inches(0.0)) + .add_tab_stop(TabAlignment::Center, Length::inches(2.5)) + .add_tab_stop(TabAlignment::Right, Length::inches(5.0)) + .add_tab_stop(TabAlignment::Decimal, Length::inches(6.5)); + + doc.add_paragraph("Item\t........\tPrice") + .add_tab_stop_with_leader(TabAlignment::Left, Length::inches(0.0), TabLeader::None) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(4.0), TabLeader::Dot) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(5.0), TabLeader::None); + + doc.add_paragraph("Widget A\t........\t$19.99") + .add_tab_stop_with_leader(TabAlignment::Left, Length::inches(0.0), TabLeader::None) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(4.0), TabLeader::Dot) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(5.0), TabLeader::None); + + doc.add_paragraph("Gadget B\t________\t$249.50") + .add_tab_stop_with_leader(TabAlignment::Left, Length::inches(0.0), TabLeader::None) + .add_tab_stop_with_leader( + TabAlignment::Right, + Length::inches(4.0), + TabLeader::Underscore, + ) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(5.0), TabLeader::None); + + // ========================================================================= + // PAGE 4: TABLES + // ========================================================================= + doc.add_paragraph("").page_break_before(true); + + doc.add_paragraph("3. Tables").style("Heading1"); + + // --- Basic table with borders --- + doc.add_paragraph("Basic Table with Borders") + .style("Heading2"); + + { + let mut tbl = doc.add_table(4, 3); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + + // Header row + for col in 0..3 { + tbl.cell(0, col).unwrap().shading("2E75B6"); + } + tbl.cell(0, 0).unwrap().set_text("Name"); + tbl.cell(0, 1).unwrap().set_text("Role"); + tbl.cell(0, 2).unwrap().set_text("Location"); + + tbl.cell(1, 0).unwrap().set_text("Alice Johnson"); + tbl.cell(1, 1).unwrap().set_text("Engineering Lead"); + tbl.cell(1, 2).unwrap().set_text("New York"); + + tbl.cell(2, 0).unwrap().set_text("Bob Smith"); + tbl.cell(2, 1).unwrap().set_text("Product Manager"); + tbl.cell(2, 2).unwrap().set_text("San Francisco"); + + tbl.cell(3, 0).unwrap().set_text("Carol Davis"); + tbl.cell(3, 1).unwrap().set_text("Designer"); + tbl.cell(3, 2).unwrap().set_text("London"); + } + + doc.add_paragraph(""); + + // --- Table with cell merging --- + doc.add_paragraph("Table with Cell Merging & Vertical Alignment") + .style("Heading2"); + + { + let mut tbl = doc.add_table(4, 4); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + tbl = tbl.width_pct(100.0); + + // Header spanning all columns + tbl.cell(0, 0).unwrap().set_text("Quarterly Revenue Report"); + tbl.cell(0, 0).unwrap().shading("1F4E79"); + tbl.cell(0, 0).unwrap().grid_span(4); + + // Sub-header + tbl.cell(1, 0).unwrap().set_text("Region"); + tbl.cell(1, 0).unwrap().shading("D6E4F0"); + tbl.cell(1, 1).unwrap().set_text("Q1"); + tbl.cell(1, 1).unwrap().shading("D6E4F0"); + tbl.cell(1, 2).unwrap().set_text("Q2"); + tbl.cell(1, 2).unwrap().shading("D6E4F0"); + tbl.cell(1, 3).unwrap().set_text("Total"); + tbl.cell(1, 3).unwrap().shading("D6E4F0"); + + // Data + tbl.cell(2, 0).unwrap().set_text("North America"); + tbl.cell(2, 1).unwrap().set_text("$2.4M"); + tbl.cell(2, 2).unwrap().set_text("$2.7M"); + tbl.cell(2, 3).unwrap().set_text("$5.1M"); + + tbl.cell(3, 0).unwrap().set_text("Europe"); + tbl.cell(3, 1).unwrap().set_text("$1.8M"); + tbl.cell(3, 2).unwrap().set_text("$2.0M"); + tbl.cell(3, 3).unwrap().set_text("$3.8M"); + + // Vertical alignment on data cells + tbl.cell(2, 3) + .unwrap() + .vertical_alignment(VerticalAlignment::Center); + tbl.cell(3, 3) + .unwrap() + .vertical_alignment(VerticalAlignment::Bottom); + } + + doc.add_paragraph(""); + + // --- Table with vertical merge --- + doc.add_paragraph("Table with Vertical Merge") + .style("Heading2"); + + { + let mut tbl = doc.add_table(4, 3); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + + tbl.cell(0, 0).unwrap().set_text("Category"); + tbl.cell(0, 0).unwrap().shading("E2EFDA"); + tbl.cell(0, 1).unwrap().set_text("Item"); + tbl.cell(0, 1).unwrap().shading("E2EFDA"); + tbl.cell(0, 2).unwrap().set_text("Price"); + tbl.cell(0, 2).unwrap().shading("E2EFDA"); + + // "Hardware" spans rows 1-2 + tbl.cell(1, 0).unwrap().set_text("Hardware"); + tbl.cell(1, 0).unwrap().v_merge_restart(); + tbl.cell(1, 1).unwrap().set_text("Laptop"); + tbl.cell(1, 2).unwrap().set_text("$1,200"); + + tbl.cell(2, 0).unwrap().v_merge_continue(); + tbl.cell(2, 1).unwrap().set_text("Monitor"); + tbl.cell(2, 2).unwrap().set_text("$450"); + + // "Software" on row 3 + tbl.cell(3, 0).unwrap().set_text("Software"); + tbl.cell(3, 1).unwrap().set_text("IDE License"); + tbl.cell(3, 2).unwrap().set_text("$200/yr"); + } + + doc.add_paragraph(""); + + // --- Nested table --- + doc.add_paragraph("Nested Table").style("Heading2"); + + { + let mut tbl = doc.add_table(2, 2); + tbl = tbl.borders(BorderStyle::Single, 6, "2E75B6"); + + tbl.cell(0, 0).unwrap().set_text("Outer Cell (0,0)"); + tbl.cell(0, 1).unwrap().set_text("Outer Cell (0,1)"); + tbl.cell(1, 0).unwrap().set_text("Outer Cell (1,0)"); + + // Nested table inside cell (1,1) + { + let mut cell = tbl.cell(1, 1).unwrap(); + cell.set_text("Contains nested table:"); + let mut nested = cell.add_table(2, 2); + nested = nested.borders(BorderStyle::Single, 2, "FF6600"); + nested.cell(0, 0).unwrap().set_text("Inner A"); + nested.cell(0, 1).unwrap().set_text("Inner B"); + nested.cell(1, 0).unwrap().set_text("Inner C"); + nested.cell(1, 1).unwrap().set_text("Inner D"); + } + } + + // ========================================================================= + // PAGE 5: IMAGES + // ========================================================================= + doc.add_paragraph("").page_break_before(true); + + doc.add_paragraph("4. Images").style("Heading1"); + + doc.add_paragraph("Inline Image").style("Heading2"); + + doc.add_paragraph("Below is an inline image (200x50 pixels, blue gradient):"); + let inline_img = create_sample_png(200, 50, [0, 80, 200]); + doc.add_picture( + &inline_img, + "inline_chart.png", + Length::inches(3.0), + Length::inches(0.75), + ); + + doc.add_paragraph(""); + + doc.add_paragraph("Header Image").style("Heading2"); + + // Replace the text-only header with an image header + let header_img = create_sample_png(400, 40, [40, 40, 40]); + doc.set_header_image( + &header_img, + "header_logo.png", + Length::inches(2.0), + Length::inches(0.2), + ); + + doc.add_paragraph( + "The document header has been replaced with an inline image. \ + Check the header area at the top of this page.", + ); + + doc.add_paragraph(""); + doc.add_paragraph( + "Note: The cover page uses a full-page background image behind the text, \ + demonstrated on page 1 via add_background_image().", + ); + + // ========================================================================= + // PAGE 6: CONTENT MANIPULATION — placeholder replacement, insertion + // ========================================================================= + doc.add_paragraph("").page_break_before(true); + + doc.add_paragraph("5. Content Manipulation") + .style("Heading1"); + + // --- Placeholder replacement --- + doc.add_paragraph("Placeholder Replacement") + .style("Heading2"); + + doc.add_paragraph( + "Before replacement, this document contained {{customer}} and {{date}} placeholders.", + ); + + { + let mut p = doc.add_paragraph(""); + p.add_run("Customer: ").bold(true); + p.add_run("{{customer}}"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Date: ").bold(true); + p.add_run("{{date}}"); + } + doc.add_paragraph("Reference: {{ref_number}}"); + + // Table with placeholders + { + let mut tbl = doc.add_table(3, 2); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + tbl.cell(0, 0).unwrap().set_text("Field"); + tbl.cell(0, 0).unwrap().shading("D6E4F0"); + tbl.cell(0, 1).unwrap().set_text("Value"); + tbl.cell(0, 1).unwrap().shading("D6E4F0"); + tbl.cell(1, 0).unwrap().set_text("Project"); + tbl.cell(1, 1).unwrap().set_text("{{project}}"); + tbl.cell(2, 0).unwrap().set_text("Status"); + tbl.cell(2, 1).unwrap().set_text("{{status}}"); + } + + // Perform replacements + let mut replacements = HashMap::new(); + replacements.insert("{{customer}}", "Acme Corporation"); + replacements.insert("{{date}}", "February 22, 2026"); + replacements.insert("{{ref_number}}", "REF-2026-001"); + replacements.insert("{{project}}", "Infrastructure Upgrade"); + replacements.insert("{{status}}", "In Progress"); + let replace_count = doc.replace_all(&replacements); + + doc.add_paragraph(""); + doc.add_paragraph(&format!( + "(Replaced {} placeholders above — in body text and table cells)", + replace_count + )); + + doc.add_paragraph(""); + + // --- Content insertion --- + doc.add_paragraph("Content Insertion").style("Heading2"); + + doc.add_paragraph("Section A: First section of content."); + doc.add_paragraph("Section C: Third section of content."); + + // Insert "Section B" between A and C + if let Some(idx) = doc.find_content_index("Section C") { + doc.insert_paragraph( + idx, + "Section B: Inserted between A and C using find_content_index().", + ); + } + + doc.add_paragraph(""); + doc.add_paragraph( + "The paragraph above ('Section B') was inserted at a specific position \ + using find_content_index() + insert_paragraph().", + ); + + // ========================================================================= + // PAGE 7: LANDSCAPE — section break, wide table + // ========================================================================= + doc.add_paragraph("").section_break(SectionBreak::NextPage); + + doc.add_paragraph("6. Mixed Page Orientation") + .style("Heading1"); + + doc.add_paragraph( + "This page is in LANDSCAPE orientation. It was created using a section break \ + followed by section_landscape(). This is useful for wide tables or charts.", + ); + + doc.add_paragraph(""); + + // Wide table for landscape + { + let mut tbl = doc.add_table(4, 7); + tbl = tbl.borders(BorderStyle::Single, 4, "2E75B6"); + + let headers = ["Region", "Jan", "Feb", "Mar", "Apr", "May", "Total"]; + for (col, h) in headers.iter().enumerate() { + tbl.cell(0, col).unwrap().set_text(h); + tbl.cell(0, col).unwrap().shading("2E75B6"); + } + + let data = [ + [ + "North America", + "$1.2M", + "$1.3M", + "$1.4M", + "$1.5M", + "$1.6M", + "$7.0M", + ], + [ + "Europe", "$0.8M", "$0.9M", "$0.9M", "$1.0M", "$1.1M", "$4.7M", + ], + [ + "Asia Pacific", + "$0.5M", + "$0.6M", + "$0.7M", + "$0.7M", + "$0.8M", + "$3.3M", + ], + ]; + for (row_idx, row_data) in data.iter().enumerate() { + for (col, val) in row_data.iter().enumerate() { + tbl.cell(row_idx + 1, col).unwrap().set_text(val); + } + } + } + + // End landscape, return to portrait + doc.add_paragraph("") + .section_break(SectionBreak::NextPage) + .section_landscape(); + + // ========================================================================= + // PAGE 8: BACK TO PORTRAIT — styles, final notes + // ========================================================================= + doc.add_paragraph("7. Custom Styles & Summary") + .style("Heading1"); + + doc.add_paragraph( + "This final page is back in portrait orientation after a section break. \ + The document has demonstrated:", + ); + + doc.add_bullet_list_item( + "Page setup: size, margins, header/footer distance, gutter", + 0, + ); + doc.add_bullet_list_item("Document metadata: title, author, subject, keywords", 0); + doc.add_bullet_list_item("Headers and footers: text, images, different first page", 0); + doc.add_bullet_list_item("Background images: full-page behind text", 0); + doc.add_bullet_list_item( + "Text formatting: bold, italic, underline, strike, color, size, font", + 0, + ); + doc.add_bullet_list_item( + "Advanced run formatting: superscript, subscript, caps, spacing", + 0, + ); + doc.add_bullet_list_item( + "Paragraph formatting: alignment, borders, shading, spacing, indentation", + 0, + ); + doc.add_bullet_list_item("Bullet and numbered lists with nesting levels", 0); + doc.add_bullet_list_item("Tab stops with dot/underscore leaders", 0); + doc.add_bullet_list_item( + "Tables: borders, shading, column spans, row spans, nesting", + 0, + ); + doc.add_bullet_list_item("Vertical alignment in table cells", 0); + doc.add_bullet_list_item("Inline images", 0); + doc.add_bullet_list_item("Placeholder replacement in body and table cells", 0); + doc.add_bullet_list_item("Content insertion at specific positions", 0); + doc.add_bullet_list_item( + "Section breaks with mixed portrait/landscape orientation", + 0, + ); + + doc.add_paragraph(""); + doc.add_paragraph("All features above were built entirely from scratch using the rdocx API.") + .alignment(Alignment::Center) + .shading("E2EFDA") + .border_all(BorderStyle::Single, 2, "00AA00"); + + doc.save(path).unwrap(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Helper: create a minimal valid PNG (solid color with slight gradient) +// ───────────────────────────────────────────────────────────────────────────── +fn create_sample_png(width: u32, height: u32, base_color: [u8; 3]) -> Vec { + let mut pixels = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + let fy = y as f64 / height as f64; + let fx = x as f64 / width as f64; + let r = (base_color[0] as f64 + (1.0 - fy) * 40.0).min(255.0) as u8; + let g = (base_color[1] as f64 + fx * 30.0).min(255.0) as u8; + let b = (base_color[2] as f64 + fy * 60.0).min(255.0) as u8; + pixels.extend_from_slice(&[r, g, b, 255]); + } + } + + let mut png_data = Vec::new(); + { + use std::io::Write; + + // PNG Signature + png_data + .write_all(&[137, 80, 78, 71, 13, 10, 26, 10]) + .unwrap(); + + // IHDR chunk + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&height.to_be_bytes()); + ihdr.push(8); // bit depth + ihdr.push(6); // color type: RGBA + ihdr.push(0); // compression + ihdr.push(0); // filter + ihdr.push(0); // interlace + write_png_chunk(&mut png_data, b"IHDR", &ihdr); + + // IDAT chunk + let mut raw_data = Vec::new(); + for y in 0..height { + raw_data.push(0); // filter: None + let row_start = (y * width * 4) as usize; + let row_end = row_start + (width * 4) as usize; + raw_data.extend_from_slice(&pixels[row_start..row_end]); + } + + let compressed = zlib_store(&raw_data); + write_png_chunk(&mut png_data, b"IDAT", &compressed); + + // IEND chunk + write_png_chunk(&mut png_data, b"IEND", &[]); + } + + png_data +} + +fn write_png_chunk(out: &mut Vec, chunk_type: &[u8; 4], data: &[u8]) { + use std::io::Write; + let len = data.len() as u32; + out.write_all(&len.to_be_bytes()).unwrap(); + out.write_all(chunk_type).unwrap(); + out.write_all(data).unwrap(); + let crc = crc32(chunk_type, data); + out.write_all(&crc.to_be_bytes()).unwrap(); +} + +fn crc32(chunk_type: &[u8], data: &[u8]) -> u32 { + static CRC_TABLE: std::sync::LazyLock<[u32; 256]> = std::sync::LazyLock::new(|| { + let mut table = [0u32; 256]; + for n in 0..256u32 { + let mut c = n; + for _ in 0..8 { + if c & 1 != 0 { + c = 0xEDB88320 ^ (c >> 1); + } else { + c >>= 1; + } + } + table[n as usize] = c; + } + table + }); + + let mut crc = 0xFFFFFFFF_u32; + for &byte in chunk_type.iter().chain(data.iter()) { + let index = ((crc ^ byte as u32) & 0xFF) as usize; + crc = CRC_TABLE[index] ^ (crc >> 8); + } + crc ^ 0xFFFFFFFF +} + +fn zlib_store(data: &[u8]) -> Vec { + let mut out = Vec::new(); + out.push(0x78); + out.push(0x01); + + let chunks: Vec<&[u8]> = data.chunks(65535).collect(); + for (i, chunk) in chunks.iter().enumerate() { + let is_last = i == chunks.len() - 1; + out.push(if is_last { 0x01 } else { 0x00 }); + let len = chunk.len() as u16; + let nlen = !len; + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&nlen.to_le_bytes()); + out.extend_from_slice(chunk); + } + + let adler = adler32(data); + out.extend_from_slice(&adler.to_be_bytes()); + out +} + +fn adler32(data: &[u8]) -> u32 { + let mut a: u32 = 1; + let mut b: u32 = 0; + for &byte in data { + a = (a + byte as u32) % 65521; + b = (b + a) % 65521; + } + (b << 16) | a +} diff --git a/vendor/rdocx/examples/header_banner.rs b/vendor/rdocx/examples/header_banner.rs new file mode 100644 index 00000000..128de251 --- /dev/null +++ b/vendor/rdocx/examples/header_banner.rs @@ -0,0 +1,444 @@ +//! Custom header banner with DrawingML group shapes. +//! +//! This example demonstrates how to build a professional header banner +//! using raw DrawingML XML with set_raw_header_with_images(). The banner +//! consists of a colored rectangle background with a logo image overlaid. +//! +//! Run with: cargo run --example header_banner + +use std::fmt::Write; +use std::path::Path; + +use rdocx::{Document, Length}; + +fn main() { + let samples_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("samples"); + std::fs::create_dir_all(&samples_dir).unwrap(); + + let out = samples_dir.join("header_banner.docx"); + generate_header_banner_doc(&out); + println!(" Created: header_banner.docx"); + println!("\nDone!"); +} + +fn generate_header_banner_doc(path: &Path) { + let mut doc = Document::new(); + + // Page setup with extra top margin for the banner + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::twips(2292), // top — extra tall for header banner + Length::twips(1440), // right + Length::twips(1440), // bottom + Length::twips(1440), // left + ); + doc.set_header_footer_distance(Length::twips(720), Length::twips(432)); + + // Generate a simple logo image (white text on transparent background) + let logo_img = create_logo_png(220, 48); + + // ── Dark blue banner ── + let banner = build_header_banner_xml( + "rId1", + &BannerOpts { + bg_color: "1A3C6E", + banner_width: 7772400, // full page width in EMU (~8.5") + banner_height: 969026, // banner height in EMU (~1.06") + logo_width: 2011680, // logo display width (~2.2") + logo_height: 438912, // logo display height (~0.48") + logo_x_offset: 295125, // left padding + logo_y_offset: 265057, // vertical centering + }, + ); + + doc.set_raw_header_with_images( + banner.clone(), + &[("rId1", &logo_img, "logo.png")], + rdocx_oxml::header_footer::HdrFtrType::Default, + ); + + // Use a different first page header (same banner, different color) + doc.set_different_first_page(true); + let first_page_banner = build_header_banner_xml( + "rId1", + &BannerOpts { + bg_color: "2E75B6", // lighter blue for cover + banner_width: 7772400, + banner_height: 969026, + logo_width: 2011680, + logo_height: 438912, + logo_x_offset: 295125, + logo_y_offset: 265057, + }, + ); + doc.set_raw_header_with_images( + first_page_banner, + &[("rId1", &logo_img, "logo.png")], + rdocx_oxml::header_footer::HdrFtrType::First, + ); + + // Footer + doc.set_footer("Confidential — Internal Use Only"); + + // ── Page 1: Cover ── + doc.add_paragraph("Company Report").style("Heading1"); + + doc.add_paragraph( + "This document demonstrates a custom header banner built with DrawingML \ + group shapes. The banner uses a colored rectangle with a logo image overlaid, \ + positioned at the top of each page.", + ); + + doc.add_paragraph(""); + + doc.add_paragraph("How the Header Banner Works") + .style("Heading2"); + + doc.add_paragraph( + "The header banner is built using set_raw_header_with_images(), which \ + accepts raw XML and a list of (rel_id, image_data, filename) tuples. \ + The XML uses a DrawingML group shape (wpg:wgp) containing:", + ); + + doc.add_bullet_list_item( + "A wps:wsp rectangle shape with a solid color fill (the background bar)", + 0, + ); + doc.add_bullet_list_item( + "A pic:pic image element positioned within the group (the logo)", + 0, + ); + doc.add_bullet_list_item( + "The group is wrapped in a wp:anchor element for absolute page positioning", + 0, + ); + + doc.add_paragraph(""); + + doc.add_paragraph("Customization").style("Heading2"); + + doc.add_paragraph( + "All dimensions are in EMU (English Metric Units) where 914400 EMU = 1 inch. \ + You can customize:", + ); + + doc.add_bullet_list_item("bg_color — any hex color for the rectangle background", 0); + doc.add_bullet_list_item("banner_width / banner_height — size of the full banner", 0); + doc.add_bullet_list_item("logo_width / logo_height — display size of the logo", 0); + doc.add_bullet_list_item( + "logo_x_offset / logo_y_offset — logo position within the banner", + 0, + ); + + doc.add_paragraph(""); + + doc.add_paragraph("Different First Page").style("Heading2"); + + doc.add_paragraph( + "This page uses a lighter blue banner (first page header). \ + Subsequent pages use a darker blue banner (default header). \ + Use set_different_first_page(true) to enable this.", + ); + + // ── Page 2 ── + doc.add_paragraph("").page_break_before(true); + + doc.add_paragraph("Second Page").style("Heading1"); + + doc.add_paragraph( + "This page shows the default header banner (dark blue). The first page \ + had a lighter blue banner because we set a different first-page header.", + ); + + doc.add_paragraph(""); + + doc.add_paragraph( + "The banner repeats on every page because it is placed in the header part. \ + You can have different banners for default, first-page, and even-page headers.", + ); + + doc.set_title("Header Banner Example"); + doc.set_author("rdocx"); + + doc.save(path).unwrap(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Header banner builder +// ───────────────────────────────────────────────────────────────────────────── + +struct BannerOpts<'a> { + bg_color: &'a str, + banner_width: i64, + banner_height: i64, + logo_width: i64, + logo_height: i64, + logo_x_offset: i64, + logo_y_offset: i64, +} + +/// Build complete `` XML for a banner header (colored rect + logo image). +/// +/// `image_rel_id` is the rId that will reference the logo image (e.g. "rId1"). +fn build_header_banner_xml(image_rel_id: &str, opts: &BannerOpts) -> Vec { + let mut xml = String::with_capacity(2048); + + write!( + xml, + r#""# + ) + .unwrap(); + write!(xml, r#""# + ) + .unwrap(); + + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + + // Anchor at top-left of page + write!( + xml, + r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!( + xml, + r#"0"# + ) + .unwrap(); + write!( + xml, + r#"0"# + ) + .unwrap(); + write!( + xml, + r#""#, + opts.banner_width, opts.banner_height + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + + // Group shape containing rect + image + write!(xml, r#""#).unwrap(); + write!( + xml, + r#""# + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + write!( + xml, + r#""#, + w = opts.banner_width, + h = opts.banner_height + ) + .unwrap(); + write!( + xml, + r#""#, + w = opts.banner_width, + h = opts.banner_height + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + + // Background rectangle + write!( + xml, + r#""# + ) + .unwrap(); + write!( + xml, + r#""#, + w = opts.banner_width, + h = opts.banner_height + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + write!( + xml, + r#""#, + opts.bg_color + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + + // Logo image + write!( + xml, + r#""# + ) + .unwrap(); + write!(xml, r#""#, image_rel_id).unwrap(); + write!( + xml, + r#""#, + opts.logo_x_offset, opts.logo_y_offset, opts.logo_width, opts.logo_height + ) + .unwrap(); + write!(xml, r#""#).unwrap(); + write!( + xml, + r#""# + ) + .unwrap(); + + // Close all + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + write!(xml, r#""#).unwrap(); + + xml.into_bytes() +} + +// ───────────────────────────────────────────────────────────────────────────── +// PNG generation helpers +// ───────────────────────────────────────────────────────────────────────────── + +/// Create a simple logo PNG (white text area on dark background). +fn create_logo_png(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity((width * height * 4) as usize); + for y in 0..height { + for x in 0..width { + // White rectangle in the center area (simulates logo text) + let in_text_area = + x > width / 8 && x < width * 7 / 8 && y > height / 4 && y < height * 3 / 4; + if in_text_area { + pixels.extend_from_slice(&[255, 255, 255, 255]); + } else { + pixels.extend_from_slice(&[255, 255, 255, 40]); // mostly transparent + } + } + } + encode_png(width, height, &pixels) +} + +fn encode_png(width: u32, height: u32, pixels: &[u8]) -> Vec { + let mut png = Vec::new(); + { + use std::io::Write as _; + png.write_all(&[137, 80, 78, 71, 13, 10, 26, 10]).unwrap(); + + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&height.to_be_bytes()); + ihdr.extend_from_slice(&[8, 6, 0, 0, 0]); // 8-bit RGBA + write_chunk(&mut png, b"IHDR", &ihdr); + + let mut raw = Vec::new(); + for y in 0..height { + raw.push(0); + let s = (y * width * 4) as usize; + raw.extend_from_slice(&pixels[s..s + (width * 4) as usize]); + } + write_chunk(&mut png, b"IDAT", &zlib_store(&raw)); + write_chunk(&mut png, b"IEND", &[]); + } + png +} + +fn write_chunk(out: &mut Vec, ct: &[u8; 4], data: &[u8]) { + use std::io::Write as _; + out.write_all(&(data.len() as u32).to_be_bytes()).unwrap(); + out.write_all(ct).unwrap(); + out.write_all(data).unwrap(); + out.write_all(&crc32(ct, data).to_be_bytes()).unwrap(); +} + +fn crc32(ct: &[u8], data: &[u8]) -> u32 { + static T: std::sync::LazyLock<[u32; 256]> = std::sync::LazyLock::new(|| { + let mut t = [0u32; 256]; + for n in 0..256u32 { + let mut c = n; + for _ in 0..8 { + c = if c & 1 != 0 { + 0xEDB88320 ^ (c >> 1) + } else { + c >> 1 + }; + } + t[n as usize] = c; + } + t + }); + let mut c = 0xFFFFFFFF_u32; + for &b in ct.iter().chain(data) { + c = T[((c ^ b as u32) & 0xFF) as usize] ^ (c >> 8); + } + c ^ 0xFFFFFFFF +} + +fn zlib_store(data: &[u8]) -> Vec { + let mut out = vec![0x78, 0x01]; + for (i, chunk) in data.chunks(65535).enumerate() { + let last = i == data.chunks(65535).count() - 1; + out.push(if last { 0x01 } else { 0x00 }); + let len = chunk.len() as u16; + out.extend_from_slice(&len.to_le_bytes()); + out.extend_from_slice(&(!len).to_le_bytes()); + out.extend_from_slice(chunk); + } + let (mut a, mut b) = (1u32, 0u32); + for &byte in data { + a = (a + byte as u32) % 65521; + b = (b + a) % 65521; + } + out.extend_from_slice(&((b << 16) | a).to_be_bytes()); + out +} diff --git a/vendor/rdocx/examples/styled_tables.rs b/vendor/rdocx/examples/styled_tables.rs new file mode 100644 index 00000000..393ad923 --- /dev/null +++ b/vendor/rdocx/examples/styled_tables.rs @@ -0,0 +1,449 @@ +//! Styled tables showcase — demonstrates all table formatting options. +//! +//! Run with: cargo run --example styled_tables + +use std::path::Path; + +use rdocx::{BorderStyle, Document, Length, VerticalAlignment}; + +fn main() { + let samples_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("samples"); + std::fs::create_dir_all(&samples_dir).unwrap(); + + let out = samples_dir.join("styled_tables.docx"); + generate_styled_tables(&out); + println!(" Created: styled_tables.docx"); + println!("\nDone!"); +} + +fn generate_styled_tables(path: &Path) { + let mut doc = Document::new(); + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(0.75), + Length::inches(0.75), + Length::inches(0.75), + Length::inches(0.75), + ); + + doc.add_paragraph("Styled Tables Showcase") + .style("Heading1"); + + doc.add_paragraph(""); + + // ========================================================================= + // 1. Professional report table with alternating rows + // ========================================================================= + doc.add_paragraph("1. Report Table with Alternating Row Colors") + .style("Heading2"); + + { + let mut tbl = doc.add_table(8, 4); + tbl = tbl.borders(BorderStyle::Single, 2, "BFBFBF"); + tbl = tbl.width_pct(100.0); + + // Header row + let headers = ["Product", "Q1 Sales", "Q2 Sales", "Growth"]; + for (col, h) in headers.iter().enumerate() { + tbl.cell(0, col).unwrap().shading("2E75B6"); + tbl.cell(0, col).unwrap().set_text(h); + } + tbl.row(0).unwrap().header(); + + // Data with alternating shading + let data = [ + ["Enterprise Suite", "$245,000", "$312,000", "+27.3%"], + ["Professional", "$189,000", "$201,000", "+6.3%"], + ["Starter Pack", "$67,000", "$84,500", "+26.1%"], + ["Add-ons", "$34,000", "$41,200", "+21.2%"], + ["Training", "$22,000", "$28,900", "+31.4%"], + ["Support Plans", "$56,000", "$62,300", "+11.3%"], + ]; + + for (i, row) in data.iter().enumerate() { + let row_idx = i + 1; + for (col, val) in row.iter().enumerate() { + tbl.cell(row_idx, col).unwrap().set_text(val); + // Alternate row colors + if i % 2 == 0 { + tbl.cell(row_idx, col).unwrap().shading("F2F7FB"); + } + } + } + + // Total row + tbl.cell(7, 0).unwrap().set_text("TOTAL"); + tbl.cell(7, 0).unwrap().shading("D6E4F0"); + tbl.cell(7, 1).unwrap().set_text("$613,000"); + tbl.cell(7, 1).unwrap().shading("D6E4F0"); + tbl.cell(7, 2).unwrap().set_text("$729,900"); + tbl.cell(7, 2).unwrap().shading("D6E4F0"); + tbl.cell(7, 3).unwrap().set_text("+19.1%"); + tbl.cell(7, 3).unwrap().shading("D6E4F0"); + } + + doc.add_paragraph(""); + + // ========================================================================= + // 2. Invoice-style table with merged header + // ========================================================================= + doc.add_paragraph("2. Invoice Table with Merged Header & Row Spans") + .style("Heading2"); + + { + let mut tbl = doc.add_table(7, 4); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + tbl = tbl.width_pct(100.0); + + // Merged title row + tbl.cell(0, 0).unwrap().set_text("INVOICE #2026-0042"); + tbl.cell(0, 0).unwrap().grid_span(4); + tbl.cell(0, 0).unwrap().shading("1F4E79"); + + // Column headers + let headers = ["Item", "Description", "Qty", "Amount"]; + for (col, h) in headers.iter().enumerate() { + tbl.cell(1, col).unwrap().set_text(h); + tbl.cell(1, col).unwrap().shading("D6E4F0"); + } + + // Line items + tbl.cell(2, 0).unwrap().set_text("LIC-ENT-500"); + tbl.cell(2, 1) + .unwrap() + .set_text("Enterprise License (500 seats)"); + tbl.cell(2, 2).unwrap().set_text("1"); + tbl.cell(2, 3).unwrap().set_text("$60,000"); + + tbl.cell(3, 0).unwrap().set_text("SVC-IMPL"); + tbl.cell(3, 1).unwrap().set_text("Implementation Services"); + tbl.cell(3, 2).unwrap().set_text("1"); + tbl.cell(3, 3).unwrap().set_text("$25,000"); + + tbl.cell(4, 0).unwrap().set_text("SVC-TRAIN"); + tbl.cell(4, 1) + .unwrap() + .set_text("On-site Training (3 days)"); + tbl.cell(4, 2).unwrap().set_text("1"); + tbl.cell(4, 3).unwrap().set_text("$4,500"); + + // Subtotal + tbl.cell(5, 0).unwrap().set_text("Subtotal"); + tbl.cell(5, 0).unwrap().grid_span(3); + tbl.cell(5, 0).unwrap().shading("F2F2F2"); + tbl.cell(5, 3).unwrap().set_text("$89,500"); + tbl.cell(5, 3).unwrap().shading("F2F2F2"); + + // Total + tbl.cell(6, 0).unwrap().set_text("TOTAL DUE"); + tbl.cell(6, 0).unwrap().grid_span(3); + tbl.cell(6, 0).unwrap().shading("1F4E79"); + tbl.cell(6, 3).unwrap().set_text("$89,500"); + tbl.cell(6, 3).unwrap().shading("1F4E79"); + } + + doc.add_paragraph(""); + + // ========================================================================= + // 3. Specification table with vertical merge + // ========================================================================= + doc.add_paragraph("3. Specification Table with Vertical Merges") + .style("Heading2"); + + { + let mut tbl = doc.add_table(8, 3); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + tbl = tbl.width_pct(100.0); + + // Header + tbl.cell(0, 0).unwrap().set_text("Category"); + tbl.cell(0, 0).unwrap().shading("2E75B6"); + tbl.cell(0, 1).unwrap().set_text("Specification"); + tbl.cell(0, 1).unwrap().shading("2E75B6"); + tbl.cell(0, 2).unwrap().set_text("Value"); + tbl.cell(0, 2).unwrap().shading("2E75B6"); + + // "Hardware" spans 3 rows + tbl.cell(1, 0).unwrap().set_text("Hardware"); + tbl.cell(1, 0).unwrap().v_merge_restart(); + tbl.cell(1, 0).unwrap().shading("E2EFDA"); + tbl.cell(1, 0) + .unwrap() + .vertical_alignment(VerticalAlignment::Center); + tbl.cell(1, 1).unwrap().set_text("Processor"); + tbl.cell(1, 2).unwrap().set_text("Intel Xeon E-2388G"); + + tbl.cell(2, 0).unwrap().v_merge_continue(); + tbl.cell(2, 1).unwrap().set_text("Memory"); + tbl.cell(2, 2).unwrap().set_text("64 GB DDR4 ECC"); + + tbl.cell(3, 0).unwrap().v_merge_continue(); + tbl.cell(3, 1).unwrap().set_text("Storage"); + tbl.cell(3, 2).unwrap().set_text("2x 1TB NVMe SSD (RAID 1)"); + + // "Network" spans 2 rows + tbl.cell(4, 0).unwrap().set_text("Network"); + tbl.cell(4, 0).unwrap().v_merge_restart(); + tbl.cell(4, 0).unwrap().shading("FCE4D6"); + tbl.cell(4, 0) + .unwrap() + .vertical_alignment(VerticalAlignment::Center); + tbl.cell(4, 1).unwrap().set_text("Ethernet"); + tbl.cell(4, 2).unwrap().set_text("4x 10GbE SFP+"); + + tbl.cell(5, 0).unwrap().v_merge_continue(); + tbl.cell(5, 1).unwrap().set_text("Management"); + tbl.cell(5, 2).unwrap().set_text("1x 1GbE IPMI"); + + // "Software" spans 2 rows + tbl.cell(6, 0).unwrap().set_text("Software"); + tbl.cell(6, 0).unwrap().v_merge_restart(); + tbl.cell(6, 0).unwrap().shading("D6E4F0"); + tbl.cell(6, 0) + .unwrap() + .vertical_alignment(VerticalAlignment::Center); + tbl.cell(6, 1).unwrap().set_text("Operating System"); + tbl.cell(6, 2).unwrap().set_text("Ubuntu 24.04 LTS"); + + tbl.cell(7, 0).unwrap().v_merge_continue(); + tbl.cell(7, 1).unwrap().set_text("Monitoring"); + tbl.cell(7, 2).unwrap().set_text("Prometheus + Grafana"); + } + + doc.add_paragraph(""); + + // ========================================================================= + // 4. Nested table (table inside a cell) + // ========================================================================= + doc.add_paragraph("4. Nested Table").style("Heading2"); + + { + let mut tbl = doc.add_table(2, 2); + tbl = tbl.borders(BorderStyle::Single, 6, "2E75B6"); + tbl = tbl.width_pct(100.0); + tbl = tbl.cell_margins( + Length::twips(72), + Length::twips(108), + Length::twips(72), + Length::twips(108), + ); + + tbl.cell(0, 0).unwrap().set_text("Project Alpha"); + tbl.cell(0, 0).unwrap().shading("2E75B6"); + tbl.cell(0, 1).unwrap().set_text("Project Beta"); + tbl.cell(0, 1).unwrap().shading("2E75B6"); + + // Nested table in cell (1,0) + { + let mut cell = tbl.cell(1, 0).unwrap(); + cell.set_text("Milestones:"); + let mut inner = cell.add_table(3, 2); + inner = inner.borders(BorderStyle::Single, 2, "70AD47"); + inner.cell(0, 0).unwrap().set_text("Phase"); + inner.cell(0, 0).unwrap().shading("E2EFDA"); + inner.cell(0, 1).unwrap().set_text("Status"); + inner.cell(0, 1).unwrap().shading("E2EFDA"); + inner.cell(1, 0).unwrap().set_text("Design"); + inner.cell(1, 1).unwrap().set_text("Complete"); + inner.cell(2, 0).unwrap().set_text("Build"); + inner.cell(2, 1).unwrap().set_text("In Progress"); + } + + // Nested table in cell (1,1) + { + let mut cell = tbl.cell(1, 1).unwrap(); + cell.set_text("Budget:"); + let mut inner = cell.add_table(3, 2); + inner = inner.borders(BorderStyle::Single, 2, "ED7D31"); + inner.cell(0, 0).unwrap().set_text("Category"); + inner.cell(0, 0).unwrap().shading("FCE4D6"); + inner.cell(0, 1).unwrap().set_text("Amount"); + inner.cell(0, 1).unwrap().shading("FCE4D6"); + inner.cell(1, 0).unwrap().set_text("Development"); + inner.cell(1, 1).unwrap().set_text("$120,000"); + inner.cell(2, 0).unwrap().set_text("Testing"); + inner.cell(2, 1).unwrap().set_text("$35,000"); + } + } + + doc.add_paragraph(""); + + // ========================================================================= + // 5. Form-style table with labels + // ========================================================================= + doc.add_paragraph("5. Form-Style Table").style("Heading2"); + + { + let mut tbl = doc.add_table(6, 4); + tbl = tbl.borders(BorderStyle::Single, 4, "808080"); + tbl = tbl.width_pct(100.0); + + // Row 0: Full-width title + tbl.cell(0, 0) + .unwrap() + .set_text("Customer Registration Form"); + tbl.cell(0, 0).unwrap().grid_span(4); + tbl.cell(0, 0).unwrap().shading("404040"); + + // Row 1: Name fields + tbl.cell(1, 0).unwrap().set_text("First Name"); + tbl.cell(1, 0).unwrap().shading("E8E8E8"); + tbl.cell(1, 1).unwrap().set_text("John"); + tbl.cell(1, 2).unwrap().set_text("Last Name"); + tbl.cell(1, 2).unwrap().shading("E8E8E8"); + tbl.cell(1, 3).unwrap().set_text("Smith"); + + // Row 2: Contact + tbl.cell(2, 0).unwrap().set_text("Email"); + tbl.cell(2, 0).unwrap().shading("E8E8E8"); + tbl.cell(2, 1).unwrap().set_text("john.smith@example.com"); + tbl.cell(2, 1).unwrap().grid_span(3); + + // Row 3: Phone + tbl.cell(3, 0).unwrap().set_text("Phone"); + tbl.cell(3, 0).unwrap().shading("E8E8E8"); + tbl.cell(3, 1).unwrap().set_text("+1 (555) 123-4567"); + tbl.cell(3, 2).unwrap().set_text("Company"); + tbl.cell(3, 2).unwrap().shading("E8E8E8"); + tbl.cell(3, 3).unwrap().set_text("Acme Corp"); + + // Row 4: Address (spanning) + tbl.cell(4, 0).unwrap().set_text("Address"); + tbl.cell(4, 0).unwrap().shading("E8E8E8"); + tbl.cell(4, 1) + .unwrap() + .set_text("123 Business Ave, Suite 400, Portland, OR 97201"); + tbl.cell(4, 1).unwrap().grid_span(3); + + // Row 5: Notes + tbl.cell(5, 0).unwrap().set_text("Notes"); + tbl.cell(5, 0).unwrap().shading("E8E8E8"); + tbl.cell(5, 0) + .unwrap() + .vertical_alignment(VerticalAlignment::Top); + { + let mut cell = tbl.cell(5, 1).unwrap().grid_span(3); + cell.set_text("Premium customer since 2020. Preferred contact method: email."); + cell.add_paragraph("Annual review scheduled for March 2026."); + } + } + + doc.add_paragraph(""); + + // ========================================================================= + // 6. Comparison table with border styles + // ========================================================================= + doc.add_paragraph("6. Comparison Table with Custom Borders") + .style("Heading2"); + + { + let mut tbl = doc.add_table(5, 3); + tbl = tbl.borders(BorderStyle::Double, 4, "2E75B6"); + tbl = tbl.width_pct(100.0); + + // Header + tbl.cell(0, 0).unwrap().set_text("Feature"); + tbl.cell(0, 0).unwrap().shading("2E75B6"); + tbl.cell(0, 1).unwrap().set_text("Basic Plan"); + tbl.cell(0, 1).unwrap().shading("2E75B6"); + tbl.cell(0, 2).unwrap().set_text("Enterprise Plan"); + tbl.cell(0, 2).unwrap().shading("2E75B6"); + + tbl.cell(1, 0).unwrap().set_text("Users"); + tbl.cell(1, 1).unwrap().set_text("Up to 10"); + tbl.cell(1, 2).unwrap().set_text("Unlimited"); + tbl.cell(1, 2).unwrap().shading("E2EFDA"); + + tbl.cell(2, 0).unwrap().set_text("Storage"); + tbl.cell(2, 1).unwrap().set_text("50 GB"); + tbl.cell(2, 2).unwrap().set_text("5 TB"); + tbl.cell(2, 2).unwrap().shading("E2EFDA"); + + tbl.cell(3, 0).unwrap().set_text("Support"); + tbl.cell(3, 1).unwrap().set_text("Email only"); + tbl.cell(3, 2).unwrap().set_text("24/7 Phone + Email"); + tbl.cell(3, 2).unwrap().shading("E2EFDA"); + + tbl.cell(4, 0).unwrap().set_text("Price"); + tbl.cell(4, 0).unwrap().shading("F2F2F2"); + tbl.cell(4, 1).unwrap().set_text("$29/month"); + tbl.cell(4, 1).unwrap().shading("F2F2F2"); + tbl.cell(4, 2).unwrap().set_text("$199/month"); + tbl.cell(4, 2).unwrap().shading("C6EFCE"); + } + + doc.add_paragraph(""); + + // ========================================================================= + // 7. Wide table with fixed layout and row height + // ========================================================================= + doc.add_paragraph("7. Fixed Layout Table with Row Height Control") + .style("Heading2"); + + { + let mut tbl = doc.add_table(4, 5); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + tbl = tbl.width(Length::inches(7.0)); + tbl = tbl.layout_fixed(); + + // Set column widths + for col in 0..5 { + tbl.cell(0, col).unwrap().width(Length::inches(1.4)); + } + + // Header with exact height + tbl.row(0).unwrap().height_exact(Length::twips(480)); + tbl.row(0).unwrap().header(); + tbl.row(0).unwrap().cant_split(); + + let headers = ["Mon", "Tue", "Wed", "Thu", "Fri"]; + for (col, h) in headers.iter().enumerate() { + tbl.cell(0, col).unwrap().set_text(h); + tbl.cell(0, col).unwrap().shading("404040"); + tbl.cell(0, col) + .unwrap() + .vertical_alignment(VerticalAlignment::Center); + } + + // Schedule rows with minimum height + tbl.row(1).unwrap().height(Length::twips(600)); + tbl.cell(1, 0).unwrap().set_text("9:00 Standup"); + tbl.cell(1, 1).unwrap().set_text("9:00 Standup"); + tbl.cell(1, 2).unwrap().set_text("9:00 Standup"); + tbl.cell(1, 3).unwrap().set_text("9:00 Standup"); + tbl.cell(1, 4).unwrap().set_text("9:00 Standup"); + + tbl.row(2).unwrap().height(Length::twips(600)); + tbl.cell(2, 0).unwrap().set_text("10:00 Dev"); + tbl.cell(2, 0).unwrap().shading("D6E4F0"); + tbl.cell(2, 1).unwrap().set_text("10:00 Design Review"); + tbl.cell(2, 1).unwrap().shading("FCE4D6"); + tbl.cell(2, 2).unwrap().set_text("10:00 Dev"); + tbl.cell(2, 2).unwrap().shading("D6E4F0"); + tbl.cell(2, 3).unwrap().set_text("10:00 Sprint Planning"); + tbl.cell(2, 3).unwrap().shading("E2EFDA"); + tbl.cell(2, 4).unwrap().set_text("10:00 Dev"); + tbl.cell(2, 4).unwrap().shading("D6E4F0"); + + tbl.row(3).unwrap().height(Length::twips(600)); + tbl.cell(3, 0).unwrap().set_text("14:00 Code Review"); + tbl.cell(3, 1).unwrap().set_text("14:00 Dev"); + tbl.cell(3, 1).unwrap().shading("D6E4F0"); + tbl.cell(3, 2).unwrap().set_text("14:00 Demo"); + tbl.cell(3, 2).unwrap().shading("FCE4D6"); + tbl.cell(3, 3).unwrap().set_text("14:00 Dev"); + tbl.cell(3, 3).unwrap().shading("D6E4F0"); + tbl.cell(3, 4).unwrap().set_text("14:00 Retro"); + tbl.cell(3, 4).unwrap().shading("E2EFDA"); + } + + doc.set_title("Styled Tables Showcase"); + doc.set_author("rdocx"); + + doc.save(path).unwrap(); +} diff --git a/vendor/rdocx/examples/template_replace.rs b/vendor/rdocx/examples/template_replace.rs new file mode 100644 index 00000000..4cd95747 --- /dev/null +++ b/vendor/rdocx/examples/template_replace.rs @@ -0,0 +1,245 @@ +//! Template-based placeholder replacement workflow. +//! +//! This example demonstrates the typical template workflow: +//! 1. Create a template document with placeholders +//! 2. Open the template +//! 3. Replace placeholders in body text, tables, and across formatting runs +//! 4. Insert additional content at specific positions +//! 5. Save the final document +//! +//! Run with: cargo run --example template_replace + +use std::collections::HashMap; +use std::path::Path; + +use rdocx::{Alignment, BorderStyle, Document, Length}; + +fn main() { + let samples_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .parent() + .unwrap() + .join("samples"); + std::fs::create_dir_all(&samples_dir).unwrap(); + + // Step 1: Create a template document with placeholders + let template_path = samples_dir.join("_template.docx"); + create_template(&template_path); + println!(" Created template: _template.docx"); + + // Step 2: Open the template and replace placeholders + let output_path = samples_dir.join("from_template.docx"); + fill_template(&template_path, &output_path); + println!(" Created filled document: from_template.docx"); + + // Clean up the intermediate template + std::fs::remove_file(&template_path).ok(); + println!("\nDone!"); +} + +/// Create a template document with placeholders in body text, tables, and +/// cross-run formatting. +fn create_template(path: &Path) { + let mut doc = Document::new(); + doc.set_page_size(Length::inches(8.5), Length::inches(11.0)); + doc.set_margins( + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + ); + + doc.set_header("{{company_name}} — Confidential"); + doc.set_footer("Prepared by {{author_name}} on {{date}}"); + + // ── Title ── + doc.add_paragraph("{{company_name}}") + .style("Heading1") + .alignment(Alignment::Center); + + doc.add_paragraph("Project Proposal") + .alignment(Alignment::Center); + + doc.add_paragraph(""); + + // ── Summary section ── + doc.add_paragraph("Executive Summary").style("Heading2"); + + doc.add_paragraph( + "This proposal outlines the {{project_name}} project for {{company_name}}. \ + The primary contact is {{contact_name}} ({{contact_email}}). \ + The proposed start date is {{start_date}} with an estimated duration of {{duration}}.", + ); + + doc.add_paragraph(""); + + // ── Cross-run placeholder (bold label + normal value) ── + doc.add_paragraph("Key Details").style("Heading2"); + + { + let mut p = doc.add_paragraph(""); + p.add_run("Project: ").bold(true); + p.add_run("{{project_name}}"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Budget: ").bold(true); + p.add_run("{{budget}}"); + } + { + let mut p = doc.add_paragraph(""); + p.add_run("Status: ").bold(true); + p.add_run("{{status}}"); + } + + doc.add_paragraph(""); + + // ── Table with placeholders ── + doc.add_paragraph("Team Members").style("Heading2"); + + { + let mut tbl = doc.add_table(4, 3); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + + // Header row + for col in 0..3 { + tbl.cell(0, col).unwrap().shading("2E75B6"); + } + tbl.cell(0, 0).unwrap().set_text("Name"); + tbl.cell(0, 1).unwrap().set_text("Role"); + tbl.cell(0, 2).unwrap().set_text("Email"); + + tbl.cell(1, 0).unwrap().set_text("{{member1_name}}"); + tbl.cell(1, 1).unwrap().set_text("{{member1_role}}"); + tbl.cell(1, 2).unwrap().set_text("{{member1_email}}"); + + tbl.cell(2, 0).unwrap().set_text("{{member2_name}}"); + tbl.cell(2, 1).unwrap().set_text("{{member2_role}}"); + tbl.cell(2, 2).unwrap().set_text("{{member2_email}}"); + + tbl.cell(3, 0).unwrap().set_text("{{member3_name}}"); + tbl.cell(3, 1).unwrap().set_text("{{member3_role}}"); + tbl.cell(3, 2).unwrap().set_text("{{member3_email}}"); + } + + doc.add_paragraph(""); + + // ── Deliverables section ── + doc.add_paragraph("Deliverables").style("Heading2"); + + doc.add_paragraph("INSERTION_POINT"); + + doc.add_paragraph(""); + + // ── Signature block ── + doc.add_paragraph("Acceptance").style("Heading2"); + + doc.add_paragraph( + "By signing below, {{company_name}} agrees to the terms outlined in this proposal.", + ); + + { + let mut tbl = doc.add_table(2, 2); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + tbl.cell(0, 0) + .unwrap() + .set_text("Customer: ___________________"); + tbl.cell(0, 1) + .unwrap() + .set_text("Provider: ___________________"); + tbl.cell(1, 0).unwrap().set_text("Date: {{date}}"); + tbl.cell(1, 1).unwrap().set_text("Date: {{date}}"); + } + + doc.set_title("{{company_name}} — Project Proposal Template"); + doc.set_author("Template Generator"); + + doc.save(path).unwrap(); +} + +/// Open the template, replace all placeholders, insert content, and save. +fn fill_template(template_path: &Path, output_path: &Path) { + let mut doc = Document::open(template_path).unwrap(); + + // ── Batch replacement ── + let mut replacements = HashMap::new(); + replacements.insert("{{company_name}}", "Riverside Medical Center"); + replacements.insert("{{project_name}}", "Network Security Upgrade"); + replacements.insert("{{contact_name}}", "Dr. Sarah Chen"); + replacements.insert("{{contact_email}}", "s.chen@riverside.org"); + replacements.insert("{{start_date}}", "March 1, 2026"); + replacements.insert("{{duration}}", "12 weeks"); + replacements.insert("{{budget}}", "$185,000"); + replacements.insert("{{status}}", "Pending Approval"); + replacements.insert("{{author_name}}", "James Wilson"); + replacements.insert("{{date}}", "February 22, 2026"); + + // Team members + replacements.insert("{{member1_name}}", "James Wilson"); + replacements.insert("{{member1_role}}", "Project Lead"); + replacements.insert("{{member1_email}}", "j.wilson@provider.com"); + replacements.insert("{{member2_name}}", "Maria Garcia"); + replacements.insert("{{member2_role}}", "Security Architect"); + replacements.insert("{{member2_email}}", "m.garcia@provider.com"); + replacements.insert("{{member3_name}}", "David Park"); + replacements.insert("{{member3_role}}", "Network Engineer"); + replacements.insert("{{member3_email}}", "d.park@provider.com"); + + let count = doc.replace_all(&replacements); + println!(" Replaced {} placeholders", count); + + // ── Insert deliverables at the insertion point ── + if let Some(idx) = doc.find_content_index("INSERTION_POINT") { + // Remove the placeholder paragraph + doc.remove_content(idx); + + // Insert deliverables list + doc.insert_paragraph(idx, "The following deliverables are included:"); + + // Insert a deliverables table + let mut tbl = doc.insert_table(idx + 1, 5, 3); + tbl = tbl.borders(BorderStyle::Single, 4, "000000"); + + for col in 0..3 { + tbl.cell(0, col).unwrap().shading("E2EFDA"); + } + tbl.cell(0, 0).unwrap().set_text("Phase"); + tbl.cell(0, 1).unwrap().set_text("Description"); + tbl.cell(0, 2).unwrap().set_text("Timeline"); + + tbl.cell(1, 0).unwrap().set_text("1. Discovery"); + tbl.cell(1, 1) + .unwrap() + .set_text("Network assessment and asset inventory"); + tbl.cell(1, 2).unwrap().set_text("Weeks 1-3"); + + tbl.cell(2, 0).unwrap().set_text("2. Design"); + tbl.cell(2, 1) + .unwrap() + .set_text("Security architecture and policy design"); + tbl.cell(2, 2).unwrap().set_text("Weeks 4-6"); + + tbl.cell(3, 0).unwrap().set_text("3. Implementation"); + tbl.cell(3, 1) + .unwrap() + .set_text("Deploy monitoring and access controls"); + tbl.cell(3, 2).unwrap().set_text("Weeks 7-10"); + + tbl.cell(4, 0).unwrap().set_text("4. Validation"); + tbl.cell(4, 1) + .unwrap() + .set_text("Testing, training, and handover"); + tbl.cell(4, 2).unwrap().set_text("Weeks 11-12"); + + println!(" Inserted deliverables table at position {}", idx); + } + + // ── Update metadata ── + doc.set_title("Riverside Medical Center — Network Security Upgrade Proposal"); + doc.set_author("James Wilson"); + doc.set_subject("Project Proposal"); + doc.set_keywords("security, network, medical, proposal"); + + doc.save(output_path).unwrap(); +} diff --git a/vendor/rdocx/src/document.rs b/vendor/rdocx/src/document.rs new file mode 100644 index 00000000..c6110ac0 --- /dev/null +++ b/vendor/rdocx/src/document.rs @@ -0,0 +1,3208 @@ +//! The main Document type — entry point for the rdocx API. + +use std::path::Path; + +use rdocx_opc::OpcPackage; +use rdocx_opc::relationship::rel_types; +use rdocx_oxml::document::{BodyContent, CT_Columns, CT_Document, CT_SectPr}; +use rdocx_oxml::drawing::{CT_Anchor, CT_Drawing, CT_Inline}; +use rdocx_oxml::header_footer::{CT_HdrFtr, HdrFtrRef, HdrFtrType}; +use rdocx_oxml::numbering::CT_Numbering; +use rdocx_oxml::properties::{CT_PPr, CT_RPr}; +use rdocx_oxml::shared::{ST_PageOrientation, ST_SectionType}; +use rdocx_oxml::styles::CT_Styles; +use rdocx_oxml::table::CT_Tbl; +use rdocx_oxml::text::{CT_P, CT_R, RunContent}; + +use rdocx_oxml::core_properties::CoreProperties; + +use crate::Length; +use crate::error::{Error, Result}; +use crate::paragraph::{Paragraph, ParagraphRef}; +use crate::style::{self, Style, StyleBuilder}; +use crate::table::{Table, TableRef}; + +/// A Word document (.docx file). +/// +/// This is the main entry point for reading, creating, and modifying +/// DOCX documents. +pub struct Document { + package: OpcPackage, + document: CT_Document, + styles: CT_Styles, + numbering: Option, + core_properties: Option, + /// Part name for the main document + doc_part_name: String, + /// Cached count of image media parts (avoids rescanning parts on each embed). + image_counter: usize, +} + +impl Document { + /// Create a new, empty document with default page setup and styles. + pub fn new() -> Self { + let mut package = OpcPackage::new_docx(); + let document = CT_Document::new(); + let styles = CT_Styles::new_default(); + + // Set up styles relationship + package + .get_or_create_part_rels("/word/document.xml") + .add(rel_types::STYLES, "styles.xml"); + + Document { + package, + document, + styles, + numbering: None, + core_properties: None, + doc_part_name: "/word/document.xml".to_string(), + image_counter: 0, + } + } + + /// Open a document from a file path. + pub fn open>(path: P) -> Result { + let package = OpcPackage::open(path)?; + Self::from_package(package) + } + + /// Open a document from bytes. + pub fn from_bytes(bytes: &[u8]) -> Result { + let cursor = std::io::Cursor::new(bytes); + let package = OpcPackage::from_reader(cursor)?; + Self::from_package(package) + } + + fn from_package(package: OpcPackage) -> Result { + let doc_part_name = package.main_document_part().ok_or(Error::NoDocumentPart)?; + + let doc_xml = package + .get_part(&doc_part_name) + .ok_or(Error::NoDocumentPart)?; + let document = CT_Document::from_xml(doc_xml)?; + + // Try to load styles + let styles = if let Some(rels) = package.get_part_rels(&doc_part_name) { + if let Some(styles_rel) = rels.get_by_type(rel_types::STYLES) { + let styles_part = + OpcPackage::resolve_rel_target(&doc_part_name, &styles_rel.target); + if let Some(styles_xml) = package.get_part(&styles_part) { + CT_Styles::from_xml(styles_xml)? + } else { + CT_Styles::new_default() + } + } else { + CT_Styles::new_default() + } + } else { + CT_Styles::new_default() + }; + + // Try to load numbering definitions + let numbering = if let Some(rels) = package.get_part_rels(&doc_part_name) { + if let Some(num_rel) = rels.get_by_type(rel_types::NUMBERING) { + let num_part = OpcPackage::resolve_rel_target(&doc_part_name, &num_rel.target); + if let Some(num_xml) = package.get_part(&num_part) { + Some(CT_Numbering::from_xml(num_xml)?) + } else { + None + } + } else { + None + } + } else { + None + }; + + // Try to load core properties from docProps/core.xml + let core_properties = package + .get_part("/docProps/core.xml") + .and_then(|xml| CoreProperties::from_xml(xml).ok()); + + let image_counter = package + .parts + .keys() + .filter(|k| k.starts_with("/word/media/image")) + .count(); + + Ok(Document { + package, + document, + styles, + numbering, + core_properties, + doc_part_name, + image_counter, + }) + } + + /// §flowstate A11.2 (vendor patch): construct a READ-ORIENTED `Document` + /// from an ALREADY-PARSED typed tree, loading styles from an ALREADY-OPEN + /// package. Skips the redundant `document.xml` re-parse, the + /// `numbering.xml`/`core.xml` parses (never consulted by the read APIs — + /// style resolution walks `self.styles` only), and the package + /// re-inflate. The stored package is an EMPTY placeholder: the read APIs + /// (`paragraphs`/`tables`/`styles`/`resolve_*`) never touch it, but + /// `save`/`to_bytes`/image-embedding on such a `Document` would write that + /// placeholder — callers on this path must not invoke them. + pub fn from_parsed_parts(source: &OpcPackage, document: CT_Document) -> Result { + let doc_part_name = source.main_document_part().ok_or(Error::NoDocumentPart)?; + let styles = if let Some(rels) = source.get_part_rels(&doc_part_name) { + if let Some(styles_rel) = rels.get_by_type(rel_types::STYLES) { + let styles_part = + OpcPackage::resolve_rel_target(&doc_part_name, &styles_rel.target); + if let Some(styles_xml) = source.get_part(&styles_part) { + CT_Styles::from_xml(styles_xml)? + } else { + CT_Styles::new_default() + } + } else { + CT_Styles::new_default() + } + } else { + CT_Styles::new_default() + }; + + Ok(Document { + package: OpcPackage::new_docx(), + document, + styles, + numbering: None, + core_properties: None, + doc_part_name, + image_counter: 0, + }) + } + + /// §flowstate A11.2 (vendor patch): hand the typed tree back out so the + /// single parse can outlive this `Document` (the caller parsed it). + pub fn into_typed_document(self) -> CT_Document { + self.document + } + + /// Save the document to a file path. + pub fn save>(&mut self, path: P) -> Result<()> { + self.flush_to_package()?; + self.package.save(path)?; + Ok(()) + } + + /// Save the document to a byte vector. + pub fn to_bytes(&mut self) -> Result> { + self.flush_to_package()?; + let mut buf = std::io::Cursor::new(Vec::new()); + self.package.write_to(&mut buf)?; + Ok(buf.into_inner()) + } + + /// Write the in-memory document/styles back into the OPC package parts. + fn flush_to_package(&mut self) -> Result<()> { + // Serialize document.xml + let doc_xml = self.document.to_xml()?; + self.package.set_part(&self.doc_part_name, doc_xml); + + // Serialize styles.xml + let styles_xml = self.styles.to_xml()?; + self.package.set_part("/word/styles.xml", styles_xml); + + // Serialize numbering.xml if we have numbering definitions + if let Some(ref numbering) = self.numbering { + let numbering_xml = numbering.to_xml()?; + self.package.set_part("/word/numbering.xml", numbering_xml); + } + + // Serialize docProps/core.xml if we have metadata + if let Some(ref props) = self.core_properties { + let core_xml = props.to_xml()?; + self.package.set_part("/docProps/core.xml", core_xml); + self.package.content_types.add_override( + "/docProps/core.xml", + "application/vnd.openxmlformats-package.core-properties+xml", + ); + } + + Ok(()) + } + + // ---- Paragraph access ---- + + /// Get immutable references to all paragraphs. + pub fn paragraphs(&self) -> Vec> { + self.document + .body + .paragraphs() + .map(|p| ParagraphRef { inner: p }) + .collect() + } + + /// Add a paragraph with the given text and return a mutable reference. + pub fn add_paragraph(&mut self, text: &str) -> Paragraph<'_> { + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + self.document.body.content.push(BodyContent::Paragraph(p)); + match self.document.body.content.last_mut().unwrap() { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + /// Get the number of paragraphs. + pub fn paragraph_count(&self) -> usize { + self.document.body.paragraphs().count() + } + + /// Get a mutable reference to a paragraph by index (among paragraphs only). + pub fn paragraph_mut(&mut self, index: usize) -> Option> { + self.document + .body + .paragraphs_mut() + .nth(index) + .map(|p| Paragraph { inner: p }) + } + + // ---- Table access ---- + + /// Get immutable references to all tables. + pub fn tables(&self) -> Vec> { + self.document + .body + .tables() + .map(|t| TableRef { inner: t }) + .collect() + } + + /// Add a table with the specified number of rows and columns. + /// Returns a mutable reference for further configuration. + pub fn add_table(&mut self, rows: usize, cols: usize) -> Table<'_> { + use rdocx_oxml::table::{CT_Row, CT_TblGrid, CT_TblGridCol, CT_TblPr, CT_TblWidth, CT_Tc}; + use rdocx_oxml::units::Twips; + + // Default column width: divide 9360tw (6.5" printable at 1" margins) evenly + let col_width = Twips(9360 / cols as i32); + + let grid = CT_TblGrid { + columns: (0..cols) + .map(|_| CT_TblGridCol { width: col_width }) + .collect(), + }; + + let mut tbl = CT_Tbl::new(); + tbl.properties = Some(CT_TblPr { + width: Some(CT_TblWidth::dxa(col_width.0 * cols as i32)), + ..Default::default() + }); + tbl.grid = Some(grid); + + for _ in 0..rows { + let mut row = CT_Row::new(); + for _ in 0..cols { + row.cells.push(CT_Tc::new()); + } + tbl.rows.push(row); + } + + self.document.body.content.push(BodyContent::Table(tbl)); + match self.document.body.content.last_mut().unwrap() { + BodyContent::Table(t) => Table { inner: t }, + _ => unreachable!(), + } + } + + /// Get the number of tables. + pub fn table_count(&self) -> usize { + self.document.body.tables().count() + } + + // ---- Content insertion ---- + + /// Get the number of body content elements (paragraphs + tables). + pub fn content_count(&self) -> usize { + self.document.body.content_count() + } + + /// Insert a paragraph at the given body index. + /// + /// Returns a mutable `Paragraph` for further configuration. + /// Panics if `index > content_count()`. + pub fn insert_paragraph(&mut self, index: usize, text: &str) -> Paragraph<'_> { + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + self.document.body.insert_paragraph(index, p); + match &mut self.document.body.content[index] { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + /// Insert a table at the given body index. + /// + /// Returns a mutable `Table` for further configuration. + /// Panics if `index > content_count()`. + pub fn insert_table(&mut self, index: usize, rows: usize, cols: usize) -> Table<'_> { + use rdocx_oxml::table::{CT_Row, CT_TblGrid, CT_TblGridCol, CT_TblPr, CT_TblWidth, CT_Tc}; + use rdocx_oxml::units::Twips; + + let col_width = Twips(9360 / cols as i32); + let grid = CT_TblGrid { + columns: (0..cols) + .map(|_| CT_TblGridCol { width: col_width }) + .collect(), + }; + + let mut tbl = CT_Tbl::new(); + tbl.properties = Some(CT_TblPr { + width: Some(CT_TblWidth::dxa(col_width.0 * cols as i32)), + ..Default::default() + }); + tbl.grid = Some(grid); + + for _ in 0..rows { + let mut row = CT_Row::new(); + for _ in 0..cols { + row.cells.push(CT_Tc::new()); + } + tbl.rows.push(row); + } + + self.document.body.insert_table(index, tbl); + match &mut self.document.body.content[index] { + BodyContent::Table(t) => Table { inner: t }, + _ => unreachable!(), + } + } + + /// Find the body content index of the first paragraph containing the given text. + pub fn find_content_index(&self, text: &str) -> Option { + self.document.body.find_paragraph_index(text) + } + + /// Remove the content at the given body index. + /// + /// Returns `true` if an element was removed, `false` if the index was out of bounds. + pub fn remove_content(&mut self, index: usize) -> bool { + self.document.body.remove(index).is_some() + } + + // ---- Image support ---- + + /// Add an inline image to the document. + /// + /// Embeds the image data (PNG, JPEG, etc.) into the package and adds a + /// paragraph containing the image. Returns a mutable reference to the + /// paragraph for further configuration. + /// + /// `width` and `height` specify the display size. + pub fn add_picture( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + ) -> Paragraph<'_> { + let rel_id = self.embed_image(image_data, image_filename); + + let inline = CT_Inline::new(&rel_id, width.to_emu(), height.to_emu()); + + let drawing = CT_Drawing::inline(inline); + let run = CT_R { + properties: None, + content: vec![RunContent::Drawing(drawing)], + extra_xml: Vec::new(), + }; + + let mut p = CT_P::new(); + p.runs.push(run); + self.document.body.content.push(BodyContent::Paragraph(p)); + match self.document.body.content.last_mut().unwrap() { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + /// Add a full-page background image behind text. + /// + /// The image is placed at position (0,0) relative to the page with + /// dimensions matching the page size from section properties. + /// It is inserted at the beginning of the document body so it renders + /// behind all other content. + pub fn add_background_image( + &mut self, + image_data: &[u8], + image_filename: &str, + ) -> Paragraph<'_> { + let rel_id = self.embed_image(image_data, image_filename); + + // Get page dimensions from section properties (default US Letter) + let sect = self + .document + .body + .sect_pr + .as_ref() + .cloned() + .unwrap_or_else(CT_SectPr::default_letter); + let page_width_emu = sect + .page_width + .unwrap_or(rdocx_oxml::units::Twips(12240)) + .to_emu() + .0; + let page_height_emu = sect + .page_height + .unwrap_or(rdocx_oxml::units::Twips(15840)) + .to_emu() + .0; + + let anchor = CT_Anchor::background(&rel_id, page_width_emu, page_height_emu); + let drawing = CT_Drawing::anchor(anchor); + let run = CT_R { + properties: None, + content: vec![RunContent::Drawing(drawing)], + extra_xml: Vec::new(), + }; + + let mut p = CT_P::new(); + p.runs.push(run); + self.document.body.insert_paragraph(0, p); + match &mut self.document.body.content[0] { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + /// Add an anchored (floating) image to the document. + /// + /// If `behind_text` is true, the image renders behind text content. + /// The image is inserted at the beginning of the document body. + pub fn add_anchored_image( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + behind_text: bool, + ) -> Paragraph<'_> { + let rel_id = self.embed_image(image_data, image_filename); + + let mut anchor = CT_Anchor::background(&rel_id, width.to_emu(), height.to_emu()); + anchor.behind_doc = behind_text; + + let drawing = CT_Drawing::anchor(anchor); + let run = CT_R { + properties: None, + content: vec![RunContent::Drawing(drawing)], + extra_xml: Vec::new(), + }; + + let mut p = CT_P::new(); + p.runs.push(run); + self.document.body.insert_paragraph(0, p); + match &mut self.document.body.content[0] { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + /// Return the next unique image number and bump the counter. + fn next_image_number(&mut self) -> usize { + self.image_counter += 1; + self.image_counter + } + + /// Embed an image into the OPC package and return the relationship ID. + fn embed_image(&mut self, image_data: &[u8], filename: &str) -> String { + use rdocx_opc::relationship::rel_types; + + // Determine content type from extension + let ext = filename.rsplit('.').next().unwrap_or("png").to_lowercase(); + let content_type = match ext.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "bmp" => "image/bmp", + "tiff" | "tif" => "image/tiff", + "svg" => "image/svg+xml", + _ => "image/png", + }; + + // Generate a unique part name using cached counter + let image_num = self.next_image_number(); + let part_name = format!("/word/media/image{image_num}.{ext}"); + + // Store the image data + self.package.set_part(&part_name, image_data.to_vec()); + + // Add content type override + self.package.content_types.add_default(&ext, content_type); + + // Add relationship + let rel_target = format!("media/image{image_num}.{ext}"); + let rels = self.package.get_or_create_part_rels(&self.doc_part_name); + rels.add(rel_types::IMAGE, &rel_target) + } + + // ---- Header/Footer ---- + + /// Set the default header text. + /// + /// Creates a header part with the given text and references it from + /// the section properties. + pub fn set_header(&mut self, text: &str) { + self.set_header_footer_part(text, true, HdrFtrType::Default); + } + + /// Set the default footer text. + pub fn set_footer(&mut self, text: &str) { + self.set_header_footer_part(text, false, HdrFtrType::Default); + } + + /// Set the first-page header text. + pub fn set_first_page_header(&mut self, text: &str) { + self.set_different_first_page(true); + self.set_header_footer_part(text, true, HdrFtrType::First); + } + + /// Set the first-page footer text. + pub fn set_first_page_footer(&mut self, text: &str) { + self.set_different_first_page(true); + self.set_header_footer_part(text, false, HdrFtrType::First); + } + + fn set_header_footer_part(&mut self, text: &str, is_header: bool, hdr_type: HdrFtrType) { + use rdocx_opc::relationship::rel_types; + + let mut hdr_ftr = CT_HdrFtr::new(); + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + hdr_ftr.paragraphs.push(p); + + // Determine part name based on type + let type_suffix = match hdr_type { + HdrFtrType::Default => "", + HdrFtrType::First => "First", + HdrFtrType::Even => "Even", + }; + let (part_name, rel_type, content_type) = if is_header { + ( + format!("/word/header{type_suffix}1.xml"), + rel_types::HEADER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml", + ) + } else { + ( + format!("/word/footer{type_suffix}1.xml"), + rel_types::FOOTER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + ) + }; + + // Serialize the header/footer + let xml = if is_header { + hdr_ftr + .to_xml_header() + .expect("header serialization failed") + } else { + hdr_ftr + .to_xml_footer() + .expect("footer serialization failed") + }; + + self.package.set_part(&part_name, xml); + self.package + .content_types + .add_override(&part_name, content_type); + + // Add relationship + let rel_target = part_name.trim_start_matches("/word/"); + let rels = self.package.get_or_create_part_rels(&self.doc_part_name); + let rel_id = rels.add(rel_type, rel_target); + + // Add reference in section properties + let sect = self.section_properties_mut(); + let refs = if is_header { + &mut sect.header_refs + } else { + &mut sect.footer_refs + }; + + // Remove existing ref of same type + refs.retain(|r| r.hdr_ftr_type != hdr_type); + refs.push(HdrFtrRef { + hdr_ftr_type: hdr_type, + rel_id, + }); + } + + /// Get the default header text, if set. + pub fn header_text(&self) -> Option { + self.get_header_footer_text(true, HdrFtrType::Default) + } + + /// Get the default footer text, if set. + pub fn footer_text(&self) -> Option { + self.get_header_footer_text(false, HdrFtrType::Default) + } + + /// Set the default header to an inline image. + /// + /// Creates a header part with an image paragraph. The image is embedded + /// in the header part's relationships. + pub fn set_header_image( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + ) { + self.set_header_footer_image_part( + image_data, + image_filename, + width, + height, + true, + HdrFtrType::Default, + ); + } + + /// Set the default footer to an inline image. + pub fn set_footer_image( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + ) { + self.set_header_footer_image_part( + image_data, + image_filename, + width, + height, + false, + HdrFtrType::Default, + ); + } + + /// Set a header from raw XML bytes with associated images. + /// + /// This is useful for copying complex headers from template documents + /// that contain grouped shapes, VML, or other elements not easily + /// recreated through the high-level API. + /// + /// Each entry in `images` is `(rel_id, image_data, image_filename)`: + /// - `rel_id`: the relationship ID referenced in the header XML (e.g. "rId1") + /// - `image_data`: the raw image bytes + /// - `image_filename`: used to derive the part name and content type (e.g. "image5.png") + pub fn set_raw_header_with_images( + &mut self, + header_xml: Vec, + images: &[(&str, &[u8], &str)], + hdr_type: HdrFtrType, + ) { + self.set_raw_hdr_ftr_with_images(header_xml, images, true, hdr_type); + } + + /// Set a footer from raw XML bytes with associated images. + pub fn set_raw_footer_with_images( + &mut self, + footer_xml: Vec, + images: &[(&str, &[u8], &str)], + hdr_type: HdrFtrType, + ) { + self.set_raw_hdr_ftr_with_images(footer_xml, images, false, hdr_type); + } + + fn set_raw_hdr_ftr_with_images( + &mut self, + xml: Vec, + images: &[(&str, &[u8], &str)], + is_header: bool, + hdr_type: HdrFtrType, + ) { + use rdocx_opc::relationship::rel_types; + + let type_suffix = match hdr_type { + HdrFtrType::Default => "", + HdrFtrType::First => "First", + HdrFtrType::Even => "Even", + }; + let (part_name, rel_type, content_type) = if is_header { + ( + format!("/word/header{type_suffix}1.xml"), + rel_types::HEADER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml", + ) + } else { + ( + format!("/word/footer{type_suffix}1.xml"), + rel_types::FOOTER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + ) + }; + + // Store the raw header/footer XML + self.package.set_part(&part_name, xml); + self.package + .content_types + .add_override(&part_name, content_type); + + // Store each image and create relationships with the specified rel_ids + for &(rel_id, image_data, image_filename) in images { + let ext = image_filename + .rsplit('.') + .next() + .unwrap_or("png") + .to_lowercase(); + let img_content_type = match ext.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + _ => "image/png", + }; + + let image_num = self.next_image_number(); + let img_part_name = format!("/word/media/image{image_num}.{ext}"); + self.package.set_part(&img_part_name, image_data.to_vec()); + self.package + .content_types + .add_default(&ext, img_content_type); + + // Create relationship in the header/footer part's rels with the EXACT rel_id + let img_rel_target = format!("media/image{image_num}.{ext}"); + let hdr_rels = self.package.get_or_create_part_rels(&part_name); + hdr_rels.add_with_id(rel_id, rel_types::IMAGE, &img_rel_target); + } + + // Add relationship from document to header/footer + let rel_target = part_name.trim_start_matches("/word/"); + let rels = self.package.get_or_create_part_rels(&self.doc_part_name); + let rel_id = rels.add(rel_type, rel_target); + + // Add reference in section properties + let sect = self.section_properties_mut(); + let refs = if is_header { + &mut sect.header_refs + } else { + &mut sect.footer_refs + }; + + refs.retain(|r| r.hdr_ftr_type != hdr_type); + refs.push(HdrFtrRef { + hdr_ftr_type: hdr_type, + rel_id, + }); + } + + /// Set the default header to an inline image with a colored background. + /// + /// Creates a header part where the paragraph has shading fill set to + /// `bg_color` (hex string, e.g. "000000" for black) and contains the + /// inline image. + pub fn set_header_image_with_background( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + bg_color: &str, + ) { + self.set_header_footer_image_bg_part( + image_data, + image_filename, + width, + height, + Some(bg_color), + true, + HdrFtrType::Default, + ); + } + + /// Set the first-page header to an inline image. + pub fn set_first_page_header_image( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + ) { + self.set_different_first_page(true); + self.set_header_footer_image_part( + image_data, + image_filename, + width, + height, + true, + HdrFtrType::First, + ); + } + + fn set_header_footer_image_part( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + is_header: bool, + hdr_type: HdrFtrType, + ) { + use rdocx_opc::relationship::rel_types; + + // Determine part name based on type + let type_suffix = match hdr_type { + HdrFtrType::Default => "", + HdrFtrType::First => "First", + HdrFtrType::Even => "Even", + }; + let (part_name, rel_type, content_type) = if is_header { + ( + format!("/word/header{type_suffix}1.xml"), + rel_types::HEADER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml", + ) + } else { + ( + format!("/word/footer{type_suffix}1.xml"), + rel_types::FOOTER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + ) + }; + + // Embed the image in the package + let ext = image_filename + .rsplit('.') + .next() + .unwrap_or("png") + .to_lowercase(); + let img_content_type = match ext.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + _ => "image/png", + }; + + // Generate unique image name using cached counter + let image_num = self.next_image_number(); + let img_part_name = format!("/word/media/image{image_num}.{ext}"); + self.package.set_part(&img_part_name, image_data.to_vec()); + self.package + .content_types + .add_default(&ext, img_content_type); + + // Create image relationship in the HEADER/FOOTER part's rels + let img_rel_target = format!("media/image{image_num}.{ext}"); + let hdr_rels = self.package.get_or_create_part_rels(&part_name); + let img_rel_id = hdr_rels.add(rel_types::IMAGE, &img_rel_target); + + // Build header/footer with image paragraph + let inline = CT_Inline::new(&img_rel_id, width.to_emu(), height.to_emu()); + let drawing = CT_Drawing::inline(inline); + let run = CT_R { + properties: None, + content: vec![RunContent::Drawing(drawing)], + extra_xml: Vec::new(), + }; + + let mut hdr_ftr = CT_HdrFtr::new(); + let mut p = CT_P::new(); + p.runs.push(run); + hdr_ftr.paragraphs.push(p); + + // Serialize + let xml = if is_header { + hdr_ftr + .to_xml_header() + .expect("header serialization failed") + } else { + hdr_ftr + .to_xml_footer() + .expect("footer serialization failed") + }; + + self.package.set_part(&part_name, xml); + self.package + .content_types + .add_override(&part_name, content_type); + + // Add relationship from document to header/footer + let rel_target = part_name.trim_start_matches("/word/"); + let rels = self.package.get_or_create_part_rels(&self.doc_part_name); + let rel_id = rels.add(rel_type, rel_target); + + // Add reference in section properties + let sect = self.section_properties_mut(); + let refs = if is_header { + &mut sect.header_refs + } else { + &mut sect.footer_refs + }; + + refs.retain(|r| r.hdr_ftr_type != hdr_type); + refs.push(HdrFtrRef { + hdr_ftr_type: hdr_type, + rel_id, + }); + } + + fn set_header_footer_image_bg_part( + &mut self, + image_data: &[u8], + image_filename: &str, + width: Length, + height: Length, + bg_color: Option<&str>, + is_header: bool, + hdr_type: HdrFtrType, + ) { + use rdocx_opc::relationship::rel_types; + use rdocx_oxml::properties::CT_Shd; + + // Determine part name based on type + let type_suffix = match hdr_type { + HdrFtrType::Default => "", + HdrFtrType::First => "First", + HdrFtrType::Even => "Even", + }; + let (part_name, rel_type, content_type) = if is_header { + ( + format!("/word/header{type_suffix}1.xml"), + rel_types::HEADER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.header+xml", + ) + } else { + ( + format!("/word/footer{type_suffix}1.xml"), + rel_types::FOOTER, + "application/vnd.openxmlformats-officedocument.wordprocessingml.footer+xml", + ) + }; + + // Embed the image in the package + let ext = image_filename + .rsplit('.') + .next() + .unwrap_or("png") + .to_lowercase(); + let img_content_type = match ext.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + _ => "image/png", + }; + + let image_num = self.next_image_number(); + let img_part_name = format!("/word/media/image{image_num}.{ext}"); + self.package.set_part(&img_part_name, image_data.to_vec()); + self.package + .content_types + .add_default(&ext, img_content_type); + + // Create image relationship in the HEADER/FOOTER part's rels + let img_rel_target = format!("media/image{image_num}.{ext}"); + let hdr_rels = self.package.get_or_create_part_rels(&part_name); + let img_rel_id = hdr_rels.add(rel_types::IMAGE, &img_rel_target); + + // Build header/footer with image paragraph + let inline = CT_Inline::new(&img_rel_id, width.to_emu(), height.to_emu()); + let drawing = CT_Drawing::inline(inline); + let run = CT_R { + properties: None, + content: vec![RunContent::Drawing(drawing)], + extra_xml: Vec::new(), + }; + + let mut hdr_ftr = CT_HdrFtr::new(); + let mut p = CT_P::new(); + p.runs.push(run); + + // Add background shading if requested + if let Some(color) = bg_color { + let ppr = CT_PPr { + shading: Some(CT_Shd { + val: "clear".to_string(), + color: Some("auto".to_string()), + fill: Some(color.to_string()), + }), + ..Default::default() + }; + p.properties = Some(ppr); + } + + hdr_ftr.paragraphs.push(p); + + // Serialize + let xml = if is_header { + hdr_ftr + .to_xml_header() + .expect("header serialization failed") + } else { + hdr_ftr + .to_xml_footer() + .expect("footer serialization failed") + }; + + self.package.set_part(&part_name, xml); + self.package + .content_types + .add_override(&part_name, content_type); + + // Add relationship from document to header/footer + let rel_target = part_name.trim_start_matches("/word/"); + let rels = self.package.get_or_create_part_rels(&self.doc_part_name); + let rel_id = rels.add(rel_type, rel_target); + + // Add reference in section properties + let sect = self.section_properties_mut(); + let refs = if is_header { + &mut sect.header_refs + } else { + &mut sect.footer_refs + }; + + refs.retain(|r| r.hdr_ftr_type != hdr_type); + refs.push(HdrFtrRef { + hdr_ftr_type: hdr_type, + rel_id, + }); + } + + fn get_header_footer_text(&self, is_header: bool, hdr_type: HdrFtrType) -> Option { + let sect = self.document.body.sect_pr.as_ref()?; + let refs = if is_header { + §.header_refs + } else { + §.footer_refs + }; + let hdr_ref = refs.iter().find(|r| r.hdr_ftr_type == hdr_type)?; + + // Resolve the part + let rels = self.package.get_part_rels(&self.doc_part_name)?; + let rel = rels.get_by_id(&hdr_ref.rel_id)?; + let part_name = OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + let xml = self.package.get_part(&part_name)?; + let hdr_ftr = CT_HdrFtr::from_xml(xml).ok()?; + Some(hdr_ftr.text()) + } + + // ---- Numbering/Lists ---- + + /// Ensure a numbering part exists, creating it and its relationship if needed. + fn ensure_numbering(&mut self) -> &mut CT_Numbering { + if self.numbering.is_none() { + self.numbering = Some(CT_Numbering::new()); + + // Set up numbering relationship and content type + self.package + .get_or_create_part_rels(&self.doc_part_name) + .add(rel_types::NUMBERING, "numbering.xml"); + self.package.content_types.add_override( + "/word/numbering.xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml", + ); + } + self.numbering.as_mut().unwrap() + } + + /// Add a bullet list item at the given indentation level (0-based). + /// + /// If no bullet list definition exists yet, one is created automatically. + /// Returns a mutable `Paragraph` for further configuration. + pub fn add_bullet_list_item(&mut self, text: &str, level: u32) -> Paragraph<'_> { + // Find or create a bullet list numId + let num_id = { + let numbering = self.ensure_numbering(); + // Look for an existing bullet list + let existing = numbering.nums.iter().find(|n| { + numbering + .get_abstract_num_for(n.num_id) + .map(|a| { + a.levels.first().and_then(|l| l.num_fmt) + == Some(rdocx_oxml::numbering::ST_NumberFormat::Bullet) + }) + .unwrap_or(false) + }); + if let Some(existing) = existing { + existing.num_id + } else { + numbering.add_bullet_list() + } + }; + + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + let ppr = CT_PPr { + num_id: Some(num_id), + num_ilvl: Some(level), + ..Default::default() + }; + p.properties = Some(ppr); + + self.document.body.content.push(BodyContent::Paragraph(p)); + match self.document.body.content.last_mut().unwrap() { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + /// Add a numbered list item at the given indentation level (0-based). + /// + /// If no numbered list definition exists yet, one is created automatically. + /// Returns a mutable `Paragraph` for further configuration. + pub fn add_numbered_list_item(&mut self, text: &str, level: u32) -> Paragraph<'_> { + // Find or create a numbered list numId + let num_id = { + let numbering = self.ensure_numbering(); + // Look for an existing numbered list + let existing = numbering.nums.iter().find(|n| { + numbering + .get_abstract_num_for(n.num_id) + .map(|a| { + a.levels.first().and_then(|l| l.num_fmt) + == Some(rdocx_oxml::numbering::ST_NumberFormat::Decimal) + }) + .unwrap_or(false) + }); + if let Some(existing) = existing { + existing.num_id + } else { + numbering.add_numbered_list() + } + }; + + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + let ppr = CT_PPr { + num_id: Some(num_id), + num_ilvl: Some(level), + ..Default::default() + }; + p.properties = Some(ppr); + + self.document.body.content.push(BodyContent::Paragraph(p)); + match self.document.body.content.last_mut().unwrap() { + BodyContent::Paragraph(p) => Paragraph { inner: p }, + _ => unreachable!(), + } + } + + // ---- Style access ---- + + /// Get all styles. + pub fn styles(&self) -> Vec> { + self.styles + .styles + .iter() + .map(|s| Style { inner: s }) + .collect() + } + + /// Find a style by its ID. + pub fn style(&self, style_id: &str) -> Option> { + self.styles.get_by_id(style_id).map(|s| Style { inner: s }) + } + + // ---- Style manipulation ---- + + /// Add a custom style to the document. + pub fn add_style(&mut self, builder: StyleBuilder) { + self.styles.styles.push(builder.build()); + } + + /// Resolve the effective paragraph properties for a given style ID, + /// walking the full inheritance chain (docDefaults → basedOn → ...). + pub fn resolve_paragraph_properties(&self, style_id: Option<&str>) -> CT_PPr { + style::resolve_paragraph_properties(style_id, &self.styles) + } + + /// Resolve the effective run properties for the given paragraph and character styles, + /// walking the full inheritance chain. + pub fn resolve_run_properties( + &self, + para_style_id: Option<&str>, + run_style_id: Option<&str>, + ) -> CT_RPr { + style::resolve_run_properties(para_style_id, run_style_id, &self.styles) + } + + // ---- Section/Page setup ---- + + /// Get the section properties (page size, margins). + pub fn section_properties(&self) -> Option<&CT_SectPr> { + self.document.body.sect_pr.as_ref() + } + + /// Get a mutable reference to section properties, creating defaults if needed. + pub fn section_properties_mut(&mut self) -> &mut CT_SectPr { + self.document + .body + .sect_pr + .get_or_insert_with(CT_SectPr::default_letter) + } + + /// Set page size. + pub fn set_page_size(&mut self, width: Length, height: Length) { + let sect = self.section_properties_mut(); + sect.page_width = Some(width.as_twips()); + sect.page_height = Some(height.as_twips()); + } + + /// Set page orientation to landscape (swaps width and height if needed). + pub fn set_landscape(&mut self) { + let sect = self.section_properties_mut(); + sect.orientation = Some(ST_PageOrientation::Landscape); + // Swap width/height if portrait dimensions + if let (Some(w), Some(h)) = (sect.page_width, sect.page_height) + && w.0 < h.0 + { + sect.page_width = Some(h); + sect.page_height = Some(w); + } + } + + /// Set page orientation to portrait (swaps width and height if needed). + pub fn set_portrait(&mut self) { + let sect = self.section_properties_mut(); + sect.orientation = Some(ST_PageOrientation::Portrait); + // Swap width/height if landscape dimensions + if let (Some(w), Some(h)) = (sect.page_width, sect.page_height) + && w.0 > h.0 + { + sect.page_width = Some(h); + sect.page_height = Some(w); + } + } + + /// Set all page margins. + pub fn set_margins(&mut self, top: Length, right: Length, bottom: Length, left: Length) { + let sect = self.section_properties_mut(); + sect.margin_top = Some(top.as_twips()); + sect.margin_right = Some(right.as_twips()); + sect.margin_bottom = Some(bottom.as_twips()); + sect.margin_left = Some(left.as_twips()); + } + + /// Set equal-width column layout. + pub fn set_columns(&mut self, num: u32, spacing: Length) { + let sect = self.section_properties_mut(); + sect.columns = Some(CT_Columns { + num: Some(num), + space: Some(spacing.as_twips()), + equal_width: Some(true), + sep: None, + columns: Vec::new(), + }); + } + + /// Set header and footer distances from page edges. + pub fn set_header_footer_distance(&mut self, header: Length, footer: Length) { + let sect = self.section_properties_mut(); + sect.header_distance = Some(header.as_twips()); + sect.footer_distance = Some(footer.as_twips()); + } + + /// Set the gutter margin. + pub fn set_gutter(&mut self, gutter: Length) { + self.section_properties_mut().gutter = Some(gutter.as_twips()); + } + + /// Enable or disable different first page header/footer. + pub fn set_different_first_page(&mut self, val: bool) { + self.section_properties_mut().title_pg = Some(val); + } + + // ---- Metadata access ---- + + /// Get the document title. + pub fn title(&self) -> Option<&str> { + self.core_properties.as_ref()?.title.as_deref() + } + + /// Set the document title. + pub fn set_title(&mut self, title: &str) { + self.ensure_core_properties().title = Some(title.to_string()); + } + + /// Get the document author/creator. + pub fn author(&self) -> Option<&str> { + self.core_properties.as_ref()?.creator.as_deref() + } + + /// Set the document author/creator. + pub fn set_author(&mut self, author: &str) { + self.ensure_core_properties().creator = Some(author.to_string()); + } + + /// Get the document subject. + pub fn subject(&self) -> Option<&str> { + self.core_properties.as_ref()?.subject.as_deref() + } + + /// Set the document subject. + pub fn set_subject(&mut self, subject: &str) { + self.ensure_core_properties().subject = Some(subject.to_string()); + } + + /// Get the document keywords. + pub fn keywords(&self) -> Option<&str> { + self.core_properties.as_ref()?.keywords.as_deref() + } + + /// Set the document keywords. + pub fn set_keywords(&mut self, keywords: &str) { + self.ensure_core_properties().keywords = Some(keywords.to_string()); + } + + fn ensure_core_properties(&mut self) -> &mut CoreProperties { + self.core_properties + .get_or_insert_with(CoreProperties::default) + } + + // ---- Document Merging ---- + + /// Append the content of another document to this document. + /// + /// Copies all body content (paragraphs and tables) from the other document. + /// Handles style deduplication and numbering remapping. + pub fn append(&mut self, other: &Document) { + self.merge_styles(other); + + let start_idx = self.document.body.content.len(); + for content in &other.document.body.content { + self.document.body.content.push(content.clone()); + } + + self.remap_merged_numbering(other, start_idx); + } + + /// Append the content of another document with a section break. + pub fn append_with_break(&mut self, other: &Document, break_type: crate::SectionBreak) { + // Insert a section break paragraph before the merged content + let mut p = CT_P::new(); + let sect_pr = match break_type { + crate::SectionBreak::NextPage => CT_SectPr::default_letter(), + crate::SectionBreak::Continuous => { + let mut sp = CT_SectPr::default_letter(); + sp.section_type = Some(ST_SectionType::Continuous); + sp + } + crate::SectionBreak::EvenPage => { + let mut sp = CT_SectPr::default_letter(); + sp.section_type = Some(ST_SectionType::EvenPage); + sp + } + crate::SectionBreak::OddPage => { + let mut sp = CT_SectPr::default_letter(); + sp.section_type = Some(ST_SectionType::OddPage); + sp + } + }; + p.properties = Some(CT_PPr { + sect_pr: Some(sect_pr), + ..Default::default() + }); + self.document.body.content.push(BodyContent::Paragraph(p)); + + self.append(other); + } + + /// Insert the content of another document at a specified body index. + pub fn insert_document(&mut self, index: usize, other: &Document) { + self.merge_styles(other); + + let insert_at = index.min(self.document.body.content.len()); + for (i, content) in other.document.body.content.iter().enumerate() { + self.document + .body + .content + .insert(insert_at + i, content.clone()); + } + + self.remap_merged_numbering(other, insert_at); + } + + /// Merge styles from another document, avoiding duplicates. + fn merge_styles(&mut self, other: &Document) { + for style in &other.styles.styles { + if self.styles.get_by_id(&style.style_id).is_none() { + self.styles.styles.push(style.clone()); + } + } + } + + /// Merge numbering from another document and remap IDs in the merged content. + /// `start_idx` is the index where the other document's content starts in self. + fn remap_merged_numbering(&mut self, other: &Document, start_idx: usize) { + let Some(other_numbering) = &other.numbering else { + return; + }; + + let numbering = self + .numbering + .get_or_insert_with(|| rdocx_oxml::numbering::CT_Numbering { + abstract_nums: Vec::new(), + nums: Vec::new(), + }); + + // Find max existing IDs to avoid collision + let max_abstract_id = numbering + .abstract_nums + .iter() + .map(|a| a.abstract_num_id) + .max() + .unwrap_or(0); + let max_num_id = numbering.nums.iter().map(|n| n.num_id).max().unwrap_or(0); + + let abstract_offset = max_abstract_id + 1; + let num_offset = max_num_id + 1; + + // Copy abstract nums with remapped IDs + for abs_num in &other_numbering.abstract_nums { + let mut new_abs = abs_num.clone(); + new_abs.abstract_num_id += abstract_offset; + numbering.abstract_nums.push(new_abs); + } + + // Copy num instances with remapped IDs + for num in &other_numbering.nums { + let mut new_num = num.clone(); + new_num.num_id += num_offset; + new_num.abstract_num_id += abstract_offset; + numbering.nums.push(new_num); + } + + // Remap numId references in the merged content + let incoming_count = other.document.body.content.len(); + for content in self.document.body.content[start_idx..start_idx + incoming_count].iter_mut() + { + Self::remap_num_ids(content, num_offset); + } + } + + /// Remap numId references in body content by adding an offset. + fn remap_num_ids(content: &mut BodyContent, offset: u32) { + match content { + BodyContent::Paragraph(p) => { + Self::remap_paragraph_num_id(p, offset); + } + BodyContent::Table(tbl) => { + Self::remap_table_num_ids(tbl, offset); + } + BodyContent::RawXml(_) => {} + } + } + + fn remap_paragraph_num_id(p: &mut CT_P, offset: u32) { + if let Some(ppr) = &mut p.properties + && let Some(num_id) = &mut ppr.num_id + && *num_id > 0 + { + *num_id += offset; + } + } + + fn remap_table_num_ids(tbl: &mut CT_Tbl, offset: u32) { + for row in &mut tbl.rows { + for cell in &mut row.cells { + for cc in &mut cell.content { + match cc { + rdocx_oxml::table::CellContent::Paragraph(p) => { + Self::remap_paragraph_num_id(p, offset); + } + rdocx_oxml::table::CellContent::Table(nested) => { + Self::remap_table_num_ids(nested, offset); + } + } + } + } + } + } + + // ---- Table of Contents ---- + + /// Insert a Table of Contents at the given body content index. + /// + /// Scans the document for heading paragraphs (Heading1..HeadingN where N <= max_level), + /// inserts bookmark markers at each heading, and generates TOC entry paragraphs + /// with internal hyperlinks and dot-leader tab stops. + /// + /// # Arguments + /// * `index` - Body content index at which to insert the TOC + /// * `max_level` - Maximum heading level to include (1-9, typically 3) + pub fn insert_toc(&mut self, index: usize, max_level: u32) { + use rdocx_oxml::borders::{CT_TabStop, CT_Tabs}; + use rdocx_oxml::shared::{ST_TabJc, ST_TabLeader}; + use rdocx_oxml::text::HyperlinkSpan; + use rdocx_oxml::units::Twips; + + let max_level = max_level.clamp(1, 9); + + // Step 1: Collect heading info from the document body + struct HeadingInfo { + content_index: usize, + level: u32, + text: String, + bookmark_name: String, + } + + let mut headings = Vec::new(); + let mut toc_counter = 0u32; + + for (idx, content) in self.document.body.content.iter().enumerate() { + if let BodyContent::Paragraph(p) = content + && let Some(level) = Self::detect_heading_level_for_toc(p) + && level <= max_level + { + let text = p.text(); + if !text.trim().is_empty() { + toc_counter += 1; + headings.push(HeadingInfo { + content_index: idx, + level, + text, + bookmark_name: format!("_Toc{toc_counter}"), + }); + } + } + } + + // Step 2: Insert bookmark markers at each heading paragraph (as raw XML in extra_xml) + // We insert bookmarkStart/bookmarkEnd as extra_xml at position 0 in the paragraph. + // Adjust for insertions that shift indices. + let mut bookmark_id = 100; // Start at a high ID to avoid collision + for heading in &headings { + if let Some(BodyContent::Paragraph(p)) = + self.document.body.content.get_mut(heading.content_index) + { + let bm_start = format!( + "", + heading.bookmark_name + ); + let bm_end = format!(""); + // Insert at position 0 (before runs) + p.extra_xml.push((0, bm_start.into_bytes())); + // Insert at end (after runs) + p.extra_xml.push((p.runs.len(), bm_end.into_bytes())); + bookmark_id += 1; + } + } + + // Step 3: Build TOC entry paragraphs + // Right margin tab stop at 9360 twips (6.5") with dot leader + let right_tab = CT_Tabs { + tabs: vec![CT_TabStop { + val: ST_TabJc::Right, + pos: Twips(9360), + leader: Some(ST_TabLeader::Dot), + }], + }; + + let mut toc_paragraphs: Vec = Vec::new(); + + // TOC title + let mut title_p = CT_P::new(); + let mut title_r = CT_R::new("Table of Contents"); + title_r.properties = Some(CT_RPr { + bold: Some(true), + ..Default::default() + }); + title_p.runs.push(title_r); + title_p.properties = Some(CT_PPr { + space_after: Some(Twips(120)), + ..Default::default() + }); + toc_paragraphs.push(title_p); + + for heading in &headings { + let mut p = CT_P::new(); + + // Indentation based on heading level (each level indented 360 twips = 0.25") + let indent = Twips(360 * (heading.level as i32 - 1)); + + p.properties = Some(CT_PPr { + tabs: Some(right_tab.clone()), + ind_left: if indent.0 > 0 { Some(indent) } else { None }, + ..Default::default() + }); + + // Run with heading text + let text_run = CT_R::new(&heading.text); + p.runs.push(text_run); + + // Tab run (separates text from page number) + p.runs.push(CT_R { + properties: None, + content: vec![rdocx_oxml::text::RunContent::Tab], + extra_xml: Vec::new(), + }); + + // Wrap the text run in a hyperlink to the bookmark + p.hyperlinks.push(HyperlinkSpan { + rel_id: None, + anchor: Some(heading.bookmark_name.clone()), + run_start: 0, + run_end: 1, // Just the text run, not the tab + }); + + toc_paragraphs.push(p); + } + + // Step 4: Insert TOC paragraphs at the specified index + let insert_at = index.min(self.document.body.content.len()); + for (i, p) in toc_paragraphs.into_iter().enumerate() { + self.document + .body + .content + .insert(insert_at + i, BodyContent::Paragraph(p)); + } + } + + /// Detect heading level from a paragraph's style ID. + fn detect_heading_level_for_toc(para: &CT_P) -> Option { + let ppr = para.properties.as_ref()?; + let style_id = ppr.style_id.as_deref()?; + let rest = style_id.strip_prefix("Heading")?; + rest.parse::().ok().filter(|n| (1..=9).contains(n)) + } + + // ---- Placeholder replacement ---- + + /// Replace all occurrences of `placeholder` with `replacement` throughout the document. + /// + /// Searches body paragraphs, tables (including nested), headers, and footers. + /// Handles placeholders split across multiple runs. Returns the total number + /// of replacements made. + pub fn replace_text(&mut self, placeholder: &str, replacement: &str) -> usize { + use rdocx_oxml::placeholder; + + let mut count = 0; + + // Replace in body paragraphs + for content in &mut self.document.body.content { + match content { + BodyContent::Paragraph(p) => { + count += placeholder::replace_in_paragraph(p, placeholder, replacement); + } + BodyContent::Table(t) => { + count += placeholder::replace_in_table(t, placeholder, replacement); + } + _ => {} // Skip RawXml elements + } + } + + // Replace in headers and footers + if let Some(sect_pr) = self.document.body.sect_pr.as_ref() { + let hdr_rel_ids: Vec = sect_pr + .header_refs + .iter() + .map(|r| r.rel_id.clone()) + .collect(); + let ftr_rel_ids: Vec = sect_pr + .footer_refs + .iter() + .map(|r| r.rel_id.clone()) + .collect(); + + for rel_id in hdr_rel_ids { + if let Some(mut hf) = self.load_header_footer(&rel_id) { + let n = + placeholder::replace_in_header_footer(&mut hf, placeholder, replacement); + if n > 0 { + self.save_header_footer(&rel_id, &hf, true); + count += n; + } + } + } + for rel_id in ftr_rel_ids { + if let Some(mut hf) = self.load_header_footer(&rel_id) { + let n = + placeholder::replace_in_header_footer(&mut hf, placeholder, replacement); + if n > 0 { + self.save_header_footer(&rel_id, &hf, false); + count += n; + } + } + } + } + + // Flush document to package, then do raw XML pass for text boxes/shapes + if let Ok(()) = self.flush_to_package() { + count += self.replace_in_xml_parts(placeholder, replacement); + } + + count + } + + /// Replace multiple placeholders at once. Returns total replacements. + pub fn replace_all(&mut self, replacements: &std::collections::HashMap<&str, &str>) -> usize { + let mut count = 0; + for (placeholder, replacement) in replacements { + count += self.replace_text(placeholder, replacement); + } + count + } + + // ---- Regex replacement ---- + + /// Replace all regex matches with `replacement` throughout the document. + /// + /// The `replacement` string supports capture groups: `$1`, `$2`, etc. + /// Searches body paragraphs, tables (including nested), headers, and footers. + /// Returns the total number of replacements made, or an error if the regex is invalid. + pub fn replace_regex(&mut self, pattern: &str, replacement: &str) -> Result { + let re = + regex::Regex::new(pattern).map_err(|e| Error::Other(format!("invalid regex: {e}")))?; + Ok(self.replace_regex_compiled(&re, replacement)) + } + + /// Replace multiple regex patterns at once. Returns total replacements. + pub fn replace_all_regex(&mut self, patterns: &[(String, String)]) -> Result { + let mut count = 0; + for (pattern, replacement) in patterns { + count += self.replace_regex(pattern, replacement)?; + } + Ok(count) + } + + /// Internal: replace using a pre-compiled regex. + fn replace_regex_compiled(&mut self, re: ®ex::Regex, replacement: &str) -> usize { + use rdocx_oxml::placeholder; + + let mut count = 0; + + // Replace in body paragraphs and tables + for content in &mut self.document.body.content { + match content { + BodyContent::Paragraph(p) => { + count += placeholder::replace_regex_in_paragraph(p, re, replacement); + } + BodyContent::Table(t) => { + count += placeholder::replace_regex_in_table(t, re, replacement); + } + _ => {} + } + } + + // Replace in headers and footers + if let Some(sect_pr) = self.document.body.sect_pr.as_ref() { + let hdr_rel_ids: Vec = sect_pr + .header_refs + .iter() + .map(|r| r.rel_id.clone()) + .collect(); + let ftr_rel_ids: Vec = sect_pr + .footer_refs + .iter() + .map(|r| r.rel_id.clone()) + .collect(); + + for rel_id in hdr_rel_ids { + if let Some(mut hf) = self.load_header_footer(&rel_id) { + let n = placeholder::replace_regex_in_header_footer(&mut hf, re, replacement); + if n > 0 { + self.save_header_footer(&rel_id, &hf, true); + count += n; + } + } + } + for rel_id in ftr_rel_ids { + if let Some(mut hf) = self.load_header_footer(&rel_id) { + let n = placeholder::replace_regex_in_header_footer(&mut hf, re, replacement); + if n > 0 { + self.save_header_footer(&rel_id, &hf, false); + count += n; + } + } + } + } + + count + } + + /// Load a header/footer part by its relationship ID. + fn load_header_footer(&self, rel_id: &str) -> Option { + let rels = self.package.get_part_rels(&self.doc_part_name)?; + let rel = rels.get_by_id(rel_id)?; + let part_name = OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + let xml = self.package.get_part(&part_name)?; + CT_HdrFtr::from_xml(xml).ok() + } + + /// Run raw XML replacement on all XML parts (for text boxes, shapes, charts, etc.). + /// + /// This is called after the typed-model replacement and flush_to_package. + fn replace_in_xml_parts(&mut self, placeholder: &str, replacement: &str) -> usize { + use rdocx_oxml::placeholder::{replace_in_chart_xml, replace_in_xml_part}; + + let mut count = 0; + + // Collect part names for XML parts to process (text boxes/shapes) + let mut xml_parts: Vec = vec![self.doc_part_name.clone()]; + if let Some(sect_pr) = self.document.body.sect_pr.as_ref() + && let Some(rels) = self.package.get_part_rels(&self.doc_part_name) + { + for href in §_pr.header_refs { + if let Some(rel) = rels.get_by_id(&href.rel_id) { + xml_parts.push(OpcPackage::resolve_rel_target( + &self.doc_part_name, + &rel.target, + )); + } + } + for fref in §_pr.footer_refs { + if let Some(rel) = rels.get_by_id(&fref.rel_id) { + xml_parts.push(OpcPackage::resolve_rel_target( + &self.doc_part_name, + &rel.target, + )); + } + } + } + + for part_name in xml_parts { + if let Some(xml) = self.package.get_part(&part_name) { + let xml = xml.to_vec(); + if let Ok((new_xml, n)) = replace_in_xml_part(&xml, placeholder, replacement) + && n > 0 + { + self.package.set_part(&part_name, new_xml); + count += n; + } + } + } + + // Collect chart part names + let chart_parts: Vec = self + .package + .get_part_rels(&self.doc_part_name) + .map(|rels| { + rels.get_all_by_type(rel_types::CHART) + .iter() + .map(|rel| OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target)) + .collect() + }) + .unwrap_or_default(); + + for part_name in chart_parts { + if let Some(xml) = self.package.get_part(&part_name) { + let xml = xml.to_vec(); + if let Ok((new_xml, n)) = replace_in_chart_xml(&xml, placeholder, replacement) + && n > 0 + { + self.package.set_part(&part_name, new_xml); + count += n; + } + } + } + + // Re-parse document from the (possibly modified) package XML + if count > 0 + && let Some(doc_xml) = self.package.get_part(&self.doc_part_name) + && let Ok(doc) = CT_Document::from_xml(doc_xml) + { + self.document = doc; + } + + count + } + + // ---- PDF conversion ---- + + /// Render the document to PDF bytes. + /// + /// This performs a full layout pass (font shaping, line breaking, pagination) + /// and then renders the result to a PDF document. + /// + /// Font resolution order: + /// 1. Fonts embedded in the DOCX file (word/fonts/) + /// 2. System fonts + /// 3. Bundled fonts (if `bundled-fonts` feature is enabled) + pub fn to_pdf(&self) -> Result> { + self.to_pdf_with_fonts(&[]) + } + + /// Render the document to PDF bytes with user-provided font files. + /// + /// User-provided fonts take highest priority in font resolution. + /// + /// # Arguments + /// * `font_files` - Additional font files to use. Each entry is `(family_name, font_bytes)`. + /// + /// Font resolution order: + /// 1. User-provided fonts (this parameter) + /// 2. Fonts embedded in the DOCX file (word/fonts/) + /// 3. System fonts + /// 4. Bundled fonts (if `bundled-fonts` feature is enabled) + pub fn to_pdf_with_fonts(&self, font_files: &[(&str, &[u8])]) -> Result> { + let mut input = self.build_layout_input(); + for (family, data) in font_files { + input.fonts.push(rdocx_layout::FontFile { + family: family.to_string(), + data: data.to_vec(), + }); + } + let layout = rdocx_layout::layout_document(&input)?; + Ok(rdocx_pdf::render_to_pdf(&layout)) + } + + /// Save the document as a PDF file. + pub fn save_pdf>(&self, path: P) -> Result<()> { + let pdf_bytes = self.to_pdf()?; + std::fs::write(path, pdf_bytes)?; + Ok(()) + } + + /// Convert the document to a complete HTML document string. + pub fn to_html(&self) -> String { + let input = self.build_html_input(); + rdocx_html::to_html_document(&input, &rdocx_html::HtmlOptions::default()) + } + + /// Convert the document to an HTML fragment (body content only, no `` wrapper). + pub fn to_html_fragment(&self) -> String { + let input = self.build_html_input(); + rdocx_html::to_html_fragment(&input, &rdocx_html::HtmlOptions::default()) + } + + /// Convert the document to Markdown. + pub fn to_markdown(&self) -> String { + let input = self.build_html_input(); + rdocx_html::to_markdown(&input) + } + + /// Build an HtmlInput from the document's current state. + fn build_html_input(&self) -> rdocx_html::HtmlInput { + use rdocx_opc::relationship::rel_types; + use std::collections::HashMap; + + let mut images: HashMap = HashMap::new(); + let mut hyperlink_urls: HashMap = HashMap::new(); + + if let Some(rels) = self.package.get_part_rels(&self.doc_part_name) { + for rel in &rels.items { + match rel.rel_type.as_str() { + t if t == rel_types::IMAGE => { + let part_name = + OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + if let Some(data) = self.package.get_part(&part_name) { + let content_type = guess_image_content_type(&part_name); + images.insert( + rel.id.clone(), + rdocx_html::ImageData { + data: data.to_vec(), + content_type, + }, + ); + } + } + t if t == rel_types::HYPERLINK => { + if rel.target_mode.as_ref().is_some_and(|m| m == "External") { + hyperlink_urls.insert(rel.id.clone(), rel.target.clone()); + } + } + _ => {} + } + } + } + + rdocx_html::HtmlInput { + document: self.document.clone(), + styles: self.styles.clone(), + numbering: self.numbering.clone(), + images, + hyperlink_urls, + } + } + + /// Render a single page of the document to PNG bytes. + /// + /// # Arguments + /// * `page_index` - 0-based page index + /// * `dpi` - Resolution (72 = 1:1, 150 = standard, 300 = high quality) + pub fn render_page_to_png(&self, page_index: usize, dpi: f64) -> Result>> { + let input = self.build_layout_input(); + let layout = rdocx_layout::layout_document(&input)?; + Ok(rdocx_pdf::render_page_to_png(&layout, page_index, dpi)) + } + + /// Render all pages of the document to PNG bytes. + pub fn render_all_pages(&self, dpi: f64) -> Result>> { + let input = self.build_layout_input(); + let layout = rdocx_layout::layout_document(&input)?; + Ok(rdocx_pdf::render_all_pages(&layout, dpi)) + } + + /// Build a LayoutInput from the document's current state. + fn build_layout_input(&self) -> rdocx_layout::LayoutInput { + use rdocx_layout::{ImageData, LayoutInput}; + use rdocx_opc::relationship::rel_types; + use std::collections::HashMap; + + let mut headers: HashMap = HashMap::new(); + let mut footers: HashMap = HashMap::new(); + let mut images: HashMap = HashMap::new(); + let mut hyperlink_urls: HashMap = HashMap::new(); + let mut footnotes = None; + let mut endnotes = None; + + // Extract embedded fonts from the DOCX package + let fonts = self.extract_embedded_fonts(); + + if let Some(rels) = self.package.get_part_rels(&self.doc_part_name) { + for rel in &rels.items { + match rel.rel_type.as_str() { + t if t == rel_types::HEADER => { + let part_name = + OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + if let Some(xml) = self.package.get_part(&part_name) + && let Ok(hf) = CT_HdrFtr::from_xml(xml) + { + headers.insert(rel.id.clone(), hf); + } + } + t if t == rel_types::FOOTER => { + let part_name = + OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + if let Some(xml) = self.package.get_part(&part_name) + && let Ok(hf) = CT_HdrFtr::from_xml(xml) + { + footers.insert(rel.id.clone(), hf); + } + } + t if t == rel_types::IMAGE => { + let part_name = + OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + if let Some(data) = self.package.get_part(&part_name) { + let content_type = guess_image_content_type(&part_name); + images.insert( + rel.id.clone(), + ImageData { + data: data.to_vec(), + content_type, + }, + ); + } + } + t if t == rel_types::HYPERLINK => { + if rel.target_mode.as_ref().is_some_and(|m| m == "External") { + hyperlink_urls.insert(rel.id.clone(), rel.target.clone()); + } + } + t if t == rel_types::FOOTNOTES => { + let part_name = + OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + if let Some(xml) = self.package.get_part(&part_name) { + footnotes = rdocx_oxml::footnotes::CT_Footnotes::from_xml(xml).ok(); + } + } + t if t == rel_types::ENDNOTES => { + let part_name = + OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target); + if let Some(xml) = self.package.get_part(&part_name) { + endnotes = rdocx_oxml::footnotes::CT_Footnotes::from_xml(xml).ok(); + } + } + _ => {} + } + } + } + + // Parse theme if available + let theme = self + .package + .get_part("/word/theme/theme1.xml") + .and_then(|data| rdocx_oxml::theme::Theme::from_xml(data).ok()); + + LayoutInput { + document: self.document.clone(), + styles: self.styles.clone(), + numbering: self.numbering.clone(), + headers, + footers, + images, + core_properties: self.core_properties.clone(), + hyperlink_urls, + footnotes, + endnotes, + theme, + fonts, + } + } + + /// Extract embedded fonts from the DOCX package. + /// + /// Word can embed fonts as `.odttf` (obfuscated TrueType) or regular `.ttf`/`.otf` + /// files in the `word/fonts/` directory. ODTTF files have the first 32 bytes + /// XOR'd with a 16-byte GUID derived from the font's relationship ID. + fn extract_embedded_fonts(&self) -> Vec { + let mut fonts = Vec::new(); + + // Look for font parts in word/fonts/ directory + for (part_name, data) in &self.package.parts { + let lower = part_name.to_lowercase(); + if !lower.contains("/word/fonts/") && !lower.contains("/word/font") { + continue; + } + + // Determine font family name from the file name + let file_name = part_name.rsplit('/').next().unwrap_or(part_name); + let family = file_name.split('.').next().unwrap_or(file_name).to_string(); + + if lower.ends_with(".odttf") { + // Deobfuscate ODTTF: XOR first 32 bytes with GUID from the file name + if let Some(deobfuscated) = deobfuscate_odttf(data, file_name) { + fonts.push(rdocx_layout::FontFile { + family, + data: deobfuscated, + }); + } + } else if lower.ends_with(".ttf") || lower.ends_with(".otf") || lower.ends_with(".ttc") + { + fonts.push(rdocx_layout::FontFile { + family, + data: data.clone(), + }); + } + } + + fonts + } + + /// Load font files from a directory and return them as FontFile entries. + /// + /// This is useful for CLI tools that accept a `--font-dir` argument. + /// Supports `.ttf`, `.otf`, and `.ttc` files. + pub fn load_fonts_from_dir>(dir: P) -> Vec { + let mut fonts = Vec::new(); + let dir = dir.as_ref(); + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + let ext = path + .extension() + .and_then(|e| e.to_str()) + .unwrap_or("") + .to_lowercase(); + if (ext == "ttf" || ext == "otf" || ext == "ttc") + && let Ok(data) = std::fs::read(&path) + { + let family = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or("Unknown") + .to_string(); + fonts.push(rdocx_layout::FontFile { family, data }); + } + } + } + fonts + } + + /// Save a header/footer part back to the OPC package. + fn save_header_footer(&mut self, rel_id: &str, hf: &CT_HdrFtr, is_header: bool) { + let part_name = { + let rels = self.package.get_part_rels(&self.doc_part_name); + rels.and_then(|r| r.get_by_id(rel_id)) + .map(|rel| OpcPackage::resolve_rel_target(&self.doc_part_name, &rel.target)) + }; + if let Some(part_name) = part_name { + let xml = if is_header { + hf.to_xml_header() + } else { + hf.to_xml_footer() + }; + if let Ok(xml) = xml { + self.package.set_part(&part_name, xml); + } + } + } + + // ---- Document Intelligence API ---- + + /// Get all headings in the document as (level, text) pairs. + /// + /// Detects heading paragraphs by their style ID (e.g. "Heading1", "Heading2"). + pub fn headings(&self) -> Vec<(u32, String)> { + let mut result = Vec::new(); + for content in &self.document.body.content { + if let BodyContent::Paragraph(p) = content + && let Some(level) = Self::detect_heading_level_for_toc(p) + { + result.push((level, p.text())); + } + } + result + } + + /// Get a hierarchical outline of the document headings. + /// + /// Returns a tree structure where each node contains the heading level, + /// text, and children (sub-headings). + pub fn document_outline(&self) -> Vec { + let headings = self.headings(); + build_outline_tree(&headings) + } + + /// Get information about all images in the document. + /// + /// Returns metadata for each inline and anchored image found in body paragraphs. + pub fn images(&self) -> Vec { + let mut result = Vec::new(); + + for content in &self.document.body.content { + Self::collect_images_from_content(content, &mut result); + } + result + } + + fn collect_images_from_content(content: &BodyContent, result: &mut Vec) { + match content { + BodyContent::Paragraph(p) => { + for run in &p.runs { + for rc in &run.content { + if let RunContent::Drawing(drawing) = rc { + if let Some(inline) = &drawing.inline { + result.push(ImageInfo { + embed_id: inline.embed_id.clone(), + name: inline.name.clone(), + description: inline.description.clone(), + width_emu: inline.extent_cx.0, + height_emu: inline.extent_cy.0, + is_anchor: false, + }); + } + if let Some(anchor) = &drawing.anchor { + result.push(ImageInfo { + embed_id: anchor.embed_id.clone(), + name: anchor.name.clone(), + description: anchor.description.clone(), + width_emu: anchor.extent_cx.0, + height_emu: anchor.extent_cy.0, + is_anchor: true, + }); + } + } + } + } + } + BodyContent::Table(tbl) => { + for row in &tbl.rows { + for cell in &row.cells { + for cc in &cell.content { + match cc { + rdocx_oxml::table::CellContent::Paragraph(p) => { + Self::collect_images_from_content( + &BodyContent::Paragraph(p.clone()), + result, + ); + } + rdocx_oxml::table::CellContent::Table(nested) => { + Self::collect_images_from_content( + &BodyContent::Table(nested.clone()), + result, + ); + } + } + } + } + } + } + BodyContent::RawXml(_) => {} + } + } + + /// Get information about all hyperlinks in the document. + /// + /// Resolves hyperlink relationship IDs to their target URLs where possible. + pub fn links(&self) -> Vec { + use rdocx_opc::relationship::rel_types; + + // Build a map of hyperlink rel_id -> target URL + let mut url_map = std::collections::HashMap::new(); + if let Some(rels) = self.package.get_part_rels(&self.doc_part_name) { + for rel in &rels.items { + if rel.rel_type == rel_types::HYPERLINK + && rel.target_mode.as_ref().is_some_and(|m| m == "External") + { + url_map.insert(rel.id.clone(), rel.target.clone()); + } + } + } + + let mut result = Vec::new(); + for content in &self.document.body.content { + if let BodyContent::Paragraph(p) = content { + for hl in &p.hyperlinks { + let text: String = p.runs[hl.run_start..hl.run_end] + .iter() + .map(|r| r.text()) + .collect::>() + .join(""); + + let url = hl.rel_id.as_ref().and_then(|id| url_map.get(id)).cloned(); + + result.push(LinkInfo { + text, + url, + anchor: hl.anchor.clone(), + rel_id: hl.rel_id.clone(), + }); + } + } + } + result + } + + /// Count the number of words in the document. + /// + /// Counts whitespace-separated tokens across all paragraphs (including + /// paragraphs inside table cells). + pub fn word_count(&self) -> usize { + let mut count = 0; + for content in &self.document.body.content { + count += Self::word_count_in_content(content); + } + count + } + + fn word_count_in_content(content: &BodyContent) -> usize { + match content { + BodyContent::Paragraph(p) => p.text().split_whitespace().count(), + BodyContent::Table(tbl) => { + let mut count = 0; + for row in &tbl.rows { + for cell in &row.cells { + for cc in &cell.content { + match cc { + rdocx_oxml::table::CellContent::Paragraph(p) => { + count += p.text().split_whitespace().count(); + } + rdocx_oxml::table::CellContent::Table(nested) => { + count += Self::word_count_in_content(&BodyContent::Table( + nested.clone(), + )); + } + } + } + } + } + count + } + BodyContent::RawXml(_) => 0, + } + } + + /// Audit the document for accessibility issues. + /// + /// Checks for common problems: missing image alt text, heading level gaps, + /// empty paragraphs, missing document metadata. + pub fn audit_accessibility(&self) -> Vec { + let mut issues = Vec::new(); + + // Check: missing document title + if self.title().is_none() { + issues.push(AccessibilityIssue { + severity: IssueSeverity::Warning, + message: "Document has no title".to_string(), + }); + } + + // Check: missing document language (author as a proxy for basic metadata) + if self.author().is_none() { + issues.push(AccessibilityIssue { + severity: IssueSeverity::Info, + message: "Document has no author".to_string(), + }); + } + + // Check: images without alt text + let images = self.images(); + for img in &images { + let has_alt = img + .description + .as_ref() + .is_some_and(|d| !d.is_empty() && d != "Background"); + if !has_alt { + let name = img + .name + .as_deref() + .or(Some(&img.embed_id)) + .unwrap_or("unknown"); + issues.push(AccessibilityIssue { + severity: IssueSeverity::Error, + message: format!("Image \"{name}\" has no alt text"), + }); + } + } + + // Check: heading level gaps + let headings = self.headings(); + let mut prev_level: Option = None; + for (level, text) in &headings { + if let Some(prev) = prev_level + && *level > prev + 1 + { + issues.push(AccessibilityIssue { + severity: IssueSeverity::Warning, + message: format!( + "Heading level gap: h{prev} -> h{level} (\"{}\")", + truncate_str(text, 40) + ), + }); + } + prev_level = Some(*level); + } + + // Check: excessive empty paragraphs + let mut consecutive_empty = 0u32; + for content in &self.document.body.content { + if let BodyContent::Paragraph(p) = content { + if p.text().trim().is_empty() { + consecutive_empty += 1; + if consecutive_empty >= 3 { + issues.push(AccessibilityIssue { + severity: IssueSeverity::Info, + message: format!( + "{consecutive_empty} consecutive empty paragraphs (consider using spacing instead)" + ), + }); + } + } else { + consecutive_empty = 0; + } + } else { + consecutive_empty = 0; + } + } + + issues + } +} + +impl Default for Document { + fn default() -> Self { + Self::new() + } +} + +/// Guess image content type from the part name extension. +fn guess_image_content_type(part_name: &str) -> String { + let ext = part_name.rsplit('.').next().unwrap_or("").to_lowercase(); + match ext.as_str() { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "bmp" => "image/bmp", + "tiff" | "tif" => "image/tiff", + _ => "image/png", + } + .to_string() +} + +/// A node in the document outline tree. +#[derive(Debug, Clone, PartialEq)] +pub struct OutlineNode { + /// The heading level (1-9). + pub level: u32, + /// The heading text. + pub text: String, + /// Child headings (sub-headings). + pub children: Vec, +} + +/// Information about an image in the document. +#[derive(Debug, Clone, PartialEq)] +pub struct ImageInfo { + /// The relationship ID for the embedded image. + pub embed_id: String, + /// Optional name attribute. + pub name: Option, + /// Optional description (alt text). + pub description: Option, + /// Width in EMUs (English Metric Units, 914400 EMU = 1 inch). + pub width_emu: i64, + /// Height in EMUs. + pub height_emu: i64, + /// Whether this is an anchored (floating) image vs inline. + pub is_anchor: bool, +} + +/// Information about a hyperlink in the document. +#[derive(Debug, Clone, PartialEq)] +pub struct LinkInfo { + /// The display text of the hyperlink. + pub text: String, + /// The resolved target URL (if external). + pub url: Option, + /// Internal document anchor (if any). + pub anchor: Option, + /// The relationship ID. + pub rel_id: Option, +} + +/// Severity level for accessibility issues. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IssueSeverity { + /// Informational suggestion. + Info, + /// Potential problem. + Warning, + /// Definite accessibility barrier. + Error, +} + +/// An accessibility issue found during audit. +#[derive(Debug, Clone, PartialEq)] +pub struct AccessibilityIssue { + /// How severe the issue is. + pub severity: IssueSeverity, + /// Human-readable description of the issue. + pub message: String, +} + +/// Build a hierarchical outline tree from a flat list of (level, text) headings. +fn build_outline_tree(headings: &[(u32, String)]) -> Vec { + let mut root: Vec = Vec::new(); + let mut stack: Vec<(u32, usize)> = Vec::new(); // (level, index in parent's children) + + for (level, text) in headings { + let node = OutlineNode { + level: *level, + text: text.clone(), + children: Vec::new(), + }; + + // Pop stack until we find a parent with a lower level + while let Some(&(stack_level, _)) = stack.last() { + if stack_level >= *level { + stack.pop(); + } else { + break; + } + } + + if stack.is_empty() { + root.push(node); + let idx = root.len() - 1; + stack.push((*level, idx)); + } else { + // Navigate to the correct parent in the tree + let target = get_outline_parent_mut(&mut root, &stack); + target.children.push(node); + let idx = target.children.len() - 1; + stack.push((*level, idx)); + } + } + + root +} + +/// Navigate to the parent node indicated by the stack. +fn get_outline_parent_mut<'a>( + root: &'a mut [OutlineNode], + stack: &[(u32, usize)], +) -> &'a mut OutlineNode { + let mut current = &mut root[stack[0].1]; + for &(_, idx) in &stack[1..] { + current = &mut current.children[idx]; + } + current +} + +/// Truncate a string to a maximum length, appending "..." if truncated. +fn truncate_str(s: &str, max_len: usize) -> String { + if s.len() <= max_len { + s.to_string() + } else { + let truncated: String = s.chars().take(max_len.saturating_sub(3)).collect(); + format!("{truncated}...") + } +} + +/// Deobfuscate an ODTTF (obfuscated TrueType) font file. +/// +/// Word embeds fonts as `.odttf` files where the first 32 bytes are XOR'd +/// with a 16-byte GUID derived from the file name. The file name follows +/// the pattern `{GUID}.odttf` where GUID is a hex string without hyphens. +fn deobfuscate_odttf(data: &[u8], file_name: &str) -> Option> { + if data.len() < 32 { + return None; + } + + // Extract GUID from file name: "00112233-4455-6677-8899-AABBCCDDEEFF.odttf" + // or "{00112233-4455-6677-8899-AABBCCDDEEFF}.odttf" + let name = file_name + .split('.') + .next() + .unwrap_or("") + .trim_start_matches('{') + .trim_end_matches('}'); + + // Remove hyphens and parse as hex bytes + let hex: String = name.chars().filter(|c| c.is_ascii_hexdigit()).collect(); + if hex.len() != 32 { + return None; + } + + let mut guid = [0u8; 16]; + for (i, byte) in guid.iter_mut().enumerate() { + *byte = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).ok()?; + } + + // Per OOXML spec, the GUID bytes are reordered for XOR key: + // bytes 0-3 reversed, 4-5 reversed, 6-7 reversed, 8-15 as-is + let key: [u8; 16] = [ + guid[3], guid[2], guid[1], guid[0], guid[5], guid[4], guid[7], guid[6], guid[8], guid[9], + guid[10], guid[11], guid[12], guid[13], guid[14], guid[15], + ]; + + let mut result = data.to_vec(); + // XOR first 32 bytes with the 16-byte key (repeated twice) + for i in 0..32 { + result[i] ^= key[i % 16]; + } + + Some(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::paragraph::Alignment; + use rdocx_oxml::units::{HalfPoint, Twips}; + + #[test] + fn create_new_document() { + let doc = Document::new(); + assert_eq!(doc.paragraph_count(), 0); + assert!(doc.section_properties().is_some()); + } + + #[test] + fn add_paragraphs() { + let mut doc = Document::new(); + doc.add_paragraph("First paragraph"); + doc.add_paragraph("Second paragraph"); + assert_eq!(doc.paragraph_count(), 2); + + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "First paragraph"); + assert_eq!(paras[1].text(), "Second paragraph"); + } + + #[test] + fn paragraph_formatting() { + let mut doc = Document::new(); + doc.add_paragraph("Centered").alignment(Alignment::Center); + + let paras = doc.paragraphs(); + assert_eq!(paras[0].alignment(), Some(Alignment::Center)); + } + + #[test] + fn run_formatting() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("Bold text").bold(true).size(14.0); + + let paras = doc.paragraphs(); + let runs: Vec<_> = paras[0].runs().collect(); + assert!(runs[0].is_bold()); + assert_eq!(runs[0].size(), Some(14.0)); + } + + #[test] + fn round_trip_in_memory() { + let mut doc = Document::new(); + doc.add_paragraph("Hello, World!"); + doc.add_paragraph("Second paragraph") + .alignment(Alignment::Center); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 2); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Hello, World!"); + assert_eq!(paras[1].text(), "Second paragraph"); + assert_eq!(paras[1].alignment(), Some(Alignment::Center)); + } + + #[test] + fn styles_present() { + let doc = Document::new(); + assert!(doc.style("Normal").is_some()); + assert!(doc.style("Heading1").is_some()); + } + + #[test] + fn paragraph_with_style() { + let mut doc = Document::new(); + doc.add_paragraph("Title").style("Heading1"); + + let paras = doc.paragraphs(); + assert_eq!(paras[0].style_id(), Some("Heading1")); + } + + #[test] + fn multiple_runs_in_paragraph() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("Normal "); + para.add_run("bold ").bold(true); + para.add_run("italic").italic(true); + + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "Normal bold italic"); + let runs: Vec<_> = paras[0].runs().collect(); + assert_eq!(runs.len(), 3); + assert!(!runs[0].is_bold()); + assert!(runs[1].is_bold()); + assert!(runs[2].is_italic()); + } + + #[test] + fn add_custom_style() { + let mut doc = Document::new(); + doc.add_style(StyleBuilder::paragraph("MyCustom", "My Custom Style").based_on("Normal")); + assert!(doc.style("MyCustom").is_some()); + let s = doc.style("MyCustom").unwrap(); + assert_eq!(s.name(), Some("My Custom Style")); + assert_eq!(s.based_on(), Some("Normal")); + } + + #[test] + fn resolve_style_properties() { + let doc = Document::new(); + // Heading1 should inherit from docDefaults and have its own overrides + let ppr = doc.resolve_paragraph_properties(Some("Heading1")); + assert_eq!(ppr.keep_next, Some(true)); + assert_eq!(ppr.space_before, Some(Twips(240))); + + // Default (None) should apply Normal style + let ppr = doc.resolve_paragraph_properties(None); + assert_eq!(ppr.space_after, Some(Twips(160))); + } + + #[test] + fn resolve_run_style_properties() { + let doc = Document::new(); + let rpr = doc.resolve_run_properties(Some("Heading1"), None); + assert_eq!(rpr.bold, Some(true)); + assert_eq!(rpr.sz, Some(HalfPoint(32))); + assert_eq!(rpr.font_ascii, Some("Calibri".to_string())); + } + + #[test] + fn set_landscape() { + let mut doc = Document::new(); + doc.set_landscape(); + let sect = doc.section_properties().unwrap(); + assert_eq!(sect.orientation, Some(ST_PageOrientation::Landscape)); + // Width should be > height in landscape + assert!(sect.page_width.unwrap().0 > sect.page_height.unwrap().0); + } + + #[test] + fn set_margins() { + let mut doc = Document::new(); + doc.set_margins( + Length::inches(0.5), + Length::inches(0.75), + Length::inches(0.5), + Length::inches(0.75), + ); + let sect = doc.section_properties().unwrap(); + assert_eq!(sect.margin_top, Some(Twips(720))); + assert_eq!(sect.margin_right, Some(Twips(1080))); + } + + #[test] + fn set_columns() { + let mut doc = Document::new(); + doc.set_columns(2, Length::inches(0.5)); + let sect = doc.section_properties().unwrap(); + let cols = sect.columns.as_ref().unwrap(); + assert_eq!(cols.num, Some(2)); + assert_eq!(cols.space, Some(Twips(720))); + assert_eq!(cols.equal_width, Some(true)); + } + + #[test] + fn set_page_size() { + let mut doc = Document::new(); + doc.set_page_size(Length::cm(21.0), Length::cm(29.7)); + let sect = doc.section_properties().unwrap(); + // A4: ~11906tw x ~16838tw + let w = sect.page_width.unwrap().0; + let h = sect.page_height.unwrap().0; + assert!((w - 11906).abs() < 5); + assert!((h - 16838).abs() < 5); + } + + #[test] + fn set_different_first_page() { + let mut doc = Document::new(); + doc.set_different_first_page(true); + assert_eq!(doc.section_properties().unwrap().title_pg, Some(true)); + } + + #[test] + fn content_insertion_api() { + let mut doc = Document::new(); + doc.add_paragraph("First"); + doc.add_paragraph("Third"); + + // Insert in middle + doc.insert_paragraph(1, "Second"); + assert_eq!(doc.content_count(), 3); + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "First"); + assert_eq!(paras[1].text(), "Second"); + assert_eq!(paras[2].text(), "Third"); + + // Insert at beginning + doc.insert_paragraph(0, "Zeroth"); + assert_eq!(doc.content_count(), 4); + assert_eq!(doc.paragraphs()[0].text(), "Zeroth"); + } + + #[test] + fn find_content_index_and_remove() { + let mut doc = Document::new(); + doc.add_paragraph("Hello"); + doc.add_paragraph("{{PLACEHOLDER}}"); + doc.add_paragraph("World"); + + assert_eq!(doc.find_content_index("{{PLACEHOLDER}}"), Some(1)); + assert_eq!(doc.find_content_index("NONEXISTENT"), None); + + assert!(doc.remove_content(1)); + assert_eq!(doc.content_count(), 2); + assert_eq!(doc.paragraphs()[1].text(), "World"); + + // Out of bounds + assert!(!doc.remove_content(10)); + } + + #[test] + fn insert_table_at_index() { + let mut doc = Document::new(); + doc.add_paragraph("Before"); + doc.add_paragraph("After"); + + doc.insert_table(1, 2, 3); + assert_eq!(doc.content_count(), 3); + assert_eq!(doc.table_count(), 1); + // Paragraphs are still in correct order + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "Before"); + assert_eq!(paras[1].text(), "After"); + } + + #[test] + fn replace_text_in_body() { + let mut doc = Document::new(); + doc.add_paragraph("Hello {{name}}!"); + doc.add_paragraph("Welcome to {{company}}."); + + let count = doc.replace_text("{{name}}", "Alice"); + assert_eq!(count, 1); + assert_eq!(doc.paragraphs()[0].text(), "Hello Alice!"); + + let count = doc.replace_text("{{company}}", "Acme"); + assert_eq!(count, 1); + assert_eq!(doc.paragraphs()[1].text(), "Welcome to Acme."); + } + + #[test] + fn replace_text_in_header_and_footer() { + let mut doc = Document::new(); + doc.set_header("Header: {{title}}"); + doc.set_footer("Footer: {{title}}"); + doc.add_paragraph("Body: {{title}}"); + + let count = doc.replace_text("{{title}}", "My Doc"); + assert_eq!(count, 3); + + assert_eq!(doc.paragraphs()[0].text(), "Body: My Doc"); + assert_eq!(doc.header_text().unwrap(), "Header: My Doc"); + assert_eq!(doc.footer_text().unwrap(), "Footer: My Doc"); + } + + #[test] + fn replace_all_batch() { + let mut doc = Document::new(); + doc.add_paragraph("{{a}} and {{b}}"); + + let mut map = std::collections::HashMap::new(); + map.insert("{{a}}", "X"); + map.insert("{{b}}", "Y"); + let count = doc.replace_all(&map); + assert_eq!(count, 2); + assert_eq!(doc.paragraphs()[0].text(), "X and Y"); + } + + #[test] + fn template_workflow_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Company: {{company}}"); + doc.add_paragraph("Date: {{date}}"); + + doc.replace_text("{{company}}", "Acme Corp"); + doc.replace_text("{{date}}", "2026-02-22"); + + // Round-trip + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraphs()[0].text(), "Company: Acme Corp"); + assert_eq!(doc2.paragraphs()[1].text(), "Date: 2026-02-22"); + } + + #[test] + fn add_background_image_round_trip() { + // Create a minimal 1x1 PNG + let png_data: Vec = vec![ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, // PNG signature + 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, // IHDR chunk + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, // 1x1 + 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, + 0x44, 0x41, 0x54, // IDAT chunk + 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0xe2, 0x21, + 0xbc, 0x33, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, // IEND chunk + 0xae, 0x42, 0x60, 0x82, + ]; + + let mut doc = Document::new(); + doc.add_paragraph("Hello World"); + doc.add_background_image(&png_data, "bg.png"); + + // Background image paragraph should be at index 0 + assert_eq!(doc.content_count(), 2); + + // Round-trip + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + // Should still have 2 content items + assert_eq!(doc2.content_count(), 2); + // The second paragraph should have our text + assert_eq!(doc2.paragraphs().last().unwrap().text(), "Hello World"); + } + + #[test] + fn add_anchored_image() { + let png_data: Vec = vec![ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, + 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, + 0x00, 0x90, 0x77, 0x53, 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, + 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0xe2, 0x21, 0xbc, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]; + + let mut doc = Document::new(); + doc.add_paragraph("Content"); + doc.add_anchored_image( + &png_data, + "overlay.png", + Length::inches(4.0), + Length::inches(3.0), + false, + ); + assert_eq!(doc.content_count(), 2); + } + + #[test] + fn insert_toc_basic() { + let mut doc = Document::new(); + doc.add_paragraph("Introduction"); + doc.add_paragraph("Chapter 1").style("Heading1"); + doc.add_paragraph("Some text in chapter 1."); + doc.add_paragraph("Section 1.1").style("Heading2"); + doc.add_paragraph("Text in section 1.1."); + doc.add_paragraph("Chapter 2").style("Heading1"); + doc.add_paragraph("Text in chapter 2."); + + // Before TOC: 7 content elements + assert_eq!(doc.content_count(), 7); + + // Insert TOC at index 0 with max_level 2 + doc.insert_toc(0, 2); + + // TOC adds: 1 title + 3 heading entries (Ch1, Sec1.1, Ch2) = 4 paragraphs + assert_eq!(doc.content_count(), 11); + + // Verify TOC title + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "Table of Contents"); + + // Verify TOC entries contain heading text + assert_eq!(paras[1].text(), "Chapter 1\t"); + assert_eq!(paras[2].text(), "Section 1.1\t"); + assert_eq!(paras[3].text(), "Chapter 2\t"); + + // Verify round-trip: save and re-open + let bytes = doc.to_bytes().expect("should serialize"); + let doc2 = Document::from_bytes(&bytes).expect("should open"); + assert_eq!(doc2.content_count(), 11); + let paras2 = doc2.paragraphs(); + assert_eq!(paras2[0].text(), "Table of Contents"); + } + + #[test] + fn append_documents() { + let mut doc_a = Document::new(); + doc_a.add_paragraph("Paragraph A1"); + doc_a.add_paragraph("Paragraph A2"); + + let mut doc_b = Document::new(); + doc_b.add_paragraph("Paragraph B1"); + doc_b.add_paragraph("Paragraph B2"); + doc_b.add_paragraph("Paragraph B3"); + + assert_eq!(doc_a.content_count(), 2); + doc_a.append(&doc_b); + assert_eq!(doc_a.content_count(), 5); + + let paras = doc_a.paragraphs(); + assert_eq!(paras[0].text(), "Paragraph A1"); + assert_eq!(paras[1].text(), "Paragraph A2"); + assert_eq!(paras[2].text(), "Paragraph B1"); + assert_eq!(paras[3].text(), "Paragraph B2"); + assert_eq!(paras[4].text(), "Paragraph B3"); + + // Verify round-trip + let bytes = doc_a.to_bytes().expect("serialize"); + let reopened = Document::from_bytes(&bytes).expect("open"); + assert_eq!(reopened.content_count(), 5); + } + + #[test] + fn append_with_section_break() { + let mut doc_a = Document::new(); + doc_a.add_paragraph("A1"); + + let mut doc_b = Document::new(); + doc_b.add_paragraph("B1"); + + doc_a.append_with_break(&doc_b, crate::SectionBreak::Continuous); + // 1 original + 1 section break paragraph + 1 merged = 3 + assert_eq!(doc_a.content_count(), 3); + } + + #[test] + fn insert_document_at_index() { + let mut doc_a = Document::new(); + doc_a.add_paragraph("First"); + doc_a.add_paragraph("Last"); + + let mut doc_b = Document::new(); + doc_b.add_paragraph("Middle 1"); + doc_b.add_paragraph("Middle 2"); + + doc_a.insert_document(1, &doc_b); + assert_eq!(doc_a.content_count(), 4); + + let paras = doc_a.paragraphs(); + assert_eq!(paras[0].text(), "First"); + assert_eq!(paras[1].text(), "Middle 1"); + assert_eq!(paras[2].text(), "Middle 2"); + assert_eq!(paras[3].text(), "Last"); + } + + #[test] + fn merge_deduplicates_styles() { + let mut doc_a = Document::new(); + doc_a.add_paragraph("A").style("Heading1"); + + let mut doc_b = Document::new(); + doc_b.add_paragraph("B").style("Heading1"); + doc_b.add_style( + crate::style::StyleBuilder::paragraph("CustomB", "Custom B").based_on("Normal"), + ); + doc_b.add_paragraph("C").style("CustomB"); + + let styles_before = doc_a.styles.styles.len(); + doc_a.append(&doc_b); + let styles_after = doc_a.styles.styles.len(); + + // Heading1 already existed, so only CustomB should be added + assert_eq!(styles_after, styles_before + 1); + } + + #[test] + fn headings_and_outline() { + let mut doc = Document::new(); + doc.add_paragraph("Intro"); + doc.add_paragraph("Chapter 1").style("Heading1"); + doc.add_paragraph("Section 1.1").style("Heading2"); + doc.add_paragraph("Section 1.2").style("Heading2"); + doc.add_paragraph("Chapter 2").style("Heading1"); + doc.add_paragraph("Section 2.1").style("Heading2"); + doc.add_paragraph("Sub 2.1.1").style("Heading3"); + + let headings = doc.headings(); + assert_eq!(headings.len(), 6); + assert_eq!(headings[0], (1, "Chapter 1".to_string())); + assert_eq!(headings[1], (2, "Section 1.1".to_string())); + assert_eq!(headings[5], (3, "Sub 2.1.1".to_string())); + + let outline = doc.document_outline(); + assert_eq!(outline.len(), 2); // Two h1 nodes + assert_eq!(outline[0].text, "Chapter 1"); + assert_eq!(outline[0].children.len(), 2); // 1.1 and 1.2 + assert_eq!(outline[1].text, "Chapter 2"); + assert_eq!(outline[1].children.len(), 1); // 2.1 + assert_eq!(outline[1].children[0].children.len(), 1); // 2.1.1 + } + + #[test] + fn word_count_basic() { + let mut doc = Document::new(); + doc.add_paragraph("Hello world"); + doc.add_paragraph("Three more words"); + assert_eq!(doc.word_count(), 5); + } + + #[test] + fn audit_accessibility_missing_metadata() { + let doc = Document::new(); + let issues = doc.audit_accessibility(); + // New document has no title or author + assert!(issues.iter().any(|i| i.message.contains("no title"))); + assert!(issues.iter().any(|i| i.message.contains("no author"))); + } + + #[test] + fn audit_heading_level_gap() { + let mut doc = Document::new(); + doc.set_title("Test"); + doc.set_author("Test"); + doc.add_paragraph("Ch 1").style("Heading1"); + doc.add_paragraph("Skip to 3").style("Heading3"); + + let issues = doc.audit_accessibility(); + assert!( + issues + .iter() + .any(|i| i.message.contains("Heading level gap")) + ); + } + + #[test] + fn links_returns_empty_for_no_hyperlinks() { + let mut doc = Document::new(); + doc.add_paragraph("No links here."); + assert!(doc.links().is_empty()); + } + + #[test] + fn images_returns_empty_for_text_only() { + let mut doc = Document::new(); + doc.add_paragraph("Just text."); + assert!(doc.images().is_empty()); + } +} diff --git a/vendor/rdocx/src/error.rs b/vendor/rdocx/src/error.rs new file mode 100644 index 00000000..5cf397e4 --- /dev/null +++ b/vendor/rdocx/src/error.rs @@ -0,0 +1,26 @@ +//! Error types for the rdocx high-level API. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("OPC package error: {0}")] + Opc(#[from] rdocx_opc::OpcError), + + #[error("OXML parsing error: {0}")] + Oxml(#[from] rdocx_oxml::OxmlError), + + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + #[error("layout error: {0}")] + Layout(#[from] rdocx_layout::LayoutError), + + #[error("document has no main document part")] + NoDocumentPart, + + #[error("{0}")] + Other(String), +} + +pub type Result = std::result::Result; diff --git a/vendor/rdocx/src/length.rs b/vendor/rdocx/src/length.rs new file mode 100644 index 00000000..c9139498 --- /dev/null +++ b/vendor/rdocx/src/length.rs @@ -0,0 +1,104 @@ +//! Length unit type with convenient constructors. + +use rdocx_oxml::units::{Emu, HalfPoint, Twips}; + +/// A length measurement that can be expressed in various units. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Length { + emu: i64, +} + +impl Length { + /// Create a length from inches. + pub fn inches(val: f64) -> Self { + Length { + emu: (val * 914400.0) as i64, + } + } + + /// Create a length from centimeters. + pub fn cm(val: f64) -> Self { + Length { + emu: (val * 360000.0) as i64, + } + } + + /// Create a length from points. + pub fn pt(val: f64) -> Self { + Length { + emu: (val * 12700.0) as i64, + } + } + + /// Create a length from EMUs. + pub fn emu(val: i64) -> Self { + Length { emu: val } + } + + /// Create a length from twips. + pub fn twips(val: i32) -> Self { + Length { + emu: val as i64 * 635, + } + } + + /// Get the value in inches. + pub fn to_inches(self) -> f64 { + self.emu as f64 / 914400.0 + } + + /// Get the value in centimeters. + pub fn to_cm(self) -> f64 { + self.emu as f64 / 360000.0 + } + + /// Get the value in points. + pub fn to_pt(self) -> f64 { + self.emu as f64 / 12700.0 + } + + /// Get the value in EMUs. + pub fn to_emu(self) -> i64 { + self.emu + } + + /// Get the value in twips. + pub fn to_twips(self) -> i32 { + (self.emu / 635) as i32 + } + + /// Convert to the OXML Twips type. + pub fn as_twips(self) -> Twips { + Twips(self.to_twips()) + } + + /// Convert to the OXML Emu type. + pub fn as_emu(self) -> Emu { + Emu(self.emu) + } + + /// Convert to the OXML HalfPoint type (for font sizes). + pub fn as_half_points(self) -> HalfPoint { + HalfPoint((self.to_pt() * 2.0) as u32) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn length_conversions() { + let one_inch = Length::inches(1.0); + assert!((one_inch.to_inches() - 1.0).abs() < 0.001); + assert!((one_inch.to_cm() - 2.54).abs() < 0.01); + assert!((one_inch.to_pt() - 72.0).abs() < 0.01); + assert_eq!(one_inch.to_twips(), 1440); + } + + #[test] + fn length_from_pt() { + let twelve_pt = Length::pt(12.0); + assert_eq!(twelve_pt.as_half_points().0, 24); + } +} diff --git a/vendor/rdocx/src/lib.rs b/vendor/rdocx/src/lib.rs new file mode 100644 index 00000000..2923f44e --- /dev/null +++ b/vendor/rdocx/src/lib.rs @@ -0,0 +1,40 @@ +//! High-level API for reading, writing, and converting DOCX documents. +//! +//! Provides a Python-docx-like interface for working with Word documents. +//! +//! # Examples +//! +//! ```no_run +//! use rdocx::Document; +//! +//! // Create a new document +//! let mut doc = Document::new(); +//! doc.add_paragraph("Hello, World!"); +//! doc.save("output.docx").unwrap(); +//! +//! // Open an existing document +//! let doc = Document::open("existing.docx").unwrap(); +//! for para in doc.paragraphs() { +//! println!("{}", para.text()); +//! } +//! ``` + +#![allow(clippy::too_many_arguments)] + +mod document; +mod error; +mod length; +pub mod paragraph; +pub mod run; +pub mod style; +pub mod table; + +pub use document::{AccessibilityIssue, Document, ImageInfo, IssueSeverity, LinkInfo, OutlineNode}; +pub use error::{Error, Result}; +pub use length::Length; +pub use paragraph::{ + Alignment, BorderStyle, Paragraph, ParagraphRef, SectionBreak, TabAlignment, TabLeader, +}; +pub use run::{Run, RunRef, UnderlineStyle}; +pub use style::{Style, StyleBuilder}; +pub use table::{Cell, CellRef, Row, RowRef, Table, TableRef, VerticalAlignment}; diff --git a/vendor/rdocx/src/paragraph.rs b/vendor/rdocx/src/paragraph.rs new file mode 100644 index 00000000..74d45aff --- /dev/null +++ b/vendor/rdocx/src/paragraph.rs @@ -0,0 +1,451 @@ +//! Paragraph — a block-level container for runs of text. + +use rdocx_oxml::borders::{CT_BorderEdge, CT_PBdr, CT_TabStop, CT_Tabs}; +use rdocx_oxml::document::CT_SectPr; +use rdocx_oxml::properties::{CT_PPr, CT_Shd}; +use rdocx_oxml::shared::{ + ST_Border, ST_Jc, ST_PageOrientation, ST_SectionType, ST_TabJc, ST_TabLeader, +}; +use rdocx_oxml::text::{CT_P, CT_R}; +use rdocx_oxml::units::Twips; + +use crate::Length; +use crate::run::{Run, RunRef}; + +/// Paragraph alignment options. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Alignment { + Left, + Center, + Right, + Justify, +} + +impl Alignment { + fn to_st_jc(self) -> ST_Jc { + match self { + Alignment::Left => ST_Jc::Left, + Alignment::Center => ST_Jc::Center, + Alignment::Right => ST_Jc::Right, + Alignment::Justify => ST_Jc::Both, + } + } + + fn from_st_jc(jc: ST_Jc) -> Self { + match jc { + ST_Jc::Center => Alignment::Center, + ST_Jc::Right | ST_Jc::End => Alignment::Right, + ST_Jc::Both | ST_Jc::Distribute => Alignment::Justify, + _ => Alignment::Left, + } + } +} + +/// Border style for paragraph borders. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BorderStyle { + None, + Single, + Thick, + Double, + Dotted, + Dashed, + DotDash, + Wave, +} + +impl BorderStyle { + /// Convert to the OXML ST_Border type (public for table module). + pub(crate) fn to_st_border(self) -> ST_Border { + self.to_st() + } + + fn to_st(self) -> ST_Border { + match self { + Self::None => ST_Border::None, + Self::Single => ST_Border::Single, + Self::Thick => ST_Border::Thick, + Self::Double => ST_Border::Double, + Self::Dotted => ST_Border::Dotted, + Self::Dashed => ST_Border::Dashed, + Self::DotDash => ST_Border::DotDash, + Self::Wave => ST_Border::Wave, + } + } +} + +/// Tab stop alignment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TabAlignment { + Left, + Center, + Right, + Decimal, +} + +impl TabAlignment { + fn to_st(self) -> ST_TabJc { + match self { + Self::Left => ST_TabJc::Left, + Self::Center => ST_TabJc::Center, + Self::Right => ST_TabJc::Right, + Self::Decimal => ST_TabJc::Decimal, + } + } +} + +/// Tab leader character. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TabLeader { + None, + Dot, + Hyphen, + Underscore, +} + +impl TabLeader { + fn to_st(self) -> ST_TabLeader { + match self { + Self::None => ST_TabLeader::None, + Self::Dot => ST_TabLeader::Dot, + Self::Hyphen => ST_TabLeader::Hyphen, + Self::Underscore => ST_TabLeader::Underscore, + } + } +} + +/// A mutable reference to a paragraph in a document. +pub struct Paragraph<'a> { + pub(crate) inner: &'a mut CT_P, +} + +impl<'a> Paragraph<'a> { + /// Get the combined text of all runs. + pub fn text(&self) -> String { + self.inner.text() + } + + /// Add a run with the given text and return a mutable reference for chaining. + pub fn add_run(&mut self, text: &str) -> Run<'_> { + self.inner.runs.push(CT_R::new(text)); + Run { + inner: self.inner.runs.last_mut().unwrap(), + } + } + + /// Get an iterator over immutable run references. + pub fn runs(&self) -> impl Iterator> { + self.inner.runs.iter().map(|r| RunRef { inner: r }) + } + + /// Set the paragraph alignment. + pub fn alignment(mut self, align: Alignment) -> Self { + self.ensure_ppr().jc = Some(align.to_st_jc()); + self + } + + /// Set the paragraph style by ID. + pub fn style(mut self, style_id: &str) -> Self { + self.ensure_ppr().style_id = Some(style_id.to_string()); + self + } + + /// Set space before the paragraph. + pub fn space_before(mut self, length: Length) -> Self { + self.ensure_ppr().space_before = Some(length.as_twips()); + self + } + + /// Set space after the paragraph. + pub fn space_after(mut self, length: Length) -> Self { + self.ensure_ppr().space_after = Some(length.as_twips()); + self + } + + /// Set left indentation. + pub fn indent_left(mut self, length: Length) -> Self { + self.ensure_ppr().ind_left = Some(length.as_twips()); + self + } + + /// Set right indentation. + pub fn indent_right(mut self, length: Length) -> Self { + self.ensure_ppr().ind_right = Some(length.as_twips()); + self + } + + /// Set first line indent. + pub fn first_line_indent(mut self, length: Length) -> Self { + self.ensure_ppr().ind_first_line = Some(length.as_twips()); + self + } + + /// Set hanging indent. + pub fn hanging_indent(mut self, length: Length) -> Self { + self.ensure_ppr().ind_hanging = Some(length.as_twips()); + self + } + + /// Set keep with next paragraph. + pub fn keep_with_next(mut self, val: bool) -> Self { + self.ensure_ppr().keep_next = Some(val); + self + } + + /// Set keep lines together. + pub fn keep_together(mut self, val: bool) -> Self { + self.ensure_ppr().keep_lines = Some(val); + self + } + + /// Set page break before. + pub fn page_break_before(mut self, val: bool) -> Self { + self.ensure_ppr().page_break_before = Some(val); + self + } + + /// Set widow/orphan control. + pub fn widow_control(mut self, val: bool) -> Self { + self.ensure_ppr().widow_control = Some(val); + self + } + + /// Set line spacing in points with "exact" rule. + pub fn line_spacing(mut self, pt: f64) -> Self { + let ppr = self.ensure_ppr(); + ppr.line_spacing = Some(Twips::from_pt(pt)); + ppr.line_rule = Some("exact".to_string()); + self + } + + /// Set line spacing with a multiplier (1.0 = single, 1.5, 2.0 = double, etc.). + pub fn line_spacing_multiple(mut self, multiple: f64) -> Self { + let ppr = self.ensure_ppr(); + // In "auto" mode, line spacing is in 240ths of a line (240 = single) + ppr.line_spacing = Some(Twips((multiple * 240.0) as i32)); + ppr.line_rule = Some("auto".to_string()); + self + } + + /// Set a background/shading fill color (hex string, e.g. "FFFF00"). + pub fn shading(mut self, fill_color: &str) -> Self { + self.ensure_ppr().shading = Some(CT_Shd { + val: "clear".to_string(), + color: Some("auto".to_string()), + fill: Some(fill_color.to_string()), + }); + self + } + + /// Add a border to all sides. + pub fn border_all(mut self, style: BorderStyle, size_eighths_pt: u32, color: &str) -> Self { + let edge = CT_BorderEdge { + val: style.to_st(), + sz: Some(size_eighths_pt), + space: Some(1), + color: Some(color.to_string()), + }; + self.ensure_ppr().borders = Some(CT_PBdr { + top: Some(edge.clone()), + bottom: Some(edge.clone()), + left: Some(edge.clone()), + right: Some(edge), + between: None, + bar: None, + }); + self + } + + /// Add a bottom border only. + pub fn border_bottom(mut self, style: BorderStyle, size_eighths_pt: u32, color: &str) -> Self { + let edge = CT_BorderEdge { + val: style.to_st(), + sz: Some(size_eighths_pt), + space: Some(1), + color: Some(color.to_string()), + }; + let borders = self + .ensure_ppr() + .borders + .get_or_insert_with(CT_PBdr::default); + borders.bottom = Some(edge); + self + } + + /// Add a tab stop. + pub fn add_tab_stop(mut self, alignment: TabAlignment, position: Length) -> Self { + let tabs = self + .ensure_ppr() + .tabs + .get_or_insert_with(|| CT_Tabs { tabs: Vec::new() }); + tabs.tabs.push(CT_TabStop { + val: alignment.to_st(), + pos: position.as_twips(), + leader: None, + }); + self + } + + /// Add a tab stop with a leader character. + pub fn add_tab_stop_with_leader( + mut self, + alignment: TabAlignment, + position: Length, + leader: TabLeader, + ) -> Self { + let tabs = self + .ensure_ppr() + .tabs + .get_or_insert_with(|| CT_Tabs { tabs: Vec::new() }); + tabs.tabs.push(CT_TabStop { + val: alignment.to_st(), + pos: position.as_twips(), + leader: Some(leader.to_st()), + }); + self + } + + /// Set outline level (0–8, used for TOC generation). + pub fn outline_level(mut self, level: u32) -> Self { + self.ensure_ppr().outline_lvl = Some(level); + self + } + + /// Add a section break after this paragraph. + /// + /// This creates a `` inside the paragraph's properties, + /// ending the current section at this paragraph. + pub fn section_break(mut self, break_type: SectionBreak) -> Self { + let sect = self.ensure_sect_pr(); + sect.section_type = Some(break_type.to_st()); + self + } + + /// Set the section ending at this paragraph to landscape orientation. + /// + /// Sets page dimensions to 11" x 8.5" (US Letter landscape). + /// Must be combined with `section_break()` to create a section break. + pub fn section_landscape(mut self) -> Self { + let sect = self.ensure_sect_pr(); + sect.orientation = Some(ST_PageOrientation::Landscape); + sect.page_width = Some(Twips(15840)); // 11" + sect.page_height = Some(Twips(12240)); // 8.5" + self + } + + /// Set the section ending at this paragraph to portrait orientation. + /// + /// Sets page dimensions to 8.5" x 11" (US Letter portrait). + /// Must be combined with `section_break()` to create a section break. + pub fn section_portrait(mut self) -> Self { + let sect = self.ensure_sect_pr(); + sect.orientation = Some(ST_PageOrientation::Portrait); + sect.page_width = Some(Twips(12240)); // 8.5" + sect.page_height = Some(Twips(15840)); // 11" + self + } + + /// Set custom page dimensions for the section ending at this paragraph. + pub fn section_page_size(mut self, width: crate::Length, height: crate::Length) -> Self { + let sect = self.ensure_sect_pr(); + sect.page_width = Some(width.as_twips()); + sect.page_height = Some(height.as_twips()); + self + } + + fn ensure_ppr(&mut self) -> &mut CT_PPr { + self.inner.properties.get_or_insert_with(CT_PPr::default) + } + + fn ensure_sect_pr(&mut self) -> &mut CT_SectPr { + let ppr = self.ensure_ppr(); + ppr.sect_pr.get_or_insert_with(CT_SectPr::default_letter) + } +} + +/// Section break type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SectionBreak { + /// Start a new section on the next page. + NextPage, + /// Start a new section on the same page (continuous). + Continuous, + /// Start a new section on the next even-numbered page. + EvenPage, + /// Start a new section on the next odd-numbered page. + OddPage, +} + +impl SectionBreak { + fn to_st(self) -> ST_SectionType { + match self { + SectionBreak::NextPage => ST_SectionType::NextPage, + SectionBreak::Continuous => ST_SectionType::Continuous, + SectionBreak::EvenPage => ST_SectionType::EvenPage, + SectionBreak::OddPage => ST_SectionType::OddPage, + } + } +} + +/// An immutable reference to a paragraph. +pub struct ParagraphRef<'a> { + pub(crate) inner: &'a CT_P, +} + +impl<'a> ParagraphRef<'a> { + /// Get the combined text of all runs. + pub fn text(&self) -> String { + self.inner.text() + } + + /// Get the paragraph style ID, if set. + pub fn style_id(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|ppr| ppr.style_id.as_deref()) + } + + /// Get the alignment, if set. + pub fn alignment(&self) -> Option { + self.inner + .properties + .as_ref() + .and_then(|ppr| ppr.jc) + .map(Alignment::from_st_jc) + } + + /// Get an iterator over immutable run references. + pub fn runs(&self) -> impl Iterator> { + self.inner.runs.iter().map(|r| RunRef { inner: r }) + } + + /// Check if paragraph has borders. + pub fn has_borders(&self) -> bool { + self.inner + .properties + .as_ref() + .and_then(|ppr| ppr.borders.as_ref()) + .map(|b| !b.is_empty()) + .unwrap_or(false) + } + + /// Get the number of tab stops defined. + pub fn tab_stop_count(&self) -> usize { + self.inner + .properties + .as_ref() + .and_then(|ppr| ppr.tabs.as_ref()) + .map(|t| t.tabs.len()) + .unwrap_or(0) + } + + /// Get the shading fill color, if set. + pub fn shading_fill(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|ppr| ppr.shading.as_ref()) + .and_then(|shd| shd.fill.as_deref()) + } +} diff --git a/vendor/rdocx/src/run.rs b/vendor/rdocx/src/run.rs new file mode 100644 index 00000000..acb446ec --- /dev/null +++ b/vendor/rdocx/src/run.rs @@ -0,0 +1,284 @@ +//! Run — a contiguous stretch of text with uniform formatting. + +use rdocx_oxml::properties::{CT_RPr, CT_Shd}; +use rdocx_oxml::shared::ST_Underline; +use rdocx_oxml::text::{CT_R, CT_Text, RunContent}; +use rdocx_oxml::units::{HalfPoint, Twips}; + +use crate::Length; + +/// Underline style for runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnderlineStyle { + None, + Single, + Double, + Thick, + Dotted, + Dash, + Wave, + Words, +} + +impl UnderlineStyle { + fn to_st(self) -> ST_Underline { + match self { + Self::None => ST_Underline::None, + Self::Single => ST_Underline::Single, + Self::Double => ST_Underline::Double, + Self::Thick => ST_Underline::Thick, + Self::Dotted => ST_Underline::Dotted, + Self::Dash => ST_Underline::Dash, + Self::Wave => ST_Underline::Wave, + Self::Words => ST_Underline::Words, + } + } +} + +/// A run of text within a paragraph. +/// +/// All text in a run shares the same formatting (font, size, bold, etc.). +pub struct Run<'a> { + pub(crate) inner: &'a mut CT_R, +} + +impl<'a> Run<'a> { + /// Get the text content of this run. + pub fn text(&self) -> String { + self.inner.text() + } + + /// Set the text content, replacing all existing content. + pub fn set_text(&mut self, text: &str) { + self.inner.content = vec![RunContent::Text(CT_Text::new(text))]; + } + + /// Add text to this run. + pub fn add_text(&mut self, text: &str) { + self.inner + .content + .push(RunContent::Text(CT_Text::new(text))); + } + + /// Set bold formatting. + pub fn bold(mut self, val: bool) -> Self { + let rpr = self.ensure_rpr(); + rpr.bold = Some(val); + rpr.bold_cs = Some(val); + self + } + + /// Set italic formatting. + pub fn italic(mut self, val: bool) -> Self { + let rpr = self.ensure_rpr(); + rpr.italic = Some(val); + rpr.italic_cs = Some(val); + self + } + + /// Set underline formatting (simple on/off). + pub fn underline(mut self, val: bool) -> Self { + self.ensure_rpr().underline = Some(if val { + ST_Underline::Single + } else { + ST_Underline::None + }); + self + } + + /// Set underline with a specific style. + pub fn underline_style(mut self, style: UnderlineStyle) -> Self { + self.ensure_rpr().underline = Some(style.to_st()); + self + } + + /// Set font size in points. + pub fn size(mut self, pt: f64) -> Self { + let hp = HalfPoint::from_pt(pt); + let rpr = self.ensure_rpr(); + rpr.sz = Some(hp); + rpr.sz_cs = Some(hp); + self + } + + /// Set the font name. + pub fn font(mut self, name: &str) -> Self { + let rpr = self.ensure_rpr(); + rpr.font_ascii = Some(name.to_string()); + rpr.font_hansi = Some(name.to_string()); + rpr.font_east_asia = Some(name.to_string()); + rpr.font_cs = Some(name.to_string()); + self + } + + /// Set text color as a hex string (e.g., "FF0000" for red). + pub fn color(mut self, hex: &str) -> Self { + self.ensure_rpr().color = Some(hex.to_string()); + self + } + + /// Set highlight color as a hex fill value. + pub fn highlight(mut self, color: &str) -> Self { + self.ensure_rpr().shading = Some(CT_Shd { + val: "clear".to_string(), + color: Some("auto".to_string()), + fill: Some(color.to_string()), + }); + self + } + + /// Set strikethrough formatting. + pub fn strike(mut self, val: bool) -> Self { + self.ensure_rpr().strike = Some(val); + self + } + + /// Set double strikethrough. + pub fn double_strike(mut self, val: bool) -> Self { + self.ensure_rpr().dstrike = Some(val); + self + } + + /// Set all caps. + pub fn all_caps(mut self, val: bool) -> Self { + self.ensure_rpr().caps = Some(val); + self + } + + /// Set small caps. + pub fn small_caps(mut self, val: bool) -> Self { + self.ensure_rpr().small_caps = Some(val); + self + } + + /// Set superscript. + pub fn superscript(mut self) -> Self { + self.ensure_rpr().vert_align = Some("superscript".to_string()); + self + } + + /// Set subscript. + pub fn subscript(mut self) -> Self { + self.ensure_rpr().vert_align = Some("subscript".to_string()); + self + } + + /// Set character spacing (positive = expanded, negative = condensed). + pub fn character_spacing(mut self, spacing: Length) -> Self { + self.ensure_rpr().spacing = Some(spacing.as_twips()); + self + } + + /// Set character width scale in percent (100 = normal). + pub fn width_scale(mut self, percent: u32) -> Self { + self.ensure_rpr().width_scale = Some(percent); + self + } + + /// Set text position (positive = raised, negative = lowered) in half-points. + pub fn position(mut self, half_points: i32) -> Self { + self.ensure_rpr().position = Some(half_points); + self + } + + /// Set hidden/vanish text. + pub fn hidden(mut self, val: bool) -> Self { + self.ensure_rpr().vanish = Some(val); + self + } + + /// Set the character style by ID. + pub fn style(mut self, style_id: &str) -> Self { + self.ensure_rpr().style_id = Some(style_id.to_string()); + self + } + + fn ensure_rpr(&mut self) -> &mut CT_RPr { + self.inner.properties.get_or_insert_with(CT_RPr::default) + } +} + +/// An immutable reference to a run. +pub struct RunRef<'a> { + pub(crate) inner: &'a CT_R, +} + +impl<'a> RunRef<'a> { + /// Get the text content of this run. + pub fn text(&self) -> String { + self.inner.text() + } + + /// Check if bold. + pub fn is_bold(&self) -> bool { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.bold) + .unwrap_or(false) + } + + /// Check if italic. + pub fn is_italic(&self) -> bool { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.italic) + .unwrap_or(false) + } + + /// Check if strikethrough. + pub fn is_strike(&self) -> bool { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.strike) + .unwrap_or(false) + } + + /// Get font size in points, if set. + pub fn size(&self) -> Option { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.sz) + .map(|hp| hp.to_pt()) + } + + /// Get font name, if set. + pub fn font_name(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.font_ascii.as_deref()) + } + + /// Get text color, if set. + pub fn color(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.color.as_deref()) + } + + /// Get character spacing in twips, if set. + pub fn character_spacing(&self) -> Option { + self.inner.properties.as_ref().and_then(|rpr| rpr.spacing) + } + + /// Get vertical alignment (superscript/subscript), if set. + pub fn vert_align(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.vert_align.as_deref()) + } + + /// Get the character style ID, if set. + pub fn style_id(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|rpr| rpr.style_id.as_deref()) + } +} diff --git a/vendor/rdocx/src/style.rs b/vendor/rdocx/src/style.rs new file mode 100644 index 00000000..f961ed23 --- /dev/null +++ b/vendor/rdocx/src/style.rs @@ -0,0 +1,278 @@ +//! Style access and manipulation for documents. + +use rdocx_oxml::properties::{CT_PPr, CT_RPr}; +use rdocx_oxml::styles::{CT_Style, CT_Styles, StyleType}; + +/// An immutable reference to a style definition. +pub struct Style<'a> { + pub(crate) inner: &'a CT_Style, +} + +impl<'a> Style<'a> { + /// The style ID (used to reference this style). + pub fn style_id(&self) -> &str { + &self.inner.style_id + } + + /// The display name of the style. + pub fn name(&self) -> Option<&str> { + self.inner.name.as_deref() + } + + /// The style ID this style is based on. + pub fn based_on(&self) -> Option<&str> { + self.inner.based_on.as_deref() + } + + /// Whether this is the default style for its type. + pub fn is_default(&self) -> bool { + self.inner.is_default + } +} + +/// Builder for creating a new paragraph style. +pub struct StyleBuilder { + style: CT_Style, +} + +impl StyleBuilder { + /// Create a new paragraph style builder. + pub fn paragraph(style_id: &str, name: &str) -> Self { + StyleBuilder { + style: CT_Style { + style_id: style_id.to_string(), + style_type: StyleType::Paragraph, + name: Some(name.to_string()), + based_on: None, + next_style: None, + is_default: false, + ppr: None, + rpr: None, + }, + } + } + + /// Create a new character style builder. + pub fn character(style_id: &str, name: &str) -> Self { + StyleBuilder { + style: CT_Style { + style_id: style_id.to_string(), + style_type: StyleType::Character, + name: Some(name.to_string()), + based_on: None, + next_style: None, + is_default: false, + ppr: None, + rpr: None, + }, + } + } + + /// Set the parent style this one inherits from. + pub fn based_on(mut self, style_id: &str) -> Self { + self.style.based_on = Some(style_id.to_string()); + self + } + + /// Set the next style (applied to the following paragraph after pressing Enter). + pub fn next_style(mut self, style_id: &str) -> Self { + self.style.next_style = Some(style_id.to_string()); + self + } + + /// Set paragraph properties for this style. + pub fn paragraph_properties(mut self, ppr: CT_PPr) -> Self { + self.style.ppr = Some(ppr); + self + } + + /// Set run properties for this style. + pub fn run_properties(mut self, rpr: CT_RPr) -> Self { + self.style.rpr = Some(rpr); + self + } + + /// Build the style (consumed by Document::add_style). + pub(crate) fn build(self) -> CT_Style { + self.style + } +} + +/// Resolve the effective paragraph properties by walking the style inheritance chain. +pub fn resolve_paragraph_properties(style_id: Option<&str>, styles: &CT_Styles) -> CT_PPr { + let mut effective = CT_PPr::default(); + + // Start from docDefaults + if let Some(ref defaults) = styles.doc_defaults + && let Some(ref ppr) = defaults.ppr + { + effective.merge_from(ppr); + } + + // Walk the basedOn chain + if let Some(sid) = style_id { + let chain = collect_style_chain(sid, styles); + // Apply from most-base to most-derived + for style in chain.iter().rev() { + if let Some(ref ppr) = style.ppr { + effective.merge_from(ppr); + } + } + } else { + // Apply the default paragraph style + if let Some(default_style) = styles.get_default(StyleType::Paragraph) + && let Some(ref ppr) = default_style.ppr + { + effective.merge_from(ppr); + } + } + + effective +} + +/// Resolve the effective run properties by walking the style inheritance chain. +pub fn resolve_run_properties( + para_style_id: Option<&str>, + run_style_id: Option<&str>, + styles: &CT_Styles, +) -> CT_RPr { + let mut effective = CT_RPr::default(); + + // Start from docDefaults + if let Some(ref defaults) = styles.doc_defaults + && let Some(ref rpr) = defaults.rpr + { + effective.merge_from(rpr); + } + + // Apply paragraph style's rpr + let para_sid = para_style_id.or_else(|| { + styles + .get_default(StyleType::Paragraph) + .map(|s| s.style_id.as_str()) + }); + if let Some(sid) = para_sid { + let chain = collect_style_chain(sid, styles); + for style in chain.iter().rev() { + if let Some(ref rpr) = style.rpr { + effective.merge_from(rpr); + } + } + } + + // Apply character style's rpr + if let Some(sid) = run_style_id { + let chain = collect_style_chain(sid, styles); + for style in chain.iter().rev() { + if let Some(ref rpr) = style.rpr { + effective.merge_from(rpr); + } + } + } + + effective +} + +/// Collect the chain of styles from the given style up through basedOn ancestors. +fn collect_style_chain<'a>(style_id: &str, styles: &'a CT_Styles) -> Vec<&'a CT_Style> { + let mut chain = Vec::new(); + let mut current_id = Some(style_id.to_string()); + let mut seen = std::collections::HashSet::new(); + + while let Some(ref sid) = current_id { + if !seen.insert(sid.clone()) { + break; // Prevent cycles + } + if let Some(style) = styles.get_by_id(sid) { + chain.push(style); + current_id = style.based_on.clone(); + } else { + break; + } + } + + chain +} + +#[cfg(test)] +mod tests { + use super::*; + use rdocx_oxml::units::{HalfPoint, Twips}; + + fn test_styles() -> CT_Styles { + let mut styles = CT_Styles::new_default(); + + // Add a Heading2 based on Heading1 + styles.styles.push(CT_Style { + style_id: "Heading2".to_string(), + style_type: StyleType::Paragraph, + name: Some("heading 2".to_string()), + based_on: Some("Heading1".to_string()), + next_style: Some("Normal".to_string()), + is_default: false, + ppr: Some(CT_PPr { + space_before: Some(Twips(40)), // Override Heading1's 240 + ..Default::default() + }), + rpr: Some(CT_RPr { + sz: Some(HalfPoint(26)), // Override Heading1's 32 + color: Some("2E74B5".to_string()), + ..Default::default() + }), + }); + + styles + } + + #[test] + fn resolve_normal_paragraph() { + let styles = test_styles(); + let ppr = resolve_paragraph_properties(Some("Normal"), &styles); + // Should have docDefaults' spacing + assert_eq!(ppr.space_after, Some(Twips(160))); + } + + #[test] + fn resolve_heading1() { + let styles = test_styles(); + let ppr = resolve_paragraph_properties(Some("Heading1"), &styles); + // keepNext from Heading1 + assert_eq!(ppr.keep_next, Some(true)); + // spaceBefore from Heading1 (overrides docDefaults which has none) + assert_eq!(ppr.space_before, Some(Twips(240))); + // spaceAfter from Heading1 overrides docDefaults + assert_eq!(ppr.space_after, Some(Twips(0))); + } + + #[test] + fn resolve_heading2_inherits_heading1() { + let styles = test_styles(); + let ppr = resolve_paragraph_properties(Some("Heading2"), &styles); + // keepNext inherited from Heading1 + assert_eq!(ppr.keep_next, Some(true)); + // spaceBefore overridden by Heading2 + assert_eq!(ppr.space_before, Some(Twips(40))); + } + + #[test] + fn resolve_heading2_rpr() { + let styles = test_styles(); + let rpr = resolve_run_properties(Some("Heading2"), None, &styles); + // Font from docDefaults + assert_eq!(rpr.font_ascii, Some("Calibri".to_string())); + // Size overridden by Heading2 (not Heading1's 32) + assert_eq!(rpr.sz, Some(HalfPoint(26))); + // Bold inherited from Heading1 + assert_eq!(rpr.bold, Some(true)); + // Color from Heading2 + assert_eq!(rpr.color, Some("2E74B5".to_string())); + } + + #[test] + fn resolve_default_when_no_style() { + let styles = test_styles(); + let ppr = resolve_paragraph_properties(None, &styles); + // Should get docDefaults + assert_eq!(ppr.space_after, Some(Twips(160))); + } +} diff --git a/vendor/rdocx/src/table.rs b/vendor/rdocx/src/table.rs new file mode 100644 index 00000000..f3e32b0a --- /dev/null +++ b/vendor/rdocx/src/table.rs @@ -0,0 +1,469 @@ +//! Table — a block-level container for rows and cells of content. + +use rdocx_oxml::borders::CT_BorderEdge; +use rdocx_oxml::properties::CT_Shd; +use rdocx_oxml::shared::ST_Jc; +use rdocx_oxml::table::{ + CT_Row, CT_Tbl, CT_TblBorders, CT_TblCellMar, CT_TblPr, CT_TblWidth, CT_Tc, CT_TcPr, CT_TrPr, + ST_VerticalJc, VMerge, +}; +use rdocx_oxml::text::CT_P; + +use crate::Length; +use crate::paragraph::{Paragraph, ParagraphRef}; + +/// Vertical alignment within a table cell. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VerticalAlignment { + Top, + Center, + Bottom, +} + +impl VerticalAlignment { + fn to_st(self) -> ST_VerticalJc { + match self { + Self::Top => ST_VerticalJc::Top, + Self::Center => ST_VerticalJc::Center, + Self::Bottom => ST_VerticalJc::Bottom, + } + } + + fn from_st(st: ST_VerticalJc) -> Self { + match st { + ST_VerticalJc::Top => Self::Top, + ST_VerticalJc::Center => Self::Center, + ST_VerticalJc::Bottom => Self::Bottom, + } + } +} + +// ---- Mutable Table ---- + +/// A mutable reference to a table in a document. +pub struct Table<'a> { + pub(crate) inner: &'a mut CT_Tbl, +} + +impl<'a> Table<'a> { + /// Set the table style by ID. + pub fn style(mut self, style_id: &str) -> Self { + self.ensure_tbl_pr().style_id = Some(style_id.to_string()); + self + } + + /// Set the table width in twips (dxa). + pub fn width(mut self, length: Length) -> Self { + self.ensure_tbl_pr().width = Some(CT_TblWidth::dxa(length.as_twips().0)); + self + } + + /// Set the table width as a percentage (0–100). + pub fn width_pct(mut self, percent: f64) -> Self { + // OOXML uses 50ths of a percent + self.ensure_tbl_pr().width = Some(CT_TblWidth::pct((percent * 50.0) as i32)); + self + } + + /// Set table alignment. + pub fn alignment(mut self, jc: crate::paragraph::Alignment) -> Self { + use crate::paragraph::Alignment; + let st_jc = match jc { + Alignment::Left => ST_Jc::Left, + Alignment::Center => ST_Jc::Center, + Alignment::Right => ST_Jc::Right, + Alignment::Justify => ST_Jc::Both, + }; + self.ensure_tbl_pr().jc = Some(st_jc); + self + } + + /// Set borders on all edges and internal gridlines. + pub fn borders(mut self, style: crate::BorderStyle, size_eighths_pt: u32, color: &str) -> Self { + let edge = CT_BorderEdge { + val: style.to_st_border(), + sz: Some(size_eighths_pt), + space: Some(0), + color: Some(color.to_string()), + }; + self.ensure_tbl_pr().borders = Some(CT_TblBorders { + top: Some(edge.clone()), + bottom: Some(edge.clone()), + left: Some(edge.clone()), + right: Some(edge.clone()), + inside_h: Some(edge.clone()), + inside_v: Some(edge), + }); + self + } + + /// Set default cell margins. + pub fn cell_margins( + mut self, + top: Length, + right: Length, + bottom: Length, + left: Length, + ) -> Self { + self.ensure_tbl_pr().cell_margin = Some(CT_TblCellMar { + top: Some(top.as_twips()), + right: Some(right.as_twips()), + bottom: Some(bottom.as_twips()), + left: Some(left.as_twips()), + }); + self + } + + /// Set the table layout to fixed or auto. + pub fn layout_fixed(mut self) -> Self { + self.ensure_tbl_pr().layout = Some("fixed".to_string()); + self + } + + /// Get the number of rows. + pub fn row_count(&self) -> usize { + self.inner.rows.len() + } + + /// Get a mutable reference to a row by index. + pub fn row(&mut self, index: usize) -> Option> { + self.inner.rows.get_mut(index).map(|r| Row { inner: r }) + } + + /// Get a mutable reference to a cell at (row, col). + pub fn cell(&mut self, row: usize, col: usize) -> Option> { + self.inner + .rows + .get_mut(row) + .and_then(|r| r.cells.get_mut(col)) + .map(|c| Cell { inner: c }) + } + + fn ensure_tbl_pr(&mut self) -> &mut CT_TblPr { + self.inner.properties.get_or_insert_with(CT_TblPr::default) + } +} + +// ---- Mutable Row ---- + +/// A mutable reference to a table row. +pub struct Row<'a> { + pub(crate) inner: &'a mut CT_Row, +} + +impl<'a> Row<'a> { + /// Set the row height. + pub fn height(mut self, length: Length) -> Self { + let pr = self.ensure_tr_pr(); + pr.height = Some(length.as_twips()); + pr.height_rule = Some("atLeast".to_string()); + self + } + + /// Set exact row height. + pub fn height_exact(mut self, length: Length) -> Self { + let pr = self.ensure_tr_pr(); + pr.height = Some(length.as_twips()); + pr.height_rule = Some("exact".to_string()); + self + } + + /// Mark this row as a header row (repeats on each page). + pub fn header(mut self) -> Self { + self.ensure_tr_pr().header = Some(true); + self + } + + /// Prevent this row from splitting across pages. + pub fn cant_split(mut self) -> Self { + self.ensure_tr_pr().cant_split = Some(true); + self + } + + /// Get a mutable reference to a cell by index. + pub fn cell(&mut self, index: usize) -> Option> { + self.inner.cells.get_mut(index).map(|c| Cell { inner: c }) + } + + /// Get the number of cells in this row. + pub fn cell_count(&self) -> usize { + self.inner.cells.len() + } + + fn ensure_tr_pr(&mut self) -> &mut CT_TrPr { + self.inner.properties.get_or_insert_with(CT_TrPr::default) + } +} + +// ---- Mutable Cell ---- + +/// A mutable reference to a table cell. +pub struct Cell<'a> { + pub(crate) inner: &'a mut CT_Tc, +} + +impl<'a> Cell<'a> { + /// Get the combined text of all paragraphs in this cell. + pub fn text(&self) -> String { + self.inner.text() + } + + /// Set the text of the first paragraph (replacing existing content). + pub fn set_text(&mut self, text: &str) { + use rdocx_oxml::table::CellContent; + // Find first paragraph or create one + let first_para = self.inner.content.iter_mut().find_map(|c| { + if let CellContent::Paragraph(p) = c { + Some(p) + } else { + None + } + }); + if let Some(para) = first_para { + para.runs.clear(); + if !text.is_empty() { + para.add_run(text); + } + } else { + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + self.inner.content.insert(0, CellContent::Paragraph(p)); + } + } + + /// Add a paragraph to the cell and return a mutable reference. + pub fn add_paragraph(&mut self, text: &str) -> Paragraph<'_> { + use rdocx_oxml::table::CellContent; + let mut p = CT_P::new(); + if !text.is_empty() { + p.add_run(text); + } + self.inner.content.push(CellContent::Paragraph(p)); + let para = self.inner.content.last_mut().unwrap(); + if let CellContent::Paragraph(p) = para { + Paragraph { inner: p } + } else { + unreachable!() + } + } + + /// Get an iterator over immutable paragraph references. + pub fn paragraphs(&self) -> impl Iterator> { + self.inner + .paragraphs() + .into_iter() + .map(|p| ParagraphRef { inner: p }) + } + + /// Set cell width. + pub fn width(mut self, length: Length) -> Self { + self.ensure_tc_pr().width = Some(CT_TblWidth::dxa(length.as_twips().0)); + self + } + + /// Set cell background shading color. + pub fn shading(mut self, fill_color: &str) -> Self { + self.ensure_tc_pr().shading = Some(CT_Shd { + val: "clear".to_string(), + color: Some("auto".to_string()), + fill: Some(fill_color.to_string()), + }); + self + } + + /// Set vertical alignment within the cell. + pub fn vertical_alignment(mut self, align: VerticalAlignment) -> Self { + self.ensure_tc_pr().v_align = Some(align.to_st()); + self + } + + /// Set horizontal merge (gridSpan). This cell spans `span` columns. + pub fn grid_span(mut self, span: u32) -> Self { + self.ensure_tc_pr().grid_span = Some(span); + self + } + + /// Start a vertical merge group (this cell is the top of the merged range). + pub fn v_merge_restart(mut self) -> Self { + self.ensure_tc_pr().v_merge = Some(VMerge::Restart); + self + } + + /// Continue a vertical merge group (this cell merges with the one above). + pub fn v_merge_continue(mut self) -> Self { + self.ensure_tc_pr().v_merge = Some(VMerge::Continue); + self + } + + /// Set no-wrap for text in this cell. + pub fn no_wrap(mut self) -> Self { + self.ensure_tc_pr().no_wrap = Some(true); + self + } + + /// Add a nested table inside this cell. + pub fn add_table(&mut self, rows: usize, cols: usize) -> Table<'_> { + use rdocx_oxml::table::{ + CT_Row, CT_Tbl, CT_TblGrid, CT_TblGridCol, CT_TblPr, CT_TblWidth, CT_Tc, CellContent, + }; + use rdocx_oxml::units::Twips; + + // Default nested table column width: use equal splits of 4500tw (~3.125") + let col_width = Twips(4500 / cols as i32); + + let grid = CT_TblGrid { + columns: (0..cols) + .map(|_| CT_TblGridCol { width: col_width }) + .collect(), + }; + + let mut tbl = CT_Tbl::new(); + tbl.properties = Some(CT_TblPr { + width: Some(CT_TblWidth::dxa(col_width.0 * cols as i32)), + ..Default::default() + }); + tbl.grid = Some(grid); + + for _ in 0..rows { + let mut row = CT_Row::new(); + for _ in 0..cols { + row.cells.push(CT_Tc::new()); + } + tbl.rows.push(row); + } + + self.inner.content.push(CellContent::Table(tbl)); + match self.inner.content.last_mut().unwrap() { + CellContent::Table(t) => Table { inner: t }, + _ => unreachable!(), + } + } + + fn ensure_tc_pr(&mut self) -> &mut CT_TcPr { + self.inner.properties.get_or_insert_with(CT_TcPr::default) + } +} + +// ---- Immutable references ---- + +/// An immutable reference to a table. +pub struct TableRef<'a> { + pub(crate) inner: &'a CT_Tbl, +} + +impl<'a> TableRef<'a> { + /// Get the number of rows. + pub fn row_count(&self) -> usize { + self.inner.rows.len() + } + + /// Get the number of columns (from the grid definition). + pub fn column_count(&self) -> usize { + self.inner + .grid + .as_ref() + .map(|g| g.columns.len()) + .unwrap_or(0) + } + + /// Get an immutable row reference. + pub fn row(&self, index: usize) -> Option> { + self.inner.rows.get(index).map(|r| RowRef { inner: r }) + } + + /// Get a cell reference at (row, col). + pub fn cell(&self, row: usize, col: usize) -> Option> { + self.inner + .rows + .get(row) + .and_then(|r| r.cells.get(col)) + .map(|c| CellRef { inner: c }) + } + + /// Get the table style ID, if set. + pub fn style_id(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|pr| pr.style_id.as_deref()) + } +} + +/// An immutable reference to a table row. +pub struct RowRef<'a> { + pub(crate) inner: &'a CT_Row, +} + +impl<'a> RowRef<'a> { + /// Get the number of cells. + pub fn cell_count(&self) -> usize { + self.inner.cells.len() + } + + /// Get a cell reference by index. + pub fn cell(&self, index: usize) -> Option> { + self.inner.cells.get(index).map(|c| CellRef { inner: c }) + } + + /// Check if this row is a header row. + pub fn is_header(&self) -> bool { + self.inner + .properties + .as_ref() + .and_then(|pr| pr.header) + .unwrap_or(false) + } +} + +/// An immutable reference to a table cell. +pub struct CellRef<'a> { + pub(crate) inner: &'a CT_Tc, +} + +impl<'a> CellRef<'a> { + /// Get the combined text of all paragraphs. + pub fn text(&self) -> String { + self.inner.text() + } + + /// Get paragraph references. + pub fn paragraphs(&self) -> impl Iterator> { + self.inner + .paragraphs() + .into_iter() + .map(|p| ParagraphRef { inner: p }) + } + + /// Get the grid span, if set. + pub fn grid_span(&self) -> Option { + self.inner.properties.as_ref().and_then(|pr| pr.grid_span) + } + + /// Get the vertical merge state, if set. + pub fn v_merge(&self) -> Option<&VMerge> { + self.inner + .properties + .as_ref() + .and_then(|pr| pr.v_merge.as_ref()) + } + + /// Get the shading fill color, if set. + pub fn shading_fill(&self) -> Option<&str> { + self.inner + .properties + .as_ref() + .and_then(|pr| pr.shading.as_ref()) + .and_then(|shd| shd.fill.as_deref()) + } + + /// Get the vertical alignment, if set. + pub fn vertical_alignment(&self) -> Option { + self.inner + .properties + .as_ref() + .and_then(|pr| pr.v_align) + .map(VerticalAlignment::from_st) + } +} diff --git a/vendor/rdocx/tests/integration_test.rs b/vendor/rdocx/tests/integration_test.rs new file mode 100644 index 00000000..af8df659 --- /dev/null +++ b/vendor/rdocx/tests/integration_test.rs @@ -0,0 +1,1492 @@ +//! Integration tests for rdocx — end-to-end document creation and round-trip. + +use rdocx::Document; +use rdocx::paragraph::Alignment; +use rdocx::table::VerticalAlignment; +use rdocx::{ + BorderStyle, Length, SectionBreak, StyleBuilder, TabAlignment, TabLeader, UnderlineStyle, +}; + +#[test] +fn create_and_round_trip_simple_document() { + let mut doc = Document::new(); + doc.add_paragraph("Hello, World!"); + doc.add_paragraph("This is a test document."); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 2); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Hello, World!"); + assert_eq!(paras[1].text(), "This is a test document."); +} + +#[test] +fn create_and_round_trip_formatted_document() { + let mut doc = Document::new(); + + // Title paragraph + doc.add_paragraph("Document Title") + .style("Heading1") + .alignment(Alignment::Center); + + // Normal paragraph with multiple formatted runs + let mut para = doc.add_paragraph(""); + para.add_run("This is ").font("Arial").size(11.0); + para.add_run("bold").bold(true).font("Arial").size(11.0); + para.add_run(" and this is ").font("Arial").size(11.0); + para.add_run("italic").italic(true).font("Arial").size(11.0); + para.add_run(".").font("Arial").size(11.0); + + // Justified paragraph with indentation + doc.add_paragraph("This paragraph has special formatting.") + .alignment(Alignment::Justify) + .indent_left(Length::inches(0.5)) + .space_before(Length::pt(12.0)) + .space_after(Length::pt(6.0)); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 3); + + // Check title + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Document Title"); + assert_eq!(paras[0].style_id(), Some("Heading1")); + assert_eq!(paras[0].alignment(), Some(Alignment::Center)); + + // Check formatted runs + let runs: Vec<_> = paras[1].runs().collect(); + assert_eq!(runs.len(), 5); + assert!(!runs[0].is_bold()); + assert!(runs[1].is_bold()); + assert!(!runs[2].is_italic()); + assert!(runs[3].is_italic()); + + // Check justified paragraph + assert_eq!(paras[2].alignment(), Some(Alignment::Justify)); +} + +#[test] +fn round_trip_preserves_styles() { + let mut doc = Document::new(); + doc.add_paragraph("Normal paragraph"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + // Should have the default styles + assert!(doc2.style("Normal").is_some()); + assert!(doc2.style("Heading1").is_some()); + + let normal = doc2.style("Normal").unwrap(); + assert!(normal.is_default()); + assert_eq!(normal.name(), Some("Normal")); +} + +#[test] +fn save_and_load_file() { + let dir = std::env::temp_dir(); + let path = dir.join("rdocx_test_output.docx"); + + // Create and save + let mut doc = Document::new(); + doc.add_paragraph("Saved to disk"); + doc.save(&path).unwrap(); + + // Load back + let doc2 = Document::open(&path).unwrap(); + assert_eq!(doc2.paragraph_count(), 1); + assert_eq!(doc2.paragraphs()[0].text(), "Saved to disk"); + + // Clean up + std::fs::remove_file(&path).ok(); +} + +#[test] +fn section_properties_preserved() { + let doc = Document::new(); + let sect = doc.section_properties().unwrap(); + + // Default US Letter + assert_eq!(sect.page_width.unwrap().0, 12240); // 8.5" + assert_eq!(sect.page_height.unwrap().0, 15840); // 11" + assert_eq!(sect.margin_top.unwrap().0, 1440); // 1" + + // Round-trip + let mut doc2 = Document::new(); + let bytes = doc2.to_bytes().unwrap(); + let doc3 = Document::from_bytes(&bytes).unwrap(); + let sect3 = doc3.section_properties().unwrap(); + assert_eq!(sect3.page_width.unwrap().0, 12240); +} + +#[test] +fn empty_document_round_trip() { + let mut doc = Document::new(); + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraph_count(), 0); +} + +#[test] +fn run_color_and_font_round_trip() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("Red text") + .color("FF0000") + .font("Times New Roman") + .size(16.0); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + let runs: Vec<_> = paras[0].runs().collect(); + assert_eq!(runs[0].color(), Some("FF0000")); + assert_eq!(runs[0].font_name(), Some("Times New Roman")); + assert_eq!(runs[0].size(), Some(16.0)); +} + +// ---- Phase 2 Integration Tests ---- + +#[test] +fn paragraph_borders_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Bordered paragraph") + .border_all(BorderStyle::Single, 4, "000000"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + assert!(paras[0].has_borders()); +} + +#[test] +fn paragraph_tab_stops_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Tab text") + .add_tab_stop(TabAlignment::Right, Length::inches(6.0)) + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(6.5), TabLeader::Dot); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + assert_eq!(paras[0].tab_stop_count(), 2); +} + +#[test] +fn paragraph_shading_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Highlighted paragraph").shading("FFFF00"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + assert_eq!(paras[0].shading_fill(), Some("FFFF00")); +} + +#[test] +fn paragraph_spacing_and_indent_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Indented text") + .indent_left(Length::inches(1.0)) + .indent_right(Length::inches(0.5)) + .first_line_indent(Length::inches(0.25)) + .space_before(Length::pt(12.0)) + .space_after(Length::pt(6.0)) + .line_spacing_multiple(1.5); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + // Verify it round-trips without error + assert_eq!(doc2.paragraph_count(), 1); + assert_eq!(doc2.paragraphs()[0].text(), "Indented text"); +} + +#[test] +fn paragraph_pagination_controls_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Keep with next") + .keep_with_next(true) + .keep_together(true) + .widow_control(true); + doc.add_paragraph("Page break before") + .page_break_before(true); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 2); +} + +#[test] +fn run_underline_styles_round_trip() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("Simple underline").underline(true); + para.add_run("Wave underline") + .underline_style(UnderlineStyle::Wave); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Simple underlineWave underline"); +} + +#[test] +fn run_advanced_formatting_round_trip() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("Strike").strike(true); + para.add_run("DStrike").double_strike(true); + para.add_run("CAPS").all_caps(true); + para.add_run("SmallCaps").small_caps(true); + para.add_run("Super").superscript(); + para.add_run("Sub").subscript(); + para.add_run("Hidden").hidden(true); + para.add_run("Spaced").character_spacing(Length::pt(2.0)); + para.add_run("Wide").width_scale(150); + para.add_run("Raised").position(6); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + let runs: Vec<_> = paras[0].runs().collect(); + assert_eq!(runs.len(), 10); + assert!(runs[0].is_strike()); + assert_eq!(runs[4].vert_align(), Some("superscript")); + assert_eq!(runs[5].vert_align(), Some("subscript")); +} + +#[test] +fn run_style_assignment_round_trip() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("Styled run").style("Heading1Char"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let paras = doc2.paragraphs(); + let runs: Vec<_> = paras[0].runs().collect(); + assert_eq!(runs[0].style_id(), Some("Heading1Char")); +} + +#[test] +fn custom_style_round_trip() { + let mut doc = Document::new(); + + doc.add_style( + StyleBuilder::paragraph("CustomHeading", "Custom Heading") + .based_on("Heading1") + .next_style("Normal"), + ); + + doc.add_style(StyleBuilder::character("Emphasis", "Emphasis Style")); + + doc.add_paragraph("Custom styled").style("CustomHeading"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let s = doc2.style("CustomHeading").unwrap(); + assert_eq!(s.name(), Some("Custom Heading")); + assert_eq!(s.based_on(), Some("Heading1")); + + assert!(doc2.style("Emphasis").is_some()); + + let paras = doc2.paragraphs(); + assert_eq!(paras[0].style_id(), Some("CustomHeading")); +} + +#[test] +fn style_inheritance_resolution() { + let doc = Document::new(); + + // Heading1's rpr should have bold from the style definition + let rpr = doc.resolve_run_properties(Some("Heading1"), None); + assert_eq!(rpr.bold, Some(true)); + // Font inherited from docDefaults + assert_eq!(rpr.font_ascii, Some("Calibri".to_string())); + + // Normal paragraph should get docDefaults spacing + let ppr = doc.resolve_paragraph_properties(Some("Normal")); + assert!(ppr.space_after.is_some()); +} + +#[test] +fn section_landscape_round_trip() { + let mut doc = Document::new(); + doc.set_landscape(); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let sect = doc2.section_properties().unwrap(); + assert!(sect.page_width.unwrap().0 > sect.page_height.unwrap().0); +} + +#[test] +fn section_margins_round_trip() { + let mut doc = Document::new(); + doc.set_margins( + Length::inches(0.5), + Length::inches(0.75), + Length::inches(0.5), + Length::inches(0.75), + ); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let sect = doc2.section_properties().unwrap(); + assert_eq!(sect.margin_top.unwrap().0, 720); + assert_eq!(sect.margin_right.unwrap().0, 1080); + assert_eq!(sect.margin_bottom.unwrap().0, 720); + assert_eq!(sect.margin_left.unwrap().0, 1080); +} + +#[test] +fn section_columns_round_trip() { + let mut doc = Document::new(); + doc.set_columns(3, Length::inches(0.25)); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let sect = doc2.section_properties().unwrap(); + let cols = sect.columns.as_ref().unwrap(); + assert_eq!(cols.num, Some(3)); + assert_eq!(cols.equal_width, Some(true)); +} + +#[test] +fn section_a4_page_size() { + let mut doc = Document::new(); + doc.set_page_size(Length::cm(21.0), Length::cm(29.7)); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let sect = doc2.section_properties().unwrap(); + let w = sect.page_width.unwrap().0; + let h = sect.page_height.unwrap().0; + // A4 dimensions: 11906tw x 16838tw (allow small rounding) + assert!((w - 11906).abs() < 5, "Expected ~11906, got {w}"); + assert!((h - 16838).abs() < 5, "Expected ~16838, got {h}"); +} + +#[test] +fn comprehensive_document_round_trip() { + // Create a document with many Phase 2 features combined + let mut doc = Document::new(); + + // Custom style + doc.add_style(StyleBuilder::paragraph("BlockQuote", "Block Quote").based_on("Normal")); + + // Page setup + doc.set_margins( + Length::inches(1.0), + Length::inches(1.25), + Length::inches(1.0), + Length::inches(1.25), + ); + + // Title + doc.add_paragraph("My Document") + .style("Heading1") + .alignment(Alignment::Center) + .space_after(Length::pt(24.0)); + + // Body paragraph with formatting + let mut para = doc.add_paragraph(""); + para.add_run("This is ").font("Calibri").size(11.0); + para.add_run("important") + .bold(true) + .color("FF0000") + .font("Calibri") + .size(11.0); + para.add_run(" text with ").font("Calibri").size(11.0); + para.add_run("underline") + .underline(true) + .font("Calibri") + .size(11.0); + para.add_run(".").font("Calibri").size(11.0); + + // Block quote with indentation and shading + doc.add_paragraph("This is a block quote.") + .style("BlockQuote") + .indent_left(Length::inches(0.5)) + .indent_right(Length::inches(0.5)) + .shading("F2F2F2") + .space_before(Length::pt(6.0)) + .space_after(Length::pt(6.0)); + + // Bordered paragraph + doc.add_paragraph("Important note") + .border_all(BorderStyle::Single, 4, "000000") + .shading("FFFFCC"); + + // Save and reload + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 4); + assert!(doc2.style("BlockQuote").is_some()); + + let paras = doc2.paragraphs(); + assert_eq!(paras[0].style_id(), Some("Heading1")); + assert_eq!(paras[0].alignment(), Some(Alignment::Center)); + + let runs: Vec<_> = paras[1].runs().collect(); + assert_eq!(runs.len(), 5); + assert!(runs[1].is_bold()); + assert_eq!(runs[1].color(), Some("FF0000")); + + assert_eq!(paras[2].shading_fill(), Some("F2F2F2")); + assert!(paras[3].has_borders()); + assert_eq!(paras[3].shading_fill(), Some("FFFFCC")); +} + +// ---- Phase 3 Integration Tests ---- + +#[test] +fn table_basic_creation_round_trip() { + let mut doc = Document::new(); + let mut table = doc.add_table(3, 4); + table.cell(0, 0).unwrap().set_text("A1"); + table.cell(0, 1).unwrap().set_text("B1"); + table.cell(1, 0).unwrap().set_text("A2"); + table.cell(2, 3).unwrap().set_text("D3"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.table_count(), 1); + let tables = doc2.tables(); + let t = &tables[0]; + assert_eq!(t.row_count(), 3); + assert_eq!(t.column_count(), 4); + assert_eq!(t.cell(0, 0).unwrap().text(), "A1"); + assert_eq!(t.cell(0, 1).unwrap().text(), "B1"); + assert_eq!(t.cell(1, 0).unwrap().text(), "A2"); + assert_eq!(t.cell(2, 3).unwrap().text(), "D3"); +} + +#[test] +fn table_with_formatting_round_trip() { + let mut doc = Document::new(); + doc.add_table(2, 2) + .borders(BorderStyle::Single, 4, "000000") + .alignment(Alignment::Center) + .layout_fixed(); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.table_count(), 1); + let tables = doc2.tables(); + assert_eq!(tables[0].row_count(), 2); + assert_eq!(tables[0].column_count(), 2); +} + +#[test] +fn table_cell_shading_and_alignment() { + let mut doc = Document::new(); + let mut table = doc.add_table(2, 2); + + table.cell(0, 0).unwrap().set_text("Header"); + table + .cell(0, 0) + .unwrap() + .shading("4472C4") + .vertical_alignment(VerticalAlignment::Center); + + table + .cell(1, 0) + .unwrap() + .shading("D9E2F3") + .vertical_alignment(VerticalAlignment::Bottom); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let tables = doc2.tables(); + let cell_00 = tables[0].cell(0, 0).unwrap(); + assert_eq!(cell_00.shading_fill(), Some("4472C4")); + assert_eq!( + cell_00.vertical_alignment(), + Some(VerticalAlignment::Center) + ); + + let cell_10 = tables[0].cell(1, 0).unwrap(); + assert_eq!(cell_10.shading_fill(), Some("D9E2F3")); + assert_eq!( + cell_10.vertical_alignment(), + Some(VerticalAlignment::Bottom) + ); +} + +#[test] +fn table_header_row_round_trip() { + let mut doc = Document::new(); + let mut table = doc.add_table(3, 2); + table.row(0).unwrap().header(); + table.cell(0, 0).unwrap().set_text("Col A"); + table.cell(0, 1).unwrap().set_text("Col B"); + table.cell(1, 0).unwrap().set_text("Data 1"); + table.cell(2, 0).unwrap().set_text("Data 2"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let tables = doc2.tables(); + assert!(tables[0].row(0).unwrap().is_header()); + assert!(!tables[0].row(1).unwrap().is_header()); +} + +#[test] +fn table_cell_grid_span_round_trip() { + let mut doc = Document::new(); + let mut table = doc.add_table(2, 3); + // First row: cell 0 spans 2 columns + table.cell(0, 0).unwrap().set_text("Merged"); + table.cell(0, 0).unwrap().grid_span(2); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let tables = doc2.tables(); + assert_eq!(tables[0].cell(0, 0).unwrap().grid_span(), Some(2)); +} + +#[test] +fn table_mixed_with_paragraphs() { + let mut doc = Document::new(); + doc.add_paragraph("Before the table"); + let mut table = doc.add_table(2, 2); + table.cell(0, 0).unwrap().set_text("Cell"); + doc.add_paragraph("After the table"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 2); + assert_eq!(doc2.table_count(), 1); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Before the table"); + assert_eq!(paras[1].text(), "After the table"); + let tables = doc2.tables(); + assert_eq!(tables[0].cell(0, 0).unwrap().text(), "Cell"); +} + +#[test] +fn table_cell_multiple_paragraphs() { + let mut doc = Document::new(); + let mut table = doc.add_table(1, 1); + let mut cell = table.cell(0, 0).unwrap(); + cell.set_text("First line"); + let mut para = cell.add_paragraph(""); + para.add_run("Second line").bold(true); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + let tables = doc2.tables(); + let cell = tables[0].cell(0, 0).unwrap(); + let paras: Vec<_> = cell.paragraphs().collect(); + assert_eq!(paras.len(), 2); + assert_eq!(paras[0].text(), "First line"); + assert_eq!(paras[1].text(), "Second line"); +} + +#[test] +fn inline_image_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Before image"); + + // Create a minimal 1x1 white PNG + let png_data: Vec = vec![ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, // PNG signature + 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, // IHDR chunk + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, // 1x1 + 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, 0xDE, // 8-bit RGB + 0x00, 0x00, 0x00, 0x0C, 0x49, 0x44, 0x41, 0x54, // IDAT chunk + 0x08, 0xD7, 0x63, 0xF8, 0xCF, 0xC0, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0xE2, 0x21, 0xBC, + 0x33, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, // IEND chunk + 0xAE, 0x42, 0x60, 0x82, + ]; + + doc.add_picture( + &png_data, + "test.png", + Length::inches(2.0), + Length::inches(1.5), + ); + doc.add_paragraph("After image"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + // Image paragraph is counted as a paragraph too + assert_eq!(doc2.paragraph_count(), 3); + assert_eq!(doc2.paragraphs()[0].text(), "Before image"); + assert_eq!(doc2.paragraphs()[2].text(), "After image"); +} + +#[test] +fn header_footer_round_trip() { + let mut doc = Document::new(); + doc.set_header("Page Header"); + doc.set_footer("Page Footer"); + doc.add_paragraph("Body text"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.header_text(), Some("Page Header".to_string())); + assert_eq!(doc2.footer_text(), Some("Page Footer".to_string())); + assert_eq!(doc2.paragraph_count(), 1); +} + +#[test] +fn first_page_header_footer() { + let mut doc = Document::new(); + doc.set_header("Default Header"); + doc.set_footer("Default Footer"); + doc.set_first_page_header("First Page Header"); + doc.set_first_page_footer("First Page Footer"); + doc.add_paragraph("Content"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.header_text(), Some("Default Header".to_string())); + assert_eq!(doc2.footer_text(), Some("Default Footer".to_string())); + // titlePg should be set + let sect = doc2.section_properties().unwrap(); + assert_eq!(sect.title_pg, Some(true)); +} + +#[test] +fn bullet_list_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Items:"); + doc.add_bullet_list_item("First item", 0); + doc.add_bullet_list_item("Second item", 0); + doc.add_bullet_list_item("Sub-item", 1); + doc.add_bullet_list_item("Third item", 0); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 5); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Items:"); + assert_eq!(paras[1].text(), "First item"); + assert_eq!(paras[2].text(), "Second item"); + assert_eq!(paras[3].text(), "Sub-item"); + assert_eq!(paras[4].text(), "Third item"); +} + +#[test] +fn numbered_list_round_trip() { + let mut doc = Document::new(); + doc.add_numbered_list_item("Step one", 0); + doc.add_numbered_list_item("Step two", 0); + doc.add_numbered_list_item("Sub-step a", 1); + doc.add_numbered_list_item("Sub-step b", 1); + doc.add_numbered_list_item("Step three", 0); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 5); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Step one"); + assert_eq!(paras[4].text(), "Step three"); +} + +#[test] +fn mixed_lists_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("Introduction"); + doc.add_bullet_list_item("Bullet 1", 0); + doc.add_bullet_list_item("Bullet 2", 0); + doc.add_paragraph("Transition"); + doc.add_numbered_list_item("Step 1", 0); + doc.add_numbered_list_item("Step 2", 0); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.paragraph_count(), 6); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Introduction"); + assert_eq!(paras[1].text(), "Bullet 1"); + assert_eq!(paras[3].text(), "Transition"); + assert_eq!(paras[4].text(), "Step 1"); +} + +#[test] +fn comprehensive_phase3_document() { + // Create a document using all Phase 3 features together + let mut doc = Document::new(); + + // Page setup + doc.set_margins( + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + Length::inches(1.0), + ); + doc.set_header("Phase 3 Test Document"); + doc.set_footer("Confidential"); + + // Title + doc.add_paragraph("Project Report") + .style("Heading1") + .alignment(Alignment::Center); + + // Intro paragraph + doc.add_paragraph("This document demonstrates Phase 3 features."); + + // Bulleted list section + doc.add_paragraph("Key Points:").style("Heading2"); + doc.add_bullet_list_item("Tables with formatting", 0); + doc.add_bullet_list_item("Inline images", 0); + doc.add_bullet_list_item("Headers and footers", 0); + doc.add_bullet_list_item("Numbered and bulleted lists", 0); + + // Table section + doc.add_paragraph("Data Summary:").style("Heading2"); + let mut table = doc + .add_table(3, 3) + .borders(BorderStyle::Single, 4, "000000"); + + // Header row + table.row(0).unwrap().header(); + table.cell(0, 0).unwrap().set_text("Category"); + table + .cell(0, 0) + .unwrap() + .shading("4472C4") + .vertical_alignment(VerticalAlignment::Center); + table.cell(0, 1).unwrap().set_text("Q1"); + table.cell(0, 1).unwrap().shading("4472C4"); + table.cell(0, 2).unwrap().set_text("Q2"); + table.cell(0, 2).unwrap().shading("4472C4"); + + // Data rows + table.cell(1, 0).unwrap().set_text("Revenue"); + table.cell(1, 1).unwrap().set_text("$1,200"); + table.cell(1, 2).unwrap().set_text("$1,500"); + table.cell(2, 0).unwrap().set_text("Expenses"); + table.cell(2, 1).unwrap().set_text("$800"); + table.cell(2, 2).unwrap().set_text("$900"); + + // Numbered steps + doc.add_paragraph("Next Steps:").style("Heading2"); + doc.add_numbered_list_item("Review Q2 financials", 0); + doc.add_numbered_list_item("Prepare Q3 forecast", 0); + doc.add_numbered_list_item("Gather input from teams", 1); + doc.add_numbered_list_item("Consolidate data", 1); + doc.add_numbered_list_item("Submit report", 0); + + // Save and reload + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + // Verify structure + assert_eq!(doc2.table_count(), 1); + let tables = doc2.tables(); + assert_eq!(tables[0].row_count(), 3); + assert_eq!(tables[0].column_count(), 3); + assert!(tables[0].row(0).unwrap().is_header()); + assert_eq!(tables[0].cell(1, 1).unwrap().text(), "$1,200"); + + // Verify header/footer + assert_eq!( + doc2.header_text(), + Some("Phase 3 Test Document".to_string()) + ); + assert_eq!(doc2.footer_text(), Some("Confidential".to_string())); + + // Count paragraphs (heading + intro + 3 heading2 + 4 bullets + 5 numbered = 15 total paragraphs) + assert!(doc2.paragraph_count() > 10); +} + +#[test] +fn metadata_round_trip() { + let mut doc = Document::new(); + doc.set_title("Test Title"); + doc.set_author("Test Author"); + doc.set_subject("Test Subject"); + doc.set_keywords("rust, docx, test"); + + assert_eq!(doc.title(), Some("Test Title")); + assert_eq!(doc.author(), Some("Test Author")); + assert_eq!(doc.subject(), Some("Test Subject")); + assert_eq!(doc.keywords(), Some("rust, docx, test")); + + // Round-trip through DOCX bytes + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.title(), Some("Test Title")); + assert_eq!(doc2.author(), Some("Test Author")); + assert_eq!(doc2.subject(), Some("Test Subject")); + assert_eq!(doc2.keywords(), Some("rust, docx, test")); +} + +#[test] +fn nested_table_round_trip() { + let mut doc = Document::new(); + + // Create outer 2x2 table + let mut tbl = doc.add_table(2, 2); + tbl.cell(0, 0).unwrap().set_text("Outer A1"); + tbl.cell(0, 1).unwrap().set_text("Outer A2"); + tbl.cell(1, 1).unwrap().set_text("Outer B2"); + + // Add a nested 2x1 table inside cell (1, 0) + let mut cell_b1 = tbl.cell(1, 0).unwrap(); + cell_b1.set_text("Before nested"); + let mut nested = cell_b1.add_table(2, 1); + nested.cell(0, 0).unwrap().set_text("Inner R1"); + nested.cell(1, 0).unwrap().set_text("Inner R2"); + + // Serialize and reload + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + // Verify outer table structure + assert_eq!(doc2.table_count(), 1); + let tables = doc2.tables(); + assert!(!tables.is_empty()); + let tbl2 = &tables[0]; + assert_eq!(tbl2.row_count(), 2); + assert_eq!(tbl2.cell(0, 0).unwrap().text(), "Outer A1"); + assert_eq!(tbl2.cell(0, 1).unwrap().text(), "Outer A2"); + assert_eq!(tbl2.cell(1, 1).unwrap().text(), "Outer B2"); + + // Cell (1,0) should have paragraph text (nested table text excluded from text()) + let cell_b1_ref = tbl2.cell(1, 0).unwrap(); + assert_eq!(cell_b1_ref.text(), "Before nested"); +} + +#[test] +fn comprehensive_document_round_trip_with_nested() { + use rdocx::paragraph::Alignment; + + let mut doc = Document::new(); + + // Metadata + doc.set_title("Comprehensive Test Document"); + doc.set_author("rdocx Test Suite"); + + // Heading 1 + doc.add_paragraph("Chapter 1: Introduction") + .style("Heading1"); + + // Normal paragraphs + doc.add_paragraph("This is a normal paragraph with some body text.") + .alignment(Alignment::Left); + + doc.add_paragraph("This paragraph is centered for emphasis.") + .alignment(Alignment::Center); + + doc.add_paragraph("This paragraph is justified for a clean look.") + .alignment(Alignment::Justify); + + // Heading 2 + doc.add_paragraph("Section 1.1: Data Table") + .style("Heading2"); + + // Table with formatting + let mut tbl = doc.add_table(3, 3); + tbl = tbl.borders(rdocx::BorderStyle::Single, 4, "000000"); + + // Header row + tbl.row(0).unwrap().header(); + tbl.cell(0, 0).unwrap().set_text("Name"); + tbl.cell(0, 1).unwrap().set_text("Value"); + tbl.cell(0, 2).unwrap().set_text("Status"); + + // Data rows + tbl.cell(1, 0).unwrap().set_text("Alpha"); + tbl.cell(1, 1).unwrap().set_text("100"); + tbl.cell(1, 2).unwrap().set_text("Active"); + + tbl.cell(2, 0).unwrap().set_text("Beta"); + tbl.cell(2, 1).unwrap().set_text("200"); + tbl.cell(2, 2).unwrap().set_text("Pending"); + + // Another heading + doc.add_paragraph("Chapter 2: Nested Content") + .style("Heading1"); + + // Table with nested table + let mut tbl2 = doc.add_table(2, 2); + tbl2.cell(0, 0).unwrap().set_text("Outer cell"); + tbl2.cell(0, 1).unwrap().set_text("Another outer cell"); + tbl2.cell(1, 0).unwrap().set_text("Simple cell"); + + let mut nested_cell = tbl2.cell(1, 1).unwrap(); + nested_cell.set_text("Contains nested table:"); + let mut nested = nested_cell.add_table(2, 2); + nested.cell(0, 0).unwrap().set_text("N1"); + nested.cell(0, 1).unwrap().set_text("N2"); + nested.cell(1, 0).unwrap().set_text("N3"); + nested.cell(1, 1).unwrap().set_text("N4"); + + // Bullet list + doc.add_paragraph("Chapter 3: Lists").style("Heading1"); + doc.add_paragraph("First bullet point").style("ListBullet"); + doc.add_paragraph("Second bullet point").style("ListBullet"); + doc.add_paragraph("Third bullet point").style("ListBullet"); + + // Final paragraph + doc.add_paragraph("End of document."); + + // Round-trip through DOCX + let docx_bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&docx_bytes).unwrap(); + + // Verify structure + assert_eq!(doc2.title(), Some("Comprehensive Test Document")); + assert_eq!(doc2.author(), Some("rdocx Test Suite")); + assert!(doc2.table_count() >= 2); +} + +// ---- Phase 7: Template Engine, Placeholder Replacement & Background Images ---- + +#[test] +fn template_open_replace_save() { + // Create a "template" document + let mut template = Document::new(); + template.set_header("Company: {{company}}"); + template.set_footer("Date: {{date}}"); + template.add_paragraph("Dear {{name}},").style("Heading1"); + template.add_paragraph("Welcome to {{company}}. Your role starts on {{date}}."); + template.add_paragraph("{{INSERT_CONTENT}}"); + template.add_paragraph("Best regards,"); + template.add_paragraph("HR Department"); + + let template_bytes = template.to_bytes().unwrap(); + + // Open template and do replacements + let mut doc = Document::from_bytes(&template_bytes).unwrap(); + doc.replace_text("{{company}}", "Acme Corp"); + doc.replace_text("{{name}}", "Alice"); + doc.replace_text("{{date}}", "2026-03-01"); + + // Find and replace content placeholder + if let Some(idx) = doc.find_content_index("{{INSERT_CONTENT}}") { + doc.remove_content(idx); + doc.insert_paragraph(idx, "Your onboarding schedule is attached."); + doc.insert_paragraph(idx + 1, "Please review and confirm."); + } + + // Verify + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "Dear Alice,"); + assert_eq!( + paras[1].text(), + "Welcome to Acme Corp. Your role starts on 2026-03-01." + ); + assert_eq!(paras[2].text(), "Your onboarding schedule is attached."); + assert_eq!(paras[3].text(), "Please review and confirm."); + assert_eq!(paras[4].text(), "Best regards,"); + + assert_eq!(doc.header_text().unwrap(), "Company: Acme Corp"); + assert_eq!(doc.footer_text().unwrap(), "Date: 2026-03-01"); + + // Round-trip + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraphs()[0].text(), "Dear Alice,"); +} + +#[test] +fn replace_all_batch_workflow() { + let mut doc = Document::new(); + doc.add_paragraph("{{a}} {{b}} {{c}}"); + + let mut map = std::collections::HashMap::new(); + map.insert("{{a}}", "X"); + map.insert("{{b}}", "Y"); + map.insert("{{c}}", "Z"); + let count = doc.replace_all(&map); + assert_eq!(count, 3); + assert_eq!(doc.paragraphs()[0].text(), "X Y Z"); +} + +#[test] +fn background_image_end_to_end() { + // Minimal 1x1 PNG + let png_data: Vec = vec![ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, + 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, + 0x77, 0x53, 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, + 0xcf, 0xc0, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0xe2, 0x21, 0xbc, 0x33, 0x00, 0x00, 0x00, + 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]; + + let mut doc = Document::new(); + doc.add_paragraph("Page content here"); + doc.add_background_image(&png_data, "background.png"); + + // Background paragraph inserted at index 0 + assert_eq!(doc.content_count(), 2); + + // Round-trip DOCX + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.content_count(), 2); +} + +#[test] +fn full_phase7_workflow() { + // Minimal PNG + let png_data: Vec = vec![ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, + 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, + 0x77, 0x53, 0xde, 0x00, 0x00, 0x00, 0x0c, 0x49, 0x44, 0x41, 0x54, 0x08, 0xd7, 0x63, 0xf8, + 0xcf, 0xc0, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0xe2, 0x21, 0xbc, 0x33, 0x00, 0x00, 0x00, + 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]; + + let mut doc = Document::new(); + doc.set_title("Template Output"); + doc.set_header("{{company}} - Confidential"); + + doc.add_paragraph("Report for {{company}}") + .style("Heading1"); + doc.add_paragraph("Date: {{date}}"); + doc.add_paragraph("{{INSERT_HERE}}"); + doc.add_paragraph("Summary: {{company}} performed well in {{date}}."); + + // Add background image + doc.add_background_image(&png_data, "bg.png"); + + // Replace placeholders + doc.replace_text("{{company}}", "Acme Corp"); + doc.replace_text("{{date}}", "2026-02-22"); + + // Insert content at placeholder position + if let Some(idx) = doc.find_content_index("{{INSERT_HERE}}") { + doc.remove_content(idx); + doc.insert_paragraph(idx, "Revenue increased by 15%."); + } + + // Verify final state + let paras = doc.paragraphs(); + // First paragraph is the background image paragraph, skip it + let text_paras: Vec<_> = paras.iter().filter(|p| !p.text().is_empty()).collect(); + assert!( + text_paras + .iter() + .any(|p| p.text() == "Report for Acme Corp") + ); + assert!(text_paras.iter().any(|p| p.text() == "Date: 2026-02-22")); + assert!( + text_paras + .iter() + .any(|p| p.text() == "Revenue increased by 15%.") + ); + assert!( + text_paras + .iter() + .any(|p| p.text() == "Summary: Acme Corp performed well in 2026-02-22.") + ); + + assert_eq!(doc.header_text().unwrap(), "Acme Corp - Confidential"); + + // Save as DOCX + let bytes = doc.to_bytes().unwrap(); + assert!(!bytes.is_empty()); +} + +// ---- Phase: Code Quality & Modernization — New Tests ---- + +#[test] +fn section_break_builders_round_trip() { + let mut doc = Document::new(); + doc.add_paragraph("First section"); + + // Create a section break to landscape + doc.add_paragraph("Landscape section break") + .section_break(SectionBreak::NextPage) + .section_landscape(); + + doc.add_paragraph("In landscape section"); + + // Switch back to portrait + doc.add_paragraph("Portrait section break") + .section_break(SectionBreak::NextPage) + .section_portrait(); + + doc.add_paragraph("Back to portrait"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraph_count(), 5); +} + +#[test] +fn section_page_size_custom() { + let mut doc = Document::new(); + // Custom page size: 6" x 9" (book size) + doc.add_paragraph("Small page") + .section_break(SectionBreak::NextPage) + .section_page_size(Length::inches(6.0), Length::inches(9.0)); + + doc.add_paragraph("On the next section"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraph_count(), 2); +} + +#[test] +fn section_break_continuous() { + let mut doc = Document::new(); + doc.add_paragraph("Before continuous break") + .section_break(SectionBreak::Continuous); + doc.add_paragraph("After continuous break"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraph_count(), 2); +} + +#[test] +fn tab_stops_all_leader_styles() { + let mut doc = Document::new(); + + // Tab stop with no leader + doc.add_paragraph("Left\tRight") + .add_tab_stop(TabAlignment::Right, Length::inches(6.0)); + + // Tab stop with dot leader + doc.add_paragraph("Item\t100").add_tab_stop_with_leader( + TabAlignment::Right, + Length::inches(6.0), + TabLeader::Dot, + ); + + // Tab stop with hyphen leader + doc.add_paragraph("Section\tPage 5") + .add_tab_stop_with_leader(TabAlignment::Right, Length::inches(6.0), TabLeader::Hyphen); + + // Tab stop with underscore leader + doc.add_paragraph("Name\t").add_tab_stop_with_leader( + TabAlignment::Right, + Length::inches(6.0), + TabLeader::Underscore, + ); + + // Multiple alignments + doc.add_paragraph("A\tB\tC") + .add_tab_stop(TabAlignment::Center, Length::inches(3.0)) + .add_tab_stop(TabAlignment::Right, Length::inches(6.0)) + .add_tab_stop(TabAlignment::Decimal, Length::inches(4.5)); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + let paras = doc2.paragraphs(); + + assert_eq!(paras[0].tab_stop_count(), 1); + assert_eq!(paras[1].tab_stop_count(), 1); + assert_eq!(paras[2].tab_stop_count(), 1); + assert_eq!(paras[3].tab_stop_count(), 1); + assert_eq!(paras[4].tab_stop_count(), 3); +} + +#[test] +fn run_formatting_all_caps_small_caps() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("UPPERCASE").all_caps(true); + para.add_run("SmallCaps").small_caps(true); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "UPPERCASESmallCaps"); +} + +#[test] +fn run_formatting_double_strike_and_spacing() { + let mut doc = Document::new(); + let mut para = doc.add_paragraph(""); + para.add_run("DStrike").double_strike(true); + para.add_run("Spaced").character_spacing(Length::pt(3.0)); + para.add_run("Super").superscript(); + para.add_run("Sub").subscript(); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + let paras = doc2.paragraphs(); + let runs: Vec<_> = paras[0].runs().collect(); + assert_eq!(runs.len(), 4); + assert_eq!(runs[2].vert_align(), Some("superscript")); + assert_eq!(runs[3].vert_align(), Some("subscript")); + assert!(runs[1].character_spacing().is_some()); +} + +#[test] +fn paragraph_border_bottom_only() { + let mut doc = Document::new(); + doc.add_paragraph("Bottom bordered") + .border_bottom(BorderStyle::Single, 4, "000000"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + let paras = doc2.paragraphs(); + assert!(paras[0].has_borders()); +} + +#[test] +fn paragraph_shading_and_indent_combined() { + let mut doc = Document::new(); + doc.add_paragraph("Shaded and indented") + .shading("E0E0E0") + .indent_left(Length::inches(0.75)) + .indent_right(Length::inches(0.5)) + .hanging_indent(Length::inches(0.25)); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + let paras = doc2.paragraphs(); + assert_eq!(paras[0].shading_fill(), Some("E0E0E0")); + assert_eq!(paras[0].text(), "Shaded and indented"); +} + +#[test] +fn document_header_footer_first_page() { + let mut doc = Document::new(); + doc.set_header("Default Header"); + doc.set_footer("Default Footer"); + doc.set_first_page_header("First Page Header"); + doc.set_first_page_footer("First Page Footer"); + doc.add_paragraph("Body content"); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + + assert_eq!(doc2.header_text(), Some("Default Header".to_string())); + assert_eq!(doc2.footer_text(), Some("Default Footer".to_string())); + let sect = doc2.section_properties().unwrap(); + assert_eq!(sect.title_pg, Some(true)); +} + +#[test] +fn insert_paragraph_at_beginning_and_end() { + let mut doc = Document::new(); + doc.add_paragraph("Middle"); + + // Insert at beginning + doc.insert_paragraph(0, "First"); + // Insert at end + let count = doc.content_count(); + doc.insert_paragraph(count, "Last"); + + assert_eq!(doc.content_count(), 3); + let paras = doc.paragraphs(); + assert_eq!(paras[0].text(), "First"); + assert_eq!(paras[1].text(), "Middle"); + assert_eq!(paras[2].text(), "Last"); +} + +#[test] +fn insert_table_at_index() { + let mut doc = Document::new(); + doc.add_paragraph("Before"); + doc.add_paragraph("After"); + + // Insert table between the two paragraphs + let mut table = doc.insert_table(1, 2, 2); + table.cell(0, 0).unwrap().set_text("Cell"); + + assert_eq!(doc.content_count(), 3); + + let bytes = doc.to_bytes().unwrap(); + let doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.table_count(), 1); + assert_eq!(doc2.paragraph_count(), 2); +} + +#[test] +fn remove_content_basic() { + let mut doc = Document::new(); + doc.add_paragraph("Keep"); + doc.add_paragraph("Remove"); + doc.add_paragraph("Keep too"); + + assert_eq!(doc.content_count(), 3); + assert!(doc.remove_content(1)); + assert_eq!(doc.content_count(), 2); + assert_eq!(doc.paragraphs()[0].text(), "Keep"); + assert_eq!(doc.paragraphs()[1].text(), "Keep too"); +} + +#[test] +fn remove_content_out_of_bounds() { + let mut doc = Document::new(); + doc.add_paragraph("Only"); + assert!(!doc.remove_content(5)); + assert_eq!(doc.content_count(), 1); +} + +#[test] +fn find_content_index_returns_none_for_missing() { + let mut doc = Document::new(); + doc.add_paragraph("Hello"); + assert_eq!(doc.find_content_index("nonexistent"), None); +} + +#[test] +fn section_break_round_trip_preserves() { + let mut doc = Document::new(); + doc.add_paragraph("Section 1") + .section_break(SectionBreak::NextPage) + .section_landscape(); + + doc.add_paragraph("Section 2 (landscape)"); + + doc.add_paragraph("Section 2 end") + .section_break(SectionBreak::NextPage) + .section_portrait(); + + doc.add_paragraph("Section 3 (portrait)"); + + // Round-trip + let bytes = doc.to_bytes().unwrap(); + let mut doc2 = Document::from_bytes(&bytes).unwrap(); + assert_eq!(doc2.paragraph_count(), 4); + + // The document should still be valid and have the paragraphs + let paras = doc2.paragraphs(); + assert_eq!(paras[0].text(), "Section 1"); + assert_eq!(paras[1].text(), "Section 2 (landscape)"); + assert_eq!(paras[2].text(), "Section 2 end"); + assert_eq!(paras[3].text(), "Section 3 (portrait)"); + + // Re-round-trip to verify stability + let bytes2 = doc2.to_bytes().unwrap(); + let doc3 = Document::from_bytes(&bytes2).unwrap(); + assert_eq!(doc3.paragraph_count(), 4); +} + +#[test] +fn empty_document_insert_and_remove() { + let mut doc = Document::new(); + assert_eq!(doc.content_count(), 0); + + doc.insert_paragraph(0, "Inserted"); + assert_eq!(doc.content_count(), 1); + assert_eq!(doc.paragraphs()[0].text(), "Inserted"); + + assert!(doc.remove_content(0)); + assert_eq!(doc.content_count(), 0); +} + +// ---- PDF rendering tests ---- + +#[test] +fn to_pdf_simple_document() { + let mut doc = Document::new(); + doc.add_paragraph("Hello, World!"); + doc.add_paragraph("This is a test document."); + + let result = doc.to_pdf(); + // On systems without fonts, layout may fail — that's OK for CI + if let Ok(pdf_bytes) = result { + // Verify it starts with PDF header + assert!(pdf_bytes.starts_with(b"%PDF")); + // Verify it's not trivially small + assert!(pdf_bytes.len() > 100); + // Verify it ends with %%EOF + let tail = String::from_utf8_lossy(&pdf_bytes[pdf_bytes.len().saturating_sub(10)..]); + assert!(tail.contains("%%EOF")); + } +} + +#[test] +fn to_pdf_with_formatting() { + let mut doc = Document::new(); + doc.add_paragraph("Title") + .style("Heading1") + .alignment(Alignment::Center); + doc.add_paragraph("Normal text with ") + .add_run("bold") + .bold(true); + doc.add_paragraph("Another paragraph"); + + let result = doc.to_pdf(); + if let Ok(pdf_bytes) = result { + assert!(pdf_bytes.starts_with(b"%PDF")); + assert!(pdf_bytes.len() > 200); + } +} + +#[test] +fn to_pdf_with_table() { + let mut doc = Document::new(); + doc.add_paragraph("Table test"); + { + let mut table = doc.add_table(2, 3); + table.cell(0, 0).unwrap().set_text("A1"); + table.cell(0, 1).unwrap().set_text("B1"); + table.cell(0, 2).unwrap().set_text("C1"); + table.cell(1, 0).unwrap().set_text("A2"); + table.cell(1, 1).unwrap().set_text("B2"); + table.cell(1, 2).unwrap().set_text("C2"); + } + doc.add_paragraph("After table"); + + let result = doc.to_pdf(); + if let Ok(pdf_bytes) = result { + assert!(pdf_bytes.starts_with(b"%PDF")); + } +} + +#[test] +fn to_pdf_with_metadata() { + let mut doc = Document::new(); + doc.set_title("Test Document"); + doc.set_author("rdocx"); + doc.add_paragraph("Content"); + + let result = doc.to_pdf(); + if let Ok(pdf_bytes) = result { + assert!(pdf_bytes.starts_with(b"%PDF")); + // Metadata should be embedded in the PDF + let pdf_str = String::from_utf8_lossy(&pdf_bytes); + assert!(pdf_str.contains("Test Document") || pdf_str.contains("rdocx-pdf")); + } +} + +#[test] +fn save_pdf_to_file() { + let mut doc = Document::new(); + doc.add_paragraph("PDF file test"); + + let path = "/tmp/rdocx_test_output.pdf"; + let result = doc.save_pdf(path); + if result.is_ok() { + let bytes = std::fs::read(path).unwrap(); + assert!(bytes.starts_with(b"%PDF")); + std::fs::remove_file(path).ok(); + } +} diff --git a/x.md b/x.md new file mode 100644 index 00000000..0516ccaf --- /dev/null +++ b/x.md @@ -0,0 +1,15 @@ + - Participants are only visible in the collaboration dialog. While editing, the tab badge and status pill show a count, but there is no persistent avatar stack or obvious “who is here?” control near the document. + - Solution: This seems to call for a top to bottom collab UX stack, where clients can define their identities, profile pictures, and possibly even preferred colors durably. It also implies UI elements indicating presence. These are probably desirable. + + - Permissions are currently implicit. Every generated invite is an Editor invite, and the UI does not let the owner choose Viewer vs. Editor. The code already has capability roles; the Share UI simply hardcodes Editor today. + - Solution: All collaborators are editors is an intentional simplification for the debate workflow. We do not intend to change it, and we should leave it unchanged. + + - There is no in-document social layer. Users can see colored remote carets and join/leave notifications, but there are no comments, mentions, chat, or activity history. + That may be intentional for now, but it limits collaboration to simultaneous editing. + - Query: How much of a lift would it be to add comments that feel nice and production-ready? The other stuff is irrelevant. Secondarily: isn't activity history represented by content provenance of the peers in the edit history of a Loro doc? Or is that list flat and not contain identity? + + - The invite is a raw text ticket. The flow is copy/paste-oriented: copy a long encoded string, send it elsewhere, and paste it into Flowstate. There is no friendlier link, QR code, contact picker, or clear expiry/permission summary. + - Solution: Agreed, but I do not know what the full frontier of discovery is and ought to be. Obviously we should maintain some form of link discovery. Is it possible to allow clients to establish each other as 'trusted' to join each other's sessions at will? Is it possible to use the Dropbox SDK to identify when two clients are working on the same synced Dropbox document, exchange discovery automatically via Dropbox SDK, and propose initiating the session seamlessly with a single click? (Most debate squads use Dropbox for the squad). What about Bluetooth discovery of nearby advertising identities for when you need to quickly link up with a partner? Is there a friendlier way to present the copy-paste link disocvery (or really just: a friendlier way to permit the total fallback discovery in general?) + + - Session management is dialog-heavy. Starting, inviting, checking participants, and leaving all happen through the Share/Collaborate dialog or a small status pill. There is not yet a dedicated collaboration sidebar or lightweight popover that stays useful while editing. + - Agreed, but let's not edit this until we have a locked down vision for the rest of the ux, since this is heavily ux dependent.