diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a95d9c5..8de2099 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: # always on pullrequest pull_request: - types: [opened, synchronize, reopened, edited] + types: [opened, synchronize, reopened] env: @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build Docker image run: docker build -t distributed-topic-tracker . diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8ecac1d..75021da 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -17,7 +17,9 @@ Contents: Components: - iroh endpoint and gossip - Auto-discovery (bootstrap loop) -- Publisher (background task) +- Publisher (background actor) +- Bubble merge (background actor) +- Message overlap merge (background actor) - DHT client (mutable records) - Crypto (signing, encryption, secret rotation) @@ -35,6 +37,8 @@ flowchart LR subgraph AutoDiscovery B[Bootstrap Loop] P[Publisher] + BM[Bubble Merge] + MO[Message Overlap Merge] end subgraph DHT @@ -44,8 +48,12 @@ flowchart LR A --> E --> G G <---> B G <---> P + G <---> BM + G <---> MO B <--> D P <--> D + BM <--> D + MO <--> D ``` Node lifecycle: @@ -53,7 +61,7 @@ Node lifecycle: - Start gossip - Auto-discovery: - Join topic, attempt bootstrap, connect - - Spawn publisher on success + - Spawn publisher, bubble merge, and message overlap merge actors on success State machine: @@ -64,7 +72,7 @@ stateDiagram-v2 Discovering --> Joining Joining --> Joined Joined --> Publishing - Publishing --> Joined : backoff or success loop + Publishing --> Joined : interval + jitter Discovering --> Discovering : retry/jitter Joining --> Discovering : no peers ``` @@ -82,25 +90,28 @@ sequenceDiagram participant Gossip Node->>Gossip: subscribe(topic_hash) - Node->>DHT: get_mutable(signing_pub, salt, 10s) + Note over Node: optionally publish startup record + Node->>DHT: get_mutable(signing_pub, salt, 10s timeout) DHT-->>Node: encrypted records (0..N) Node->>Node: decrypt, verify, filter(not self) alt candidates exist loop each candidate - Node->>Gossip: join_peers([node_id]) - Node->>Node: sleep 100ms + Node->>Gossip: join_peers([pub_key]) + Node->>Node: sleep per_peer_join_settle_time (100ms) Gossip-->>Node: NeighborUp? end - Node->>Node: final wait 500ms + Node->>Node: final wait join_confirmation_wait_time (500ms) else no candidates Node->>Node: maybe publish own (rate-limited) + Node->>Node: sleep no_peers_retry_interval (1500ms) end - Node->>Node: joined? if yes, spawn publisher + Node->>Node: joined? if yes, spawn publisher + merge actors ``` Key points: -- First iteration: also check previous unix minute. -- Pacing avoids bursts and “bubbles.” +- First iteration: optionally check older records first (`check_older_records_first_on_startup`). +- Both `unix_minute` and `unix_minute - 1` records are always fetched. +- Pacing avoids bursts and "bubbles." - Keep trying until joined. Pseudocode: @@ -109,23 +120,23 @@ Pseudocode: loop: if joined(): return sender, receiver - minute = first_attempt ? -1 : 0 - recs = get_unix_minute_records(minute) + minute = first_attempt && check_last_minute_first ? -1 : 0 + recs = get_records(unix_minute(minute) - 1) + get_records(unix_minute(minute)) if recs.is_empty(): maybe_publish_this_minute() - sleep(100ms) + sleep(no_peers_retry_interval = 1500ms) continue for peer in extract_bootstrap_nodes(recs): if joined(): break join_peer(peer) - sleep(100ms) + sleep(per_peer_join_settle_time = 100ms) - sleep(500ms) + sleep(join_confirmation_wait_time = 500ms) if joined(): return maybe_publish_this_minute() - sleep(100ms) + sleep(discovery_poll_interval = 2000ms) ``` ## Publishing @@ -136,36 +147,36 @@ Flow: ```mermaid flowchart TD - A[Start Cycle] --> B[Get minute=now] - B --> C[Discover existing records] - C --> D[Filter active participants] - D --> E{>= 10 active?} - E -- Yes --> F[Stop rate-limited] + A[Tick] --> B[Get minute=now] + B --> C[Get existing records] + C --> E{>= max_bootstrap_records = 5?} + E -- Yes --> F[Skip - rate-limited] E -- No --> G[Build record: peers + msg hashes] G --> H[Sign + Encrypt] - H --> I[Publish with retries + jitter] - I --> J[Return records, including own on success] + H --> I[Publish to DHT] + I --> J[Reset ticker: base_interval + random jitter] ``` Pseudocode: ```text -records = get_unix_minute_records(now) -active = filter_active(records) -if active.len >= 10: return records - -rec = make_record(neighbors(<=5), last_hashes(<=5)) -enc = encrypt(sign(rec)) -publish_with_retry(enc, retries=3, jitter=0..2000ms) -return records + [rec_if_success] +// Publisher actor loop (interval: base_interval + random jitter) +on tick: + records = get_records(unix_minute(0)) + if records.len >= max_bootstrap_records(5): return + + rec = make_record(neighbors(<=5), last_hashes(<=5)) + enc = encrypt(sign(rec)) + publish(enc) + reset_ticker(base_interval + random(0, max_jitter)) ``` ## Bubble detection and merging -Signal 1: small cluster \(neighbors < 4\). +Signal 1: small cluster \(neighbors < min\_neighbors, default 4\). - Extract peer ids from discovered records. - Exclude zeros, self, current neighbors. -- Join up to MAX_JOIN_PEERS_COUNT. +- Join up to max_join_peer_count (default 4). Signal 2: non-overlapping message sets. - Compare local last_message_hashes with others. @@ -176,12 +187,12 @@ Decision graph: ```mermaid flowchart LR - A[Post-Publish Records] --> B{neighbors < 4?} + A[Tick] --> B{neighbors < min_neighbors?} B -- Yes --> C[Join peers from records] B -- No --> D{local_msgs >= 1?} - D -- No --> E[Sleep random 0..60s] + D -- No --> E[Sleep until next tick] D -- Yes --> F{overlap with others?} - F -- No --> G[Join from non-overlap records] + F -- No --> G[Join from non-overlapping records] F -- Yes --> E ``` @@ -190,9 +201,8 @@ flowchart LR Record (summary): - topic hash (32) - unix_minute (u64) -- node_id (publisher) -- active_peers[5] (node ids) -- last_message_hashes[5] +- pub_key (publisher ed25519 public key) +- content (serialized GossipRecordContent: active_peers + last_message_hashes) - signature (64) EncryptedRecord: @@ -206,16 +216,22 @@ classDiagram class Record { +topic: [u8;32] +unix_minute: u64 - +node_id: [u8;32] + +pub_key: [u8;32] + +content: GossipRecordContent + +signature: [u8;64] + } + + class GossipRecordContent { +active_peers: [[u8;32];5] +last_message_hashes: [[u8;32];5] - +signature: [u8;64] } class EncryptedRecord { +encrypted_record: Vec +encrypted_decryption_key: Vec } + + Record --* GossipRecordContent : content deserializes to ``` Key derivation: @@ -225,7 +241,9 @@ flowchart TD T[topic_hash] --> A[SHA512 topic+minute] M[unix_minute] --> A A --> S[signing_keypair seed -> Ed25519] - A --> L[salt = first 32 bytes] + + T --> L["salt = SHA512('salt' + topic + minute)[..32]"] + M --> L T --> R[secret_rotation topic,minute,initial_secret_hash] M --> R @@ -239,23 +257,24 @@ flowchart TD - Decrypt/verify failure: - Drop record; proceed. - Publish failure: - - Exponential backoff (1..60 s), then retry. + - DHT layer retries with jittered intervals (3 retries, 5s base + 0-10s jitter). - Join failure: - - Continue to next peer; final 500 ms wait; loop. + - Continue to next peer; final 500ms wait; loop. ## Tuning -- Per-minute cap \(N_{active} \ge 10\) gates publishing. -- Pacing (100 ms) reduces bursts. -- Backoff (1..60 s) stabilizes DHT load. +- Per-minute cap \(records \ge max\_bootstrap\_records, default 5\) gates publishing. +- Per-peer pacing (100ms) reduces bursts. +- No-peers retry (1500ms) and discovery poll (2000ms) stabilize DHT load. - Message window size (5 peers, 5 hashes) is a trade-off: - Larger window = better visibility, larger records. - Smaller window = lower bandwidth, less overlap detection. -Parameters: -- MAX_BOOTSTRAP_RECORDS -- MAX_JOIN_PEERS_COUNT -- DHT timeout -- Retry count and jitter -- Join pacing and final wait -- Publisher backoff and success jitter \ No newline at end of file +Parameters (all configurable): +- `max_bootstrap_records` (default 5) +- `max_join_peer_count` (default 4) +- `min_neighbors` for bubble merge (default 4) +- DHT timeouts, retry count, and jitter +- Bootstrap timing: no_peers_retry, per_peer_settle, join_confirmation, discovery_poll +- Publisher timing: initial_delay, base_interval, max_jitter +- Merge timing: base_interval, max_jitter (separate for bubble and overlap) diff --git a/Cargo.lock b/Cargo.lock index f024c20..bf6626b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "actor-helper" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db99f3032635124f4ad5639cfdb8fc571c84fd9c6f351e05dab7c6558ca2b157" +checksum = "e27b633e8510b190e2a968f872136b8a53b38d855089f56030ce0ead131b3377" dependencies = [ "anyhow", "flume 0.12.0", @@ -33,7 +33,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -178,16 +178,16 @@ checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.57" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "shlex", @@ -268,7 +268,7 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -368,6 +368,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -461,7 +470,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "fiat-crypto 0.2.9", "rustc_version", @@ -476,7 +485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f9200d1d13637f15a6acb71e758f64624048d85b31a5fdbfd8eca1e2687d0b7" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.11.0-rc.10", "fiat-crypto 0.3.0", @@ -676,7 +685,7 @@ dependencies = [ [[package]] name = "distributed-topic-tracker" -version = "0.2.8" +version = "0.3.0" dependencies = [ "actor-helper", "anyhow", @@ -692,6 +701,7 @@ dependencies = [ "serde", "sha2 0.10.9", "tokio", + "tokio-util", "tracing", "tracing-subscriber", ] @@ -845,9 +855,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "ff" @@ -1208,6 +1218,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heapless" version = "0.7.17" @@ -1373,18 +1389,18 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1396,7 +1412,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1404,15 +1419,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "c2b52f86d1d4bc0d6b4e6826d960b1b333217e07d36b882dca570a5e1c48895b" dependencies = [ "http", "hyper", "hyper-util", "rustls", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -1436,7 +1450,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "tokio", "tower-service", "tracing", @@ -1468,12 +1482,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1481,9 +1496,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1494,9 +1509,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1508,15 +1523,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1528,15 +1543,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1588,12 +1603,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -1609,14 +1624,15 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.5.10", + "socket2", "widestring", - "windows-sys 0.48.0", - "winreg", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] @@ -1627,9 +1643,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -1841,16 +1857,18 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1869,9 +1887,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libm" @@ -1881,9 +1899,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -1979,9 +1997,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -1990,9 +2008,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -2154,7 +2172,7 @@ dependencies = [ "objc2-system-configuration", "pin-project-lite", "serde", - "socket2 0.6.3", + "socket2", "time", "tokio", "tokio-util", @@ -2178,7 +2196,7 @@ dependencies = [ "pin-project-lite", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror", "tokio", "tokio-stream", @@ -2221,7 +2239,7 @@ checksum = "bb9be4fedd6b98f3ba82ccd3506f4d0219fb723c3f97c67e12fe1494aa020e44" dependencies = [ "cfg_aliases", "libc", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.61.2", ] @@ -2252,9 +2270,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-traits" @@ -2377,9 +2395,9 @@ dependencies = [ [[package]] name = "papaya" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f" +checksum = "997ee03cd38c01469a7046643714f0ad28880bcb9e6679ff0666e24817ca19b7" dependencies = [ "equivalent", "seize", @@ -2471,17 +2489,11 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkarr" -version = "5.0.3" +version = "5.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f950360d31be432c0c9467fba5024a94f55128e7f32bc9d32db140369f24c77" +checksum = "d7bfb9143bbba379f246211eb68074d78db9cc048e4c5701f3b0e6cb1ec67ca2" dependencies = [ "base32", "bytes", @@ -2525,7 +2537,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2537,7 +2549,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2578,9 +2590,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2659,7 +2671,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror", "tokio", "tracing", @@ -2696,7 +2708,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -2724,9 +2736,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -2849,9 +2861,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -2864,9 +2876,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "log", "once_cell", @@ -2889,9 +2901,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" dependencies = [ "ring", "rustls-pki-types", @@ -2953,9 +2965,9 @@ checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "send_wrapper" @@ -3051,7 +3063,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -3062,7 +3074,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1e3878ab0f98e35b2df35fe53201d088299b41a6bb63e3e34dada2ac4abd924" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.11.0-rc.10", ] @@ -3105,9 +3117,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple-dns" -version = "0.9.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee851d0e5e7af3721faea1843e8015e820a234f81fda3dea9247e15bac9a86a" +checksum = "df350943049174c4ae8ced56c604e28270258faec12a6a48637a7655287c9ce0" dependencies = [ "bitflags", ] @@ -3130,16 +3142,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -3184,9 +3186,9 @@ checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" [[package]] name = "spki" -version = "0.8.0-rc.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8baeff88f34ed0691978ec34440140e1572b68c7dd4a495fd14a3dc1944daa80" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", "der 0.8.0", @@ -3333,9 +3335,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -3358,25 +3360,25 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3443,18 +3445,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.1+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.5+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca1a40644a28bce036923f6a431df0b34236949d111cc07cb6dca830c9ef2e1" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", "toml_datetime", @@ -3464,9 +3466,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] @@ -3598,9 +3600,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-xid" @@ -3645,9 +3647,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -3749,9 +3751,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -3762,23 +3764,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3786,9 +3784,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -3799,9 +3797,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -3855,9 +3853,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -3993,6 +3991,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -4011,15 +4020,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -4056,21 +4056,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -4113,12 +4098,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4131,12 +4110,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4149,12 +4122,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4179,12 +4146,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4197,12 +4158,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4215,12 +4170,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4233,12 +4182,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4253,23 +4196,13 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -4360,9 +4293,9 @@ dependencies = [ [[package]] name = "wmi" -version = "0.18.3" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "003e65f4934cf9449b9ce913ad822cd054a5af669d24f93db101fdb02856bb23" +checksum = "7c81b85c57a57500e56669586496bf2abd5cf082b9d32995251185d105208b64" dependencies = [ "chrono", "futures", @@ -4375,9 +4308,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "ws_stream_wasm" @@ -4412,9 +4345,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4423,9 +4356,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -4441,18 +4374,18 @@ checksum = "2164e798d9e3d84ee2c91139ace54638059a3b23e361f5c11781c2c6459bde0f" [[package]] name = "zerocopy" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -4461,18 +4394,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -4502,9 +4435,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -4513,9 +4446,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -4524,9 +4457,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index ed523b1..5bdf301 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://rustonbsd.github.io/2025/09/03/distributed-topic-tracker.htm readme = "README.md" keywords = ["networking"] categories = ["network-programming"] -version = "0.2.8" +version = "0.3.0" edition = "2024" [features] @@ -24,6 +24,7 @@ ed25519-dalek = { version = "3.0.0-pre.1", default-features = false, features = ed25519-dalek-hpke = { version = "0.0.5" } tokio = { version = "1", default-features = false, features = ["macros", "time", "sync", "rt-multi-thread"] } +tokio-util = { version = "0.7", default-features = false } iroh = { version = "0.97", optional = true, default-features = false } iroh-gossip = { version = "0.97", optional = true, default-features = false, features = ["net"]} @@ -34,20 +35,13 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } mainline = { version = "6", default-features = false, features = ["async"]} rand = { version = "0.9", default-features = false, features = ["std", "std_rng"] } -actor-helper = { version = "0.2", features = ["tokio", "anyhow"] } +actor-helper = { version = "0.3", features = ["tokio", "anyhow"] } postcard = "1" serde = { version = "1", default-features = false, features = ["std"] } tracing = { version = "0.1", default-features = false, features = ["std"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "env-filter", "ansi"] } -[lib] -crate-type = ["cdylib", "rlib"] - -[[test]] -name = "test_gossip" -required-features = ["iroh-gossip"] - [[test]] name = "tests" @@ -71,3 +65,11 @@ required-features = ["iroh-gossip"] [[example]] name = "simple" required-features = ["iroh-gossip"] + +[[example]] +name = "full_config" +required-features = ["iroh-gossip"] + +[[example]] +name = "without_mergers" +required-features = ["iroh-gossip"] \ No newline at end of file diff --git a/PROTOCOL.md b/PROTOCOL.md index 171c5e1..3a80d19 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -1,54 +1,48 @@ # How this works: Protocol (spec) -## Publishing procedure (refined implementation) +## Publishing procedure -The publishing procedure is a rate-limited mechanism that prevents DHT overload while ensuring active participation in the gossip network. `publish_proc` function in `p01_refined.rs`: +The publishing procedure is a rate-limited mechanism that prevents DHT overload while ensuring active participation in the gossip network. Implemented in `RecordPublisher::publish_record()`: -- `MAX_BOOTSTRAP_RECORDS`: 10 (maximum active participant records allowed per unix minute) -- DHT timeout: 10 seconds for get_mutable operations -- Retry count: 3 attempts for DHT publishing operations -- Random delay: 0-2000ms between retry attempts +- `max_bootstrap_records`: 5 (default, configurable via `BootstrapConfig`) +- DHT get timeout: 10s (default, configurable via `DhtConfig`) +- DHT put retries: 3 (default, configurable via `DhtConfig`) +- DHT retry interval: 5s base + random 0-10s jitter (default, configurable via `DhtConfig`) ### Publishing Procedure 1. **Record Discovery** - - Call `get_unix_minute_records()` to fetch, decrypt, and verify existing records for the current unix minute + - Call `get_records()` to fetch, decrypt, and verify existing records for the current unix minute - Use the same key derivation as bootstrap: - - Derive signing keypair: `keypair_seed = hash(topic + unix_minute)` - - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic, unix_minute, initial_secret_hash)` - - Calculate salt: `salt = hash(topic + unix_minute)` + - Derive signing keypair: `keypair_seed = SHA512(topic_hash + unix_minute)[..32]` + - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic_hash, unix_minute, initial_secret_hash)` + - Calculate salt: `salt = SHA512("salt" + topic_hash + unix_minute)[..32]` - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout -2. **Active Participant Filtering** - - Filter records to include only "active participants" - records that have: - - Non-zero entries in `active_peers[5]` array, OR - - Non-zero entries in `last_message_hashes[5]` array - - This ensures only nodes actively participating in gossip are counted and network *Bubbles* are detected based on the `last_message_hashes` and merged. - -3. **Rate Limiting Check** - - If `active_participant_records.len() >= MAX_BOOTSTRAP_RECORDS` (10): - - **Do not publish** - return existing records to prevent DHT overload +2. **Rate Limiting Check** + - If `records.len() >= max_bootstrap_records` (default 5): + - **Do not publish** - return silently to prevent DHT overload - This implements the core rate limiting mechanism -4. **Record Creation** (if under limit) +3. **Record Creation** (if under limit) - Prepare `active_peers[5]` array: - - Fill with up to 5 current iroh-gossip neighbors + - Fill with up to 5 (`MAX_RECORD_PEERS`) current iroh-gossip neighbors - Remaining slots filled with zeros `[0; 32]` - Prepare `last_message_hashes[5]` array: - - Fill with up to 5 recent message hashes for proof of relay + - Fill with up to 5 (`MAX_MESSAGE_HASHES`) recent message hashes for proof of relay - Remaining slots filled with zeros `[0; 32]` -5. **Record Signing and Publishing** +4. **Record Signing and Publishing** - Create signed record using `Record::sign()`: - - Include: `topic_hash`, `unix_minute`, `node_id`, `active_peers`, `last_message_hashes` - - Sign with node's ed25519 signing key (`iroh::NodeId`) + - Include: `topic_hash`, `unix_minute`, `pub_key`, content (serialized `active_peers` + `last_message_hashes`) + - Sign with node's ed25519 signing key - Encrypt record using one-time encryption key - - Publish to DHT via `publish_unix_minute_record()` with 3 retry attempts + - Publish to DHT via `Dht::put_mutable()` with retry support -6. **Error Handling** - - DHT timeouts return empty record sets (non-fatal) +5. **Error Handling** + - DHT timeouts return error after retries are exhausted - Failed record decryption/verification are ignored - - Random delays between retries prevent synchronized access patterns + - DHT put retries with jittered intervals prevent synchronized access patterns ### Publishing Flow Diagram @@ -56,104 +50,104 @@ The publishing procedure is a rate-limited mechanism that prevents DHT overload flowchart TD A[Start Publishing Procedure] --> B[Get Current Unix Minute] B --> C[Derive Keys: Signing & Encryption] - C --> D[Calculate Salt: hash = topic + unix_minute] + C --> D[Calculate Salt: SHA512 = "salt" + topic_hash + unix_minute] D --> E[Query DHT: get_mutable = signing_pubkey, salt; Timeout: 10s] E --> F[Decrypt & Verify Records] - F --> G[Filter Active Participants] - G --> H{Active Records >= MAX_BOOTSTRAP_RECORDS = 10?} + F --> G{records.len >= max_bootstrap_records = default 5?} - H -- Yes --> I[Return Existing Records - Rate Limited] - H -- No --> J[Prepare Active Peers Array] + G -- Yes --> I[Return silently - Rate Limited] + G -- No --> J[Prepare Active Peers Array] J --> K[Fill active_peers with up to 5 gossip neighbors] K --> L[Prepare Last Message Hashes Array] L --> M[Fill last_message_hashes with up to 5 recent hashes] M --> N[Create Signed Record] - N --> O[Sign with: topic + unix_minute + node_id + active_peers + last_message_hashes] + N --> O[Sign with: topic + unix_minute + pub_key + content] O --> P[Encrypt Record with One-Time Key] - P --> Q[Publish to DHT with 3 Retries] + P --> Q[Publish to DHT with retries] Q --> R{Publish Success?} - R -- Yes --> S[Return All Records Including Own] - R -- No --> T[Random Delay 0-2000ms] + R -- Yes --> S[Done] + R -- No --> T[Retry with jittered interval] T --> U{Retries Left?} U -- Yes --> Q - U -- No --> V[Return Error - Failed to Publish] + U -- No --> V[Return Error] style A fill:#f4f4f4,stroke:#333,stroke-width:1px style I fill:#ffcccc,stroke:#333,stroke-width:1px style S fill:#ccffcc,stroke:#333,stroke-width:1px style V fill:#ffcccc,stroke:#333,stroke-width:1px - style G fill:#f4f4f4,stroke:#333,stroke-width:1px style N fill:#f4f4f4,stroke:#333,stroke-width:1px ``` -## Bootstrap procedure (refined implementation) +## Bootstrap procedure -The bootstrap procedure is a continuous loop that attempts to discover and connect to existing nodes in the gossip network. Here's the detailed flow based on the `p01_refined.rs` implementation: +The bootstrap procedure is a continuous loop that attempts to discover and connect to existing nodes in the gossip network. Implemented in `BootstrapActor::start_bootstrap()`: -- `MAX_JOIN_PEERS_COUNT`: maximum peers to attempt joining -- `MAX_BOOTSTRAP_RECORDS`: maximum records allowed per unix minute -- DHT timeout: 10 seconds for get_mutable operations -- Connection retry delay: 100ms between attempts -- Final connection check delay: 500ms +- `max_join_peer_count`: 4 (default, configurable via `Config`) +- `max_bootstrap_records`: 5 (default, configurable via `BootstrapConfig`) +- DHT get timeout: 10s (default, configurable via `DhtConfig`) +- No peers retry interval: 1500ms (default, configurable via `BootstrapConfig`) +- Per-peer join settle time: 100ms (default, configurable via `BootstrapConfig`) +- Final join confirmation wait: 500ms (default, configurable via `BootstrapConfig`) +- Discovery poll interval: 2000ms (default, configurable via `BootstrapConfig`) +- Publish on startup: true (default, configurable via `BootstrapConfig`) +- Check older records first on startup (`check_older_records_first_on_startup`): false (default, configurable via `BootstrapConfig`) ### Bootstrap Loop 1. **Initial Setup** - Subscribe to the gossip topic using `topic_id.hash` - - Initialize `last_published_unix_minute = 0` to track publishing state + - Optionally publish own record before the first DHT get (`publish_record_on_startup`) - Enter the main bootstrap loop 2. **Connection Check** - Check if already connected to at least one gossip peer via `gossip_receiver.is_joined()` - - If connected, return successfully with gossip sender/receiver pair + - If connected, exit bootstrap loop 3. **Time Window Selection** - - On first attempt: check previous unix minute (`unix_minute(-1)`) - - On subsequent attempts: check current unix minute (`unix_minute(0)`) - - This ensures we reliably discover existing gossip network records on the first try + - If `check_older_records_first_on_startup` is enabled and this is the first attempt: check previous `unix minute-1` and `unix_minute-2` + - Otherwise: check current unix minute `unix_minute` and `unix_minute-1` + - Two records are always fetched 4. **Record Discovery** - - Call `get_unix_minute_records()` to fetch, decrypt, and verify records: - - Derive signing keypair: `keypair_seed = hash(topic + unix_minute)` - - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic, unix_minute, initial_secret_hash)` - - Calculate salt: `salt = hash(topic + unix_minute)` + - Call `get_records()` for both `unix_minute - 1` and `unix_minute`: + - Derive signing keypair: `keypair_seed = SHA512(topic_hash + unix_minute)[..32]` + - Derive encryption keypair from shared secret + - Calculate salt: `SHA512("salt" + topic_hash + unix_minute)[..32]` - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout - Decrypt each record using the encryption keypair - Verify signature, unix_minute, and topic hash - - Filter out own records (matching node_id) + - Filter out own records (matching pub_key) 5. **If no valid Records Found** - - If no valid records found, attempt to publish own record via `publish_proc()` - - Only publish if haven't published in this unix minute - - Sleep 100ms and continue loop + - If no valid records found and haven't published in this unix minute, publish own record + - Wait for `no_peers_retry_interval` (default 1500ms) or a `joined()` event, then continue loop -6. **else if valid Records Found** +6. **If valid Records Found** - Extract bootstrap nodes from records: - - Include `record.node_id` (the publisher) + - Include `record.pub_key` (the publisher) - Include all non-zero entries from `record.active_peers[5]` - - Convert byte arrays to valid `iroh::NodeId` instances + - Convert byte arrays to valid `iroh::EndpointId` instances 7. **Connection Attempts** - Check again if already connected (someone might have connected to us) - If not connected, attempt to join peers one by one: - - Call `gossip_sender.join_peers(vec![node_id])` for each bootstrap node - - Sleep 100ms between attempts to minimize disruption + - Call `gossip_sender.join_peers(vec![pub_key])` for each bootstrap node + - Wait `per_peer_join_settle_time` (default 100ms) between attempts - Break early if connection established - - (findings showed connecting more too many nodes at once can cause the formation of netowrk *Bubbles*, isolated subnetworks that are not connected to the main network) 8. **Final Connection Verification** - - If still not connected, wait 500ms for iroh-gossip connection timeout + - If still not connected, wait `join_confirmation_wait_time` (default 500ms) - Check `gossip_receiver.is_joined()` one final time - - If connected: return successfully; spawn publisher task - - If not connected: attempt to publish own record (if not done this minute) - - Sleep 100ms and continue loop + - If connected: exit loop successfully + - If not connected: publish own record if not done this minute + - Wait `discovery_poll_interval` (default 2000ms) and continue loop ### Error Handling -- DHT timeouts return empty record sets (non-fatal) +- DHT timeouts return error after retries are exhausted - Failed record decryption/verification are treated as invalid records and ignored - Failed peer connections don't interrupt the process - Publishing failures don't prevent continued bootstrapping @@ -162,54 +156,54 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne ```mermaid flowchart TD - A[Start Bootstrap] --> B[Subscribe to Gossip Topic] - B --> C[Initialize last_published_unix_minute = 0] - C --> D{Already Connected?} - + A[Start Bootstrap] --> AA{publish_record_on_startup?} + AA -- Yes --> AB[Publish startup record] + AA -- No --> B + AB --> B[Subscribe to Gossip Topic] + B --> D{Already Connected?} - D -- Yes --> Z[Return Success; Spawn Publisher] + D -- Yes --> Z[Return Success] D -- No --> E[Determine Unix Minute] - E --> F{First attempt?} + E --> F{First attempt AND check_older_records_first?} F -- Yes --> G[Use Previous Minute unix_minute = -1] - F -- No --> H[Use Current Minute unix_minute = 0 ] + F -- No --> H[Use Current Minute unix_minute = 0] - G --> I[Get Unix Minute Records] + G --> I[Get Records for unix_minute-1 AND unix_minute] H --> I - I --> J[Derive Signing Keypair hash = topic + unix_minute ] + I --> J[Derive Signing Keypair] J --> K[Derive Encryption Keypair from shared secret] - K --> L[Calculate Salt hash = topic + unix_minute] + K --> L[Calculate Salt] L --> M[Query DHT: get_mutable = signing_pubkey, salt; Timeout: 10s] M --> N{Records Found?} N -- No --> O{Published This Minute?} - O -- No --> P[Publish Own Record via publish_proc] - O -- Yes --> Q[Sleep 100ms] + O -- No --> P[Publish Own Record] + O -- Yes --> Q[Wait no_peers_retry_interval = 1500ms or joined event] P --> Q Q --> D N -- Yes --> R[Decrypt & Verify Records] - R --> S[Filter Valid Records 1.Decrypt with encryption key 2.Verify signature 3.Check topic & unix_minute 4.Exclude own node_id] - S --> T[Extract Bootstrap Nodes 1.record.node_id 2.record.active_peers] - T --> U[Convert to iroh::NodeId] - U --> V{Already Connected?} + R --> S[Filter Valid Records] + S --> T[Extract Bootstrap Nodes: pub_key + active_peers] + T --> V{Already Connected?} V -- Yes --> Z V -- No --> W[Join Peers One by One] - W --> X[For each bootstrap node: gossip_sender.join_peers = node_id] - X --> Y[Sleep 100ms] - Y --> AA{Connected?} - AA -- Yes --> Z - AA -- No --> BB{More Nodes?} + W --> X[For each bootstrap node: join_peers = pub_key] + X --> Y[Wait per_peer_join_settle_time = 100ms] + Y --> AA2{Connected?} + AA2 -- Yes --> Z + AA2 -- No --> BB{More Nodes?} BB -- Yes --> X - BB -- No --> CC[Sleep 500ms Final connection timeout] + BB -- No --> CC[Wait join_confirmation_wait_time = 500ms] CC --> DD{Connected?} DD -- Yes --> Z DD -- No --> EE{Should Publish?} EE -- Yes --> FF[Publish Own Record] - EE -- No --> GG[Sleep 100ms] + EE -- No --> GG[Wait discovery_poll_interval = 2000ms] FF --> GG GG --> D @@ -220,131 +214,102 @@ flowchart TD style W fill:#f4f4f4,stroke:#333,stroke-width:1px ``` -## Spawn Publisher (refined implementation) +## Publisher -The Publisher is a background task that runs continuously after successful bootstrap to maintain topic presence on the DHT and detect/merge network *Bubbles*. `spawn_publisher` function in `p01_refined.rs`: +The Publisher is a separate background actor that runs after successful bootstrap to maintain topic presence on the DHT. Implemented in `PublisherActor`: -- `MAX_JOIN_PEERS_COUNT`: maximum peers to attempt joining during bubble merging -- Backoff mechanism: starts at 1 second, doubles on failure, caps at 60 seconds -- Random sleep interval: 0-60 seconds between successful publishing cycles -- Bubble detection threshold: less than 4 neighbors or non overlapping messages indicates potential network isolation +- Interval: `base_interval + random(0, max_jitter)` (default 10s base + 0-50s jitter) +- Initial delay: 10s (default, configurable via `PublisherConfig`) +- No exponential backoff; interval is constant with jitter ### Publisher Loop 1. **Initialization** - - Spawned as a background task after successful bootstrap - - Receives cloned gossip sender/receiver, topic configuration, and signing key - - Initializes exponential backoff counter starting at 1 second + - Spawned as a background actor after successful bootstrap + - Configures a ticker with `initial_delay` then repeating at `base_interval` 2. **Publishing Cycle** - - Get current unix minute: `unix_minute(0)` - - Call `publish_proc()` with live gossip data: - - Current neighbors from `gossip_receiver.neighbors()` - - Recent message hashes from `gossip_receiver.last_message_hashes()` - - This ensures published records contain real-time gossip network state + - On each tick: call `publish()` which creates a new record with current gossip state + - After publishing, reset ticker to `base_interval + random(0, max_jitter)` + - Record includes current neighbors (up to 5) and recent message hashes (up to 5) -3. **Bubble Detection and Merging** +### Publisher Flow Diagram - **Small Cluster Detection:** - - If `neighbors.len() < 4` AND valid records exist: - - Extract node IDs from `record.active_peers` in discovered records - - Filter out: zero entries, current neighbors, own node ID - - Attempt to join up to `MAX_JOIN_PEERS_COUNT` (100) new peers - - This helps merge small isolated clusters back into the main network +```mermaid +flowchart TD + A[Spawn Publisher Actor] --> B[Wait initial_delay = 10s] + B --> C[Tick] + C --> D[Get Live Gossip Data: neighbors + message_hashes] + D --> E[Create Record for current unix_minute] + E --> F[Publish via RecordPublisher::publish_record] + F --> G[Reset ticker: base_interval + random jitter] + G --> C - **Message Overlap Analysis:** - - If local node has received messages (`last_message_hashes.len() >= 1`): - - Compare local message hashes with `record.last_message_hashes` from other nodes - - Identify records with non-overlapping message sets (potential bubble or stale node indicator) - - Extract all node IDs (publisher + active_peers) from non-overlapping records - - Attempt to join these peers to merge network bubbles + style A fill:#f4f4f4,stroke:#333,stroke-width:1px +``` -4. **Error Handling and Backoff** - - On `publish_proc()` failure: - - Sleep for current backoff duration (1, 2, 4, 8, ..., 60 seconds) - - Double backoff duration, capped at 60 seconds - - Continue loop (retry publishing) +## Bubble detection and merging -5. **Success Handling** - - On successful publishing: - - Reset backoff to 1 second - - Sleep for random duration: 0-60 seconds - - Continue loop +Bubble detection and merging run as separate background actors alongside the publisher, each on their own interval timer. -### Network Bubble Detection Logic +### Bubble Merge (small cluster detection) -The publisher implements two bubble detection mechanisms: +Implemented in `BubbleMergeActor`. Interval: 60s base + 0-120s jitter (default). -1. **Cluster Size Analysis**: Small neighbor counts (< 4) suggest network fragmentation -2. **Message Flow Analysis**: Non-overlapping message hashes indicate isolated subnetworks or stale nodes +- If `neighbors.len() < min_neighbors` (default 4) AND DHT records exist: + - Extract node IDs from `record.active_peers` in discovered records + - Filter out: zero entries, current neighbors, own node ID + - Attempt to join up to `max_join_peer_count` (default 4) new peers -When bubbles are detected, the publisher proactively attempts to join peers from other network segments to restore connectivity. +### Message Overlap Merge (partition detection) -### Publisher Flow Diagram +Implemented in `MessageOverlapMergeActor`. Interval: 60s base + 0-120s jitter (default). -```mermaid -flowchart TD - A[Spawn Publisher Task] --> B[Initialize Backoff = 1s] - B --> C[Start Publisher Loop] - C --> D[Get Current Unix Minute] - D --> E[Get Live Gossip Data: neighbors + message_hashes] - E --> F[Call publish_proc = unix_minute, neighbors, message_hashes] - - F --> G{Publish Success?} - G -- No --> H[Failure] - H --> I[Sleep = backoff seconds] - I --> J[Backoff = min = backoff * 2, 60] - J --> C - - G -- Yes --> K[Bubble Detection Analysis] - K --> L{neighbors.len < 4 AND records exist?} - L -- Yes --> M[Small Cluster Detection] - M --> N[Extract active_peers from records] - N --> O[Filter: exclude zeros, current neighbors, self] - O --> P[Join up to MAX_JOIN_PEERS_COUNT peers] - P --> Q[Message Overlap Analysis] - - L -- No --> Q - Q --> R{Have local messages?} - R -- Yes --> S[Compare message_hashes with records] - S --> T{Non-overlapping messages found?} - T -- Yes --> U[Extract node_ids from non-overlapping records] - U --> V[Join peers to merge bubbles] - V --> W[Success] - - T -- No --> W - R -- No --> W - W --> X[Reset Backoff = 1s] - X --> Y[Sleep = random = 0-60s] - Y --> C +- If local node has received messages (`last_message_hashes.len() >= 1`): + - Compare local message hashes with `record.last_message_hashes` from other nodes + - Identify records with non-overlapping message sets (potential network partition) + - Extract all node IDs (publisher + active_peers) from non-overlapping records + - Attempt to join these peers to bridge partitions - style A fill:#f4f4f4,stroke:#333,stroke-width:1px - style K fill:#fff2cc,stroke:#333,stroke-width:1px - style M fill:#fff2cc,stroke:#333,stroke-width:1px - style S fill:#fff2cc,stroke:#333,stroke-width:1px - style W fill:#ccffcc,stroke:#333,stroke-width:1px - style H fill:#ffcccc,stroke:#333,stroke-width:1px +### Bubble Detection Decision Graph + +```mermaid +flowchart LR + A[Tick] --> B{neighbors < min_neighbors?} + B -- Yes --> C[Join peers from records] + B -- No --> D{local_msgs >= 1?} + D -- No --> E[Sleep until next tick] + D -- Yes --> F{overlap with others?} + F -- No --> G[Join from non-overlapping records] + F -- Yes --> E ``` -## Record structure (refined implementation) +## Record structure + +The record struct wraps a serialized `RecordContent`: ```rust -// 489 bytes total (S=5) #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Record { - // Record Content - topic: [u8; 32], // sha512(topic_string)[..32] + // Header + topic: [u8; 32], // SHA512(topic_string)[..32] unix_minute: u64, // floor(unixtime / 60) - node_id: [u8; 32], // publisher ed25519 public key - active_peers: [[u8; 32]; 5], // 5 node ids of active gossip peers - last_message_hashes: [[u8; 32]; 5], // 5 recent message hashes for proof of relay + pub_key: [u8; 32], // publisher ed25519 public key + + // Content (serialized via postcard) + content: RecordContent, // Vec wrapping serialized data + + // Signature + signature: [u8; 64], // ed25519 signature over topic + unix_minute + pub_key + content +} - // Record Signature - signature: [u8; 64], // ed25519 signature over above fields - // signed by the publisher's ed25519 private key +// Default content used by the gossip module: +pub struct GossipRecordContent { + pub active_peers: [[u8; 32]; 5], // MAX_RECORD_PEERS node ids + pub last_message_hashes: [[u8; 32]; 5], // MAX_MESSAGE_HASHES recent hashes } -// Variable size (>= 493 bytes at S=5) +// Variable size #[derive(Debug, Clone)] pub struct EncryptedRecord { encrypted_record: Vec, // encrypted Record using one-time key @@ -353,7 +318,7 @@ pub struct EncryptedRecord { } ``` -## Verification (refined implementation) +## Verification The `Record::verify()` method performs the following checks: @@ -361,8 +326,8 @@ The `Record::verify()` method performs the following checks: 2. **Time Verification**: Verify `record.unix_minute` matches the unix minute used for key derivation 3. **Signature Verification**: - Extract signature data: all record bytes except the last 64 bytes (signature) - - Signature data includes: `topic + unix_minute + node_id + active_peers + last_message_hashes` - - Verify ed25519 signature using `record.node_id` as the public key + - Signature data includes: `topic + unix_minute + pub_key + content` + - Verify ed25519 signature using `record.pub_key` as the public key - Use `verify_strict()` for enhanced security ### Additional Filtering @@ -389,7 +354,7 @@ A one-time key encryption scheme is used to protect record content while allowin **Salt Calculation:** - Purpose: Used as salt parameter for DHT mutable record storage -- Derivation: `salt = SHA512(topic_hash + unix_minute)[..32]` +- Derivation: `salt = SHA512("salt" + topic_hash + unix_minute)[..32]` ### Encryption Process @@ -419,7 +384,6 @@ A one-time key encryption scheme is used to protect record content while allowin 1. **Decrypt One-Time Key** - Derive encryption keypair from shared secret (same as encryption) - Attempt decryption: `one_time_key_bytes = encryption_keypair.decrypt(encrypted_decryption_key)` - - Fallback to previous key if rotation occurred: `last_decryption_key.decrypt(encrypted_decryption_key)` - Reconstruct one-time key: `one_time_key = ed25519_dalek::SigningKey::from_bytes(one_time_key_bytes)` 2. **Decrypt Record Data** @@ -435,14 +399,14 @@ A one-time key encryption scheme is used to protect record content while allowin ``` [4 bytes: encrypted_record_length (little-endian u32)] [variable: encrypted_record data] -[variable: encrypted_decryption_key data] +[88 bytes: encrypted_decryption_key] ``` ### Security Properties - **Forward Secrecy**: One-time keys are generated randomly for each record - **Access Control**: Only nodes with the shared secret can decrypt records -- **Key Rotation**: Supports fallback to previous encryption keys during rotation +- **Key Rotation**: Supports secret rotation via the `SecretRotation` trait - **Replay Protection**: Unix minute coupling prevents replay attacks - **Public Discovery**: DHT discovery remains public while content stays private @@ -464,11 +428,8 @@ flowchart TD J --> K[Derive Encryption Keypair from Shared Secret] K --> L[Decrypt One-Time Key] L --> M{Decryption Success?} - M -- No --> N[Try Previous Key] - N --> O{Success?} - O -- No --> P[Decryption Failed] - O -- Yes --> Q[Reconstruct One-Time Key] - M -- Yes --> Q + M -- No --> P[Decryption Failed] + M -- Yes --> Q[Reconstruct One-Time Key] Q --> R[Decrypt Record Data] R --> S[Deserialize Record] S --> T[Verify Signature & Metadata] diff --git a/README.md b/README.md index c91985c..ab6821f 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,11 @@ Add dependencies to `Cargo.toml`: anyhow = "1" tokio = "1" rand = "0.9" -iroh = "*" -iroh-gossip = "*" +ed25519-dalek = "3.0.0-pre.1" +iroh = "^0.97" +iroh-gossip = "^0.97" -distributed-topic-tracker = "0.2" +distributed-topic-tracker = "0.3" ``` Basic iroh-gossip integration: @@ -29,7 +30,7 @@ use iroh_gossip::net::Gossip; use ed25519_dalek::SigningKey; // Imports from distributed-topic-tracker -use distributed_topic_tracker::{TopicId, AutoDiscoveryGossip, RecordPublisher}; +use distributed_topic_tracker::{TopicId, AutoDiscoveryGossip, RecordPublisher, Config}; #[tokio::main] async fn main() -> Result<()> { @@ -56,10 +57,10 @@ async fn main() -> Result<()> { let initial_secret = b"my-initial-secret".to_vec(); let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); // Use new `subscribe_and_join_with_auto_discovery` on Gossip @@ -111,15 +112,183 @@ Verify peer discovery across Docker containers: The E2E test confirms multiple nodes discover each other via DHT and join the same gossip topic. -## Roadmap +## Upgrading from 0.2 to 0.3 -- [x] Finalize crate name and publish to crates.io -- [x] Tests and CI -- [x] Add more examples -- [x] Optimize configuration -- [x] Major refactor -- [x] Make `iroh-gossip` integration a feature (repurposed for rustpatcher) -- [x] API docs +**0.3** resolves several **stability issues** present in **0.2**. Circular and dangling references between actors caused resource leaks and **tasks could outlive topic and channel handles** (after `Topic`, `GossipSender` and `GossipReceiver` were dropped). Reduced unnecessary DHT writes and reads, adjusted timeouts, **reduced time to bootstrap**. All actor lifecycles are now token-gated, references now work as expected (if all dropped, all background tasks shut down gracefully), resolved bugs in merge workers, and many more improvements. If you find any issues, please report them. + +**tldr: background tasks shutdown as expected, faster bootstrap time, better all around** + +### Breaking changes + +**`RecordTopic` removed, use `TopicId` instead** + +`RecordTopic` has been removed. `TopicId` now serves as the unified topic identifier across the entire API and supports conversion from `&str`, `String`, `Vec`, and `FromStr`. + +```rust,ignore +// 0.2 +use distributed_topic_tracker::RecordTopic; +let topic = RecordTopic::from_str("my-topic")?; +let publisher = RecordPublisher::new(topic, signing_key, None, secret); + +// 0.3 +use distributed_topic_tracker::TopicId; +let topic = TopicId::new("my-topic".to_string()); +// or: let topic: TopicId = "my-topic".into(); +// or: let topic: TopicId = "my-topic".parse()?; +let publisher = RecordPublisher::new(topic, signing_key, None, secret, Config::default()); +``` + +**`RecordPublisher::new()` now requires a `Config` parameter** + +A 5th `Config` parameter was added. Use `Config::default()` for most use cases, tune as needed. + +```rust,ignore +// 0.2 +let publisher = RecordPublisher::new(topic, pub_key, signing_key, None, secret); + +// 0.3 +use distributed_topic_tracker::Config; +let publisher = RecordPublisher::new(topic, signing_key, None, secret, Config::default()); + +// or use the new builder: +let publisher = RecordPublisher::builder(topic_id, signing_key, initial_secret) + .config( + Config::builder() + .max_join_peer_count(4) + .build(), + ) + .build(); +``` + +**`MAX_BOOTSTRAP_RECORDS` constant removed** + +The per-minute record cap is now configurable via `BootstrapConfig::max_bootstrap_records` (default: 5, was hardcoded 100). + +```rust,ignore +// 0.2 +use distributed_topic_tracker::MAX_BOOTSTRAP_RECORDS; // was 100 + +// 0.3 - configure via Config +let config = Config::builder() + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(10) + .build() + ) + .build(); +``` + +**`TopicId::raw()` removed** + +`TopicId` no longer stores the original string. Only the 32-byte hash is retained. + +```rust,ignore +// 0.2 +let topic = TopicId::new("my-topic".to_string()); +let original: &str = topic.raw(); // no longer available + +// 0.3 - store the raw string yourself if needed +let raw = "my-topic".to_string(); +let topic = TopicId::new(raw.clone()); +``` + +### New features + +**Full configuration system** + +All timing, retry, and threshold parameters are now configurable: + +```rust,no_run +use std::time::Duration; +use distributed_topic_tracker::{ + Config, DhtConfig, BootstrapConfig, PublisherConfig, MergeConfig, + BubbleMergeConfig, MessageOverlapMergeConfig, TimeoutConfig, +}; + +Config::builder() + .dht_config( + DhtConfig::builder() + .retries(3) + .base_retry_interval(Duration::from_secs(5)) + .max_retry_jitter(Duration::from_secs(10)) + .get_timeout(Duration::from_secs(10)) + .put_timeout(Duration::from_secs(10)) + .build(), + ) + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(5) + .publish_record_on_startup(true) + .check_older_records_first_on_startup(false) + .discovery_poll_interval(Duration::from_millis(2000)) + .no_peers_retry_interval(Duration::from_millis(1500)) + .per_peer_join_settle_time(Duration::from_millis(100)) + .join_confirmation_wait_time(Duration::from_millis(500)) + .build(), + ) + .max_join_peer_count(4) + .publisher_config( + PublisherConfig::builder() + .initial_delay(Duration::from_secs(10)) + .base_interval(Duration::from_secs(10)) + .max_jitter(Duration::from_secs(50)) + .build(), + ) + .merge_config( + MergeConfig::builder() + .bubble_merge( + BubbleMergeConfig::builder() + .min_neighbors(4) + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) + .message_overlap_merge( + MessageOverlapMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) + .build(), + ) + .timeouts( + TimeoutConfig::builder() + .join_peer_timeout(Duration::from_secs(5)) + .broadcast_neighbors_timeout(Duration::from_secs(5)) + .broadcast_timeout(Duration::from_secs(5)) + .build(), + ) + .build(); +``` + +**Disable merge strategies or publishing** + +```rust,ignore +// Run without any merge strategies (bootstrap-only) +let config = Config::builder() + .merge_config( + MergeConfig::builder() + .bubble_merge(BubbleMergeConfig::Disabled) + .message_overlap_merge(MessageOverlapMergeConfig::Disabled) + .build(), + ) + .build(); +``` + +**`RecordPublisher::builder()` for ergonomic construction** + +```rust,ignore +let publisher = RecordPublisher::builder("my-topic", signing_key, b"secret") + .secret_rotation(rotation_handle) + .config(config) + .build(); +``` + +## Todo's + +- [ ] Network degradation testing ## License diff --git a/examples/chat.rs b/examples/chat.rs index 5023adb..59b6e0f 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -5,7 +5,9 @@ use iroh_gossip::{api::Event, net::Gossip}; use ed25519_dalek::SigningKey; // Imports from distributed-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +use distributed_topic_tracker::{ + AutoDiscoveryGossip, BootstrapConfig, Config, RecordPublisher, TopicId, +}; #[tokio::main] async fn main() -> Result<()> { @@ -44,14 +46,20 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::builder() + .bootstrap_config( + BootstrapConfig::builder() + .check_older_records_first_on_startup(false) + .build(), + ) + .build(), ); // Split into sink (sending) and stream (receiving) - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await? .split() @@ -61,7 +69,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", @@ -72,6 +80,7 @@ async fn main() -> Result<()> { println!("\nJoined by {}", &peer.to_string()[0..8]); } } + println!("\nGossip receiver stream ended"); }); // Main input loop for sending messages @@ -81,7 +90,12 @@ async fn main() -> Result<()> { print!("\n> "); stdin.read_line(&mut buffer).unwrap(); gossip_sender - .broadcast(buffer.clone().replace("\n", "").into()) + .broadcast( + buffer + .trim_end_matches(&['\r', '\n'][..]) + .as_bytes() + .to_vec(), + ) .await .unwrap(); println!(" - (sent)"); diff --git a/examples/chat_no_wait.rs b/examples/chat_no_wait.rs index 11b2022..e599c49 100644 --- a/examples/chat_no_wait.rs +++ b/examples/chat_no_wait.rs @@ -2,12 +2,23 @@ use anyhow::Result; use iroh::{Endpoint, SecretKey}; use iroh_gossip::{api::Event, net::Gossip}; -// Imports from distrubuted-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; -use mainline::SigningKey; +use ed25519_dalek::SigningKey; + +// Imports from distributed-topic-tracker +use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; +use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_thread_ids(true) + .with_ansi(true) + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("distributed_topic_tracker=debug")), + ) + .init(); + // Generate a new random secret key let secret_key = SecretKey::generate(&mut rand::rng()); let signing_key = SigningKey::from_bytes(&secret_key.to_bytes()); @@ -31,12 +42,12 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery_no_wait(record_publisher) .await? .split() @@ -46,7 +57,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", @@ -57,6 +68,7 @@ async fn main() -> Result<()> { println!("\nJoined by {}", &peer.to_string()[0..8]); } } + println!("\nGossip receiver stream ended"); }); // Main input loop for sending messages @@ -66,7 +78,12 @@ async fn main() -> Result<()> { print!("\n> "); stdin.read_line(&mut buffer).unwrap(); gossip_sender - .broadcast(buffer.clone().replace("\n", "").into()) + .broadcast( + buffer + .trim_end_matches(&['\r', '\n'][..]) + .as_bytes() + .to_vec(), + ) .await .unwrap(); println!(" - (sent)"); diff --git a/examples/e2e_test.rs b/examples/e2e_test.rs index d356e2e..a3938b9 100644 --- a/examples/e2e_test.rs +++ b/examples/e2e_test.rs @@ -4,8 +4,8 @@ use iroh::{ }; use iroh_gossip::net::Gossip; -// Imports from distrubuted-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +// Imports from distributed-topic-tracker +use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; #[tokio::main] async fn main() -> Result<()> { @@ -32,21 +32,22 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await? .split() .await?; tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { println!("event: {event:?}"); } + println!("\nGossip receiver stream ended"); }); tokio::time::sleep(std::time::Duration::from_secs(3)).await; diff --git a/examples/full_config.rs b/examples/full_config.rs new file mode 100644 index 0000000..0bf54a8 --- /dev/null +++ b/examples/full_config.rs @@ -0,0 +1,116 @@ +use std::time::Duration; + +use anyhow::Result; +use ed25519_dalek::SigningKey; +use iroh::{Endpoint, SecretKey}; +use iroh_gossip::net::Gossip; + +// Imports from distributed-topic-tracker +use distributed_topic_tracker::{ + AutoDiscoveryGossip, BootstrapConfig, BubbleMergeConfig, Config, DefaultSecretRotation, + DhtConfig, MergeConfig, MessageOverlapMergeConfig, PublisherConfig, RecordPublisher, + RotationHandle, TimeoutConfig, TopicId, +}; + +// Default configuration, all in one spot as an overview +// same as `Config::default()` +fn config_builder() -> Config { + Config::builder() + .dht_config( + DhtConfig::builder() + .retries(3) + .base_retry_interval(Duration::from_secs(5)) + .max_retry_jitter(Duration::from_secs(10)) + .get_timeout(Duration::from_secs(10)) + .put_timeout(Duration::from_secs(10)) + .build(), + ) + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(5) + .publish_record_on_startup(true) + .check_older_records_first_on_startup(false) + .discovery_poll_interval(Duration::from_millis(2000)) + .no_peers_retry_interval(Duration::from_millis(1500)) + .per_peer_join_settle_time(Duration::from_millis(100)) + .join_confirmation_wait_time(Duration::from_millis(500)) + .build(), + ) + .max_join_peer_count(4) + .publisher_config( + PublisherConfig::builder() + .initial_delay(Duration::from_secs(10)) + .base_interval(Duration::from_secs(10)) + .max_jitter(Duration::from_secs(50)) + .build(), + ) + .merge_config( + MergeConfig::builder() + .bubble_merge( + BubbleMergeConfig::builder() + .min_neighbors(4) + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) + .message_overlap_merge( + MessageOverlapMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) + .build(), + ) + .timeouts( + TimeoutConfig::builder() + .join_peer_timeout(Duration::from_secs(5)) + .broadcast_neighbors_timeout(Duration::from_secs(5)) + .broadcast_timeout(Duration::from_secs(5)) + .build(), + ) + .build() +} + +#[tokio::main] +async fn main() -> Result<()> { + // Generate a new random secret key + let secret_key = SecretKey::generate(&mut rand::rng()); + let signing_key = SigningKey::from_bytes(&secret_key.to_bytes()); + + // Set up endpoint with discovery enabled + let endpoint = Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await?; + + // Initialize gossip with auto-discovery + let gossip = Gossip::builder().spawn(endpoint.clone()); + + // Set up protocol router + let _router = iroh::protocol::Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .spawn(); + + let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + let record_publisher = + RecordPublisher::builder(topic_id.clone(), signing_key.clone(), initial_secret) + .config(config_builder()) + .secret_rotation(RotationHandle::new(DefaultSecretRotation)) + .build(); + + let topic = gossip + .subscribe_and_join_with_auto_discovery(record_publisher) + .await?; + + println!("[joined topic]"); + + // Do something with the gossip topic + // (bonus: GossipSender and GossipReceiver are safely clonable) + let (_gossip_sender, _gossip_receiver) = topic.split().await?; + + Ok(()) +} diff --git a/examples/secret_rotation.rs b/examples/secret_rotation.rs index 25f7068..76b4e64 100644 --- a/examples/secret_rotation.rs +++ b/examples/secret_rotation.rs @@ -4,9 +4,9 @@ use iroh::{Endpoint, SecretKey}; use iroh_gossip::{api::Event, net::Gossip}; use sha2::Digest; -// Imports from distrubuted-topic-tracker +// Imports from distributed-topic-tracker use distributed_topic_tracker::{ - AutoDiscoveryGossip, RecordPublisher, RotationHandle, SecretRotation, TopicId, + AutoDiscoveryGossip, Config, RecordPublisher, RotationHandle, SecretRotation, TopicId }; struct MySecretRotation; @@ -49,16 +49,14 @@ async fn main() -> Result<()> { let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - // Split into sink (sending) and stream (receiving) - let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), Some(RotationHandle::new(MySecretRotation)), initial_secret, + Config::default(), ); - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await? .split() @@ -68,7 +66,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", @@ -79,6 +77,7 @@ async fn main() -> Result<()> { println!("\nJoined by {}", &peer.to_string()[0..8]); } } + println!("\nGossip receiver stream ended"); }); // Main input loop for sending messages diff --git a/examples/simple.rs b/examples/simple.rs index 79b4354..976b1c5 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -3,8 +3,8 @@ use ed25519_dalek::SigningKey; use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; -// Imports from distrubuted-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +// Imports from distributed-topic-tracker +use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; #[tokio::main] async fn main() -> Result<()> { @@ -29,14 +29,12 @@ async fn main() -> Result<()> { let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - // Split into sink (sending) and stream (receiving) - let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); let topic = gossip diff --git a/examples/without_mergers.rs b/examples/without_mergers.rs new file mode 100644 index 0000000..3152523 --- /dev/null +++ b/examples/without_mergers.rs @@ -0,0 +1,62 @@ +use anyhow::Result; +use ed25519_dalek::SigningKey; +use iroh::{Endpoint, SecretKey}; +use iroh_gossip::net::Gossip; + +// Imports from distributed-topic-tracker +use distributed_topic_tracker::{ + AutoDiscoveryGossip, BubbleMergeConfig, Config, MergeConfig, MessageOverlapMergeConfig, + RecordPublisher, TopicId, +}; + +#[tokio::main] +async fn main() -> Result<()> { + // Generate a new random secret key + let secret_key = SecretKey::generate(&mut rand::rng()); + let signing_key = SigningKey::from_bytes(&secret_key.to_bytes()); + + // Set up endpoint with discovery enabled + let endpoint = Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await?; + + // Initialize gossip with auto-discovery + let gossip = Gossip::builder().spawn(endpoint.clone()); + + // Set up protocol router + let _router = iroh::protocol::Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .spawn(); + + let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + let record_publisher = RecordPublisher::new( + topic_id.clone(), + signing_key.clone(), + None, + initial_secret, + // [!] Disable merge workers (BubbleMerge and MessageOverlapMerge) + Config::builder() + .merge_config( + MergeConfig::builder() + .bubble_merge(BubbleMergeConfig::Disabled) + .message_overlap_merge(MessageOverlapMergeConfig::Disabled) + .build(), + ) + .build(), + ); + + let topic = gossip + .subscribe_and_join_with_auto_discovery(record_publisher) + .await?; + + println!("[joined topic]"); + + // Do something with the gossip topic + // (bonus: GossipSender and GossipReceiver are safely clonable) + let (_gossip_sender, _gossip_receiver) = topic.split().await?; + + Ok(()) +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..671fe91 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,977 @@ +use std::time::Duration; + +/// Timeout settings for gossip operations. +#[derive(Debug, Clone)] +pub struct TimeoutConfig { + join_peer_timeout: Duration, + broadcast_timeout: Duration, + broadcast_neighbors_timeout: Duration, +} + +impl TimeoutConfig { + /// Create a new `TimeoutConfigBuilder` with default values. + pub fn builder() -> TimeoutConfigBuilder { + TimeoutConfigBuilder { + timeouts: TimeoutConfig::default(), + } + } + + /// How long to wait when joining a peer. + /// + /// Default: 5s. + pub fn join_peer_timeout(&self) -> Duration { + self.join_peer_timeout + } + + /// How long to wait when broadcasting messages. + /// + /// Default: 5s. + pub fn broadcast_timeout(&self) -> Duration { + self.broadcast_timeout + } + + /// How long to wait when broadcasting to neighbors. + /// + /// Default: 5s. + pub fn broadcast_neighbors_timeout(&self) -> Duration { + self.broadcast_neighbors_timeout + } +} + +impl Default for TimeoutConfig { + fn default() -> Self { + Self { + join_peer_timeout: Duration::from_secs(5), + broadcast_timeout: Duration::from_secs(5), + broadcast_neighbors_timeout: Duration::from_secs(5), + } + } +} + +/// Builder for `TimeoutConfig`. +#[derive(Debug)] +pub struct TimeoutConfigBuilder { + timeouts: TimeoutConfig, +} + +impl TimeoutConfigBuilder { + /// How long to wait when joining a peer. + /// + /// Default: 5s. + pub fn join_peer_timeout(mut self, timeout: Duration) -> Self { + self.timeouts.join_peer_timeout = timeout; + self + } + + /// How long to wait when broadcasting messages. + /// + /// Default: 5s. + pub fn broadcast_timeout(mut self, timeout: Duration) -> Self { + self.timeouts.broadcast_timeout = timeout; + self + } + + /// How long to wait when broadcasting to neighbors. + /// + /// Default: 5s. + pub fn broadcast_neighbors_timeout(mut self, timeout: Duration) -> Self { + self.timeouts.broadcast_neighbors_timeout = timeout; + self + } + + /// Build the `TimeoutConfig`. + pub fn build(self) -> TimeoutConfig { + self.timeouts + } +} + +/// DHT operation settings including retry logic and timeouts. +#[derive(Debug, Clone)] +pub struct DhtConfig { + retries: usize, + base_retry_interval: Duration, + max_retry_jitter: Duration, + put_timeout: Duration, + get_timeout: Duration, +} + +/// Builder for `DhtConfig`. +#[derive(Debug, Clone)] +pub struct DhtConfigBuilder { + config: DhtConfig, +} + +impl DhtConfigBuilder { + /// Number of retries after the initial attempt. + /// + /// Total attempts = 1 + retries. + /// + /// Default: 3. + pub fn retries(mut self, retries: usize) -> Self { + self.config.retries = retries; + self + } + + /// Base delay between retries. No-op if `interval` is `Duration::ZERO`. + /// + /// If `base_retry_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_retry_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 5s. + pub fn base_retry_interval(mut self, interval: Duration) -> Self { + if interval > Duration::ZERO { + self.config.base_retry_interval = interval; + } + self + } + + /// Max random jitter added to retry interval. + /// + /// Default: 10s. + pub fn max_retry_jitter(mut self, jitter: Duration) -> Self { + self.config.max_retry_jitter = jitter; + self + } + + /// Timeout for DHT put operations. No-op if `timeout` is `Duration::ZERO`. + /// + /// If `put_timeout` is called only once with `Duration::ZERO`, default value prevails. + /// If `put_timeout` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 10s. + pub fn put_timeout(mut self, timeout: Duration) -> Self { + if timeout > Duration::ZERO { + self.config.put_timeout = timeout; + } + self + } + + /// Timeout for DHT get operations. No-op if `timeout` is `Duration::ZERO`. + /// + /// If `get_timeout` is called only once with `Duration::ZERO`, default value prevails. + /// If `get_timeout` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 10s. + pub fn get_timeout(mut self, timeout: Duration) -> Self { + if timeout > Duration::ZERO { + self.config.get_timeout = timeout; + } + self + } + + /// Build the `DhtConfig`. + pub fn build(self) -> DhtConfig { + self.config + } +} + +impl DhtConfig { + /// Create a new `DhtConfigBuilder` with default values. + pub fn builder() -> DhtConfigBuilder { + DhtConfigBuilder { + config: DhtConfig::default(), + } + } + + /// Number of retries after the initial attempt. + /// + /// Total attempts = 1 + retries. + /// + /// Default: 3. + pub fn retries(&self) -> usize { + self.retries + } + + /// Base delay between retries. + /// + /// Default: 5s. + pub fn base_retry_interval(&self) -> Duration { + self.base_retry_interval + } + + /// Max random jitter added to retry interval. + /// + /// Default: 10s. + pub fn max_retry_jitter(&self) -> Duration { + self.max_retry_jitter + } + + /// Timeout for DHT put operations. + /// + /// Default: 10s. + pub fn put_timeout(&self) -> Duration { + self.put_timeout + } + + /// Timeout for DHT get operations. + /// + /// Default: 10s. + pub fn get_timeout(&self) -> Duration { + self.get_timeout + } +} + +impl Default for DhtConfig { + fn default() -> Self { + Self { + retries: 3, + base_retry_interval: Duration::from_secs(5), + max_retry_jitter: Duration::from_secs(10), + put_timeout: Duration::from_secs(10), + get_timeout: Duration::from_secs(10), + } + } +} + +/// Bubble merge strategy config for detecting and healing split-brain scenarios by joining small clusters with peers advertised in DHT that are not our neighbors. +#[derive(Debug, Clone)] +pub enum BubbleMergeConfig { + Enabled(BubbleMergeConfigInner), + Disabled, +} + +#[derive(Debug, Clone)] +pub struct BubbleMergeConfigInner { + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, + fail_topic_creation_on_merge_startup_failure: bool, +} + +#[derive(Debug, Clone)] +pub struct BubbleMergeConfigBuilder { + config: BubbleMergeConfigInner, +} + +impl Default for BubbleMergeConfig { + fn default() -> Self { + Self::Enabled(BubbleMergeConfigInner::default()) + } +} + +impl Default for BubbleMergeConfigInner { + fn default() -> Self { + Self { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + min_neighbors: 4, + fail_topic_creation_on_merge_startup_failure: true, + } + } +} + +impl BubbleMergeConfig { + /// Create a new `BubbleMergeConfigBuilder` with default values. + pub fn builder() -> BubbleMergeConfigBuilder { + BubbleMergeConfigBuilder { + config: BubbleMergeConfigInner::default(), + } + } +} + +impl BubbleMergeConfigInner { + /// Base interval for bubble merge attempts. + /// + /// `base_interval` > Duration::ZERO + /// + /// Default: 60s. + pub fn base_interval(&self) -> Duration { + self.base_interval + } + + /// Max random jitter added to bubble merge interval. + /// + /// Default: 120s. + pub fn max_jitter(&self) -> Duration { + self.max_jitter + } + + /// Minimum number of neighbors required to attempt a bubble merge. + /// + /// Default: 4. + pub fn min_neighbors(&self) -> usize { + self.min_neighbors + } + + /// Whether to fail topic creation + /// + /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(&self) -> bool { + self.fail_topic_creation_on_merge_startup_failure + } +} + +impl BubbleMergeConfigBuilder { + /// Base interval for bubble merge attempts. No-op if `interval` is `Duration::ZERO`. + /// + /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 60s. + pub fn base_interval(mut self, interval: Duration) -> Self { + if interval > Duration::ZERO { + self.config.base_interval = interval; + } + self + } + + /// Max random jitter added to bubble merge interval. + /// + /// Default: 120s + pub fn max_jitter(mut self, jitter: Duration) -> Self { + self.config.max_jitter = jitter; + self + } + + /// Minimum number of neighbors required to attempt a bubble merge. + /// + /// Default: 4. + pub fn min_neighbors(mut self, min_neighbors: usize) -> Self { + self.config.min_neighbors = min_neighbors; + self + } + + /// Whether to fail topic creation + /// + /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { + self.config.fail_topic_creation_on_merge_startup_failure = fail; + self + } + + /// Build the `BubbleMergeConfig`. + pub fn build(self) -> BubbleMergeConfig { + BubbleMergeConfig::Enabled(self.config) + } +} + +/// Message overlap merge strategy config for detecting and healing split-brain scenarios by checking for overlapping message hashes with other cluster peers. +#[derive(Debug, Clone)] +pub enum MessageOverlapMergeConfig { + Enabled(MessageOverlapMergeConfigInner), + Disabled, +} + +#[derive(Debug, Clone)] +pub struct MessageOverlapMergeConfigInner { + base_interval: Duration, + max_jitter: Duration, + fail_topic_creation_on_merge_startup_failure: bool, +} + +#[derive(Debug, Clone)] +pub struct MessageOverlapMergeConfigBuilder { + config: MessageOverlapMergeConfigInner, +} + +impl Default for MessageOverlapMergeConfigInner { + fn default() -> Self { + Self { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + fail_topic_creation_on_merge_startup_failure: true, + } + } +} + +impl Default for MessageOverlapMergeConfig { + fn default() -> Self { + Self::Enabled(MessageOverlapMergeConfigInner::default()) + } +} + +impl MessageOverlapMergeConfig { + /// Create a new `MessageOverlapMergeConfigBuilder` with default values. + pub fn builder() -> MessageOverlapMergeConfigBuilder { + MessageOverlapMergeConfigBuilder { + config: MessageOverlapMergeConfigInner::default(), + } + } +} + +impl MessageOverlapMergeConfigInner { + /// Base interval for message overlap merge attempts. + /// + /// `base_interval` > Duration::ZERO + /// + /// Default: 60s. + pub fn base_interval(&self) -> Duration { + self.base_interval + } + + /// Max random jitter added to message overlap merge interval. + /// + /// Default: 120s. + pub fn max_jitter(&self) -> Duration { + self.max_jitter + } + + /// Whether to fail topic creation + /// + /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(&self) -> bool { + self.fail_topic_creation_on_merge_startup_failure + } +} + +impl MessageOverlapMergeConfigBuilder { + /// Base interval for message overlap merge attempts. No-op if `interval` is `Duration::ZERO`. + /// + /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 60s. + pub fn base_interval(mut self, interval: Duration) -> Self { + if interval > Duration::ZERO { + self.config.base_interval = interval; + } + self + } + + /// Max random jitter added to message overlap merge interval. + /// + /// Default: 120s. Minimum is 0s. + pub fn max_jitter(mut self, jitter: Duration) -> Self { + self.config.max_jitter = jitter; + self + } + + /// Whether to fail topic creation + /// + /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { + self.config.fail_topic_creation_on_merge_startup_failure = fail; + self + } + + /// Build the `MessageOverlapMergeConfig`. + pub fn build(self) -> MessageOverlapMergeConfig { + MessageOverlapMergeConfig::Enabled(self.config) + } +} + +/// Publisher strategy config for publishing bootstrap records to DHT for peer discovery. +#[derive(Debug, Clone)] +pub enum PublisherConfig { + Enabled(PublisherConfigInner), + Disabled, +} + +#[derive(Debug, Clone)] +pub struct PublisherConfigInner { + initial_delay: Duration, + base_interval: Duration, + max_jitter: Duration, + fail_topic_creation_on_publishing_startup_failure: bool, +} + +#[derive(Debug, Clone)] +pub struct PublisherConfigBuilder { + config: PublisherConfigInner, +} + +impl Default for PublisherConfigInner { + fn default() -> Self { + Self { + initial_delay: Duration::from_secs(10), + base_interval: Duration::from_secs(10), + max_jitter: Duration::from_secs(50), + fail_topic_creation_on_publishing_startup_failure: true, + } + } +} + +impl Default for PublisherConfig { + fn default() -> Self { + Self::Enabled(PublisherConfigInner::default()) + } +} + +impl PublisherConfig { + /// Create a new `PublisherConfigBuilder` with default values. + pub fn builder() -> PublisherConfigBuilder { + PublisherConfigBuilder { + config: PublisherConfigInner::default(), + } + } +} + +impl PublisherConfigInner { + /// Initial delay before starting publisher. + /// + /// Default: 10s. + pub fn initial_delay(&self) -> Duration { + self.initial_delay + } + + /// Base interval for publisher attempts. + /// + /// `base_interval` > Duration::ZERO + /// + /// Default: 10s. + pub fn base_interval(&self) -> Duration { + self.base_interval + } + + /// Max random jitter added to publisher interval. + /// + /// Default: 50s. + pub fn max_jitter(&self) -> Duration { + self.max_jitter + } + + /// Whether to fail topic creation + /// + /// If a publisher startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_publishing_startup_failure(&self) -> bool { + self.fail_topic_creation_on_publishing_startup_failure + } +} + +impl PublisherConfigBuilder { + /// Initial delay before starting publisher. + /// + /// Default: 10s. + pub fn initial_delay(mut self, delay: Duration) -> Self { + self.config.initial_delay = delay; + self + } + + /// Base interval for publisher attempts. No-op if `interval` is `Duration::ZERO`. + /// + /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 10s. + pub fn base_interval(mut self, interval: Duration) -> Self { + if interval > Duration::ZERO { + self.config.base_interval = interval; + } + self + } + + /// Max random jitter added to publisher interval. + /// + /// Default: 50s. + pub fn max_jitter(mut self, jitter: Duration) -> Self { + self.config.max_jitter = jitter; + self + } + + /// Whether to fail topic creation + /// + /// If a publisher startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_publishing_startup_failure(mut self, fail: bool) -> Self { + self.config + .fail_topic_creation_on_publishing_startup_failure = fail; + self + } + + /// Build the `PublisherConfig`. + pub fn build(self) -> PublisherConfig { + PublisherConfig::Enabled(self.config) + } +} + +/// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records +/// and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes +/// for message overlap detection and merging. +#[derive(Debug, Clone, Default)] +pub struct MergeConfig { + bubble_merge: BubbleMergeConfig, + message_overlap_merge: MessageOverlapMergeConfig, +} + +/// Builder for `MergeConfig`. +#[derive(Debug, Clone)] +pub struct MergeConfigBuilder { + config: MergeConfig, +} + +impl MergeConfig { + /// Create a new `MergeConfigBuilder` with default values. + pub fn builder() -> MergeConfigBuilder { + MergeConfigBuilder { + config: MergeConfig::default(), + } + } + + /// Bubble merge strategy config. + /// + /// Default: BubbleMergeConfig::default() + pub fn bubble_merge(&self) -> &BubbleMergeConfig { + &self.bubble_merge + } + + /// Message overlap merge strategy config. + /// + /// Default: MessageOverlapMergeConfig::default() + pub fn message_overlap_merge(&self) -> &MessageOverlapMergeConfig { + &self.message_overlap_merge + } +} + +impl MergeConfigBuilder { + /// Bubble merge strategy config. + /// + /// Default: BubbleMergeConfig::default() + pub fn bubble_merge(mut self, bubble_merge: BubbleMergeConfig) -> Self { + self.config.bubble_merge = bubble_merge; + self + } + + /// Message overlap merge strategy config. + /// + /// Default: MessageOverlapMergeConfig::default() + pub fn message_overlap_merge( + mut self, + message_overlap_merge: MessageOverlapMergeConfig, + ) -> Self { + self.config.message_overlap_merge = message_overlap_merge; + self + } + + /// Build the `MergeConfig`. + pub fn build(self) -> MergeConfig { + self.config + } +} + +/// Bootstrap process settings for peer discovery. +#[derive(Debug, Clone)] +pub struct BootstrapConfig { + max_bootstrap_records: usize, + no_peers_retry_interval: Duration, + per_peer_join_settle_time: Duration, + join_confirmation_wait_time: Duration, + discovery_poll_interval: Duration, + publish_record_on_startup: bool, + check_older_records_first_on_startup: bool, +} + +impl Default for BootstrapConfig { + fn default() -> Self { + Self { + max_bootstrap_records: 5, + no_peers_retry_interval: Duration::from_millis(1500), + per_peer_join_settle_time: Duration::from_millis(100), + join_confirmation_wait_time: Duration::from_millis(500), + discovery_poll_interval: Duration::from_millis(2000), + publish_record_on_startup: true, + check_older_records_first_on_startup: false, + } + } +} + +/// Builder for `BootstrapConfig`. +#[derive(Debug)] +pub struct BootstrapConfigBuilder { + config: BootstrapConfig, +} + +impl BootstrapConfigBuilder { + /// Max bootstrap records per topic per minute slot. + /// + /// If zero, we don't publish (PublisherConfig will be set to Disabled). + /// + /// Default: 5. + pub fn max_bootstrap_records(mut self, max_records: usize) -> Self { + self.config.max_bootstrap_records = max_records; + self + } + + /// How long to wait when no peers are found before retrying. + /// + /// Default: 1500ms. + pub fn no_peers_retry_interval(mut self, interval: Duration) -> Self { + self.config.no_peers_retry_interval = interval; + self + } + + /// How long to wait after joining a peer before attempting to join another. + /// + /// Default: 100ms. + pub fn per_peer_join_settle_time(mut self, interval: Duration) -> Self { + self.config.per_peer_join_settle_time = interval; + self + } + + /// How long to wait after joining a peer before checking if joined successfully. + /// + /// Default: 500ms. + pub fn join_confirmation_wait_time(mut self, interval: Duration) -> Self { + self.config.join_confirmation_wait_time = interval; + self + } + + /// How long to wait between DHT discovery attempts. + /// + /// Default: 2000ms. + pub fn discovery_poll_interval(mut self, interval: Duration) -> Self { + self.config.discovery_poll_interval = interval; + self + } + + /// Whether to publish a bootstrap record unconditionally on startup before dht get. + /// + /// Default: true. + pub fn publish_record_on_startup(mut self, publish: bool) -> Self { + self.config.publish_record_on_startup = publish; + self + } + + /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. + /// + /// If this is enabled, we first fetch `unix_minute-1` and `unix_minute-2`. + /// + /// If joining longer running, existing topics is priority, set to true. + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly + /// at the same time into a topic without peers), set to false. + /// + /// Default: false. + pub fn check_older_records_first_on_startup(mut self, check: bool) -> Self { + self.config.check_older_records_first_on_startup = check; + self + } + + /// Build the `BootstrapConfig`. + pub fn build(self) -> BootstrapConfig { + self.config + } +} + +impl BootstrapConfig { + /// Create a new `BootstrapConfigBuilder` with default values. + pub fn builder() -> BootstrapConfigBuilder { + BootstrapConfigBuilder { + config: BootstrapConfig::default(), + } + } + + /// Max bootstrap records per topic per minute slot. + /// + /// If zero, we don't publish (PublisherConfig will be set to Disabled). + /// + /// Default: 5. + pub fn max_bootstrap_records(&self) -> usize { + self.max_bootstrap_records + } + + /// How long to wait when no peers are found before retrying. + /// + /// Default: 1500ms. + pub fn no_peers_retry_interval(&self) -> Duration { + self.no_peers_retry_interval + } + + /// How long to wait after joining a peer before attempting to join another. + /// + /// Default: 100ms. + pub fn per_peer_join_settle_time(&self) -> Duration { + self.per_peer_join_settle_time + } + + /// How long to wait after joining a peer before checking if joined successfully. + /// + /// Default: 500ms. + pub fn join_confirmation_wait_time(&self) -> Duration { + self.join_confirmation_wait_time + } + + /// How long to wait between DHT discovery attempts. + /// + /// Default: 2000ms. + pub fn discovery_poll_interval(&self) -> Duration { + self.discovery_poll_interval + } + + /// Whether to publish a bootstrap record unconditionally on startup before dht get. + /// + /// Default: true. + pub fn publish_record_on_startup(&self) -> bool { + self.publish_record_on_startup + } + + /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. + /// + /// If this is enabled, we first fetch `unix_minute-1` and `unix_minute-2`. + /// + /// If joining longer running, existing topics is priority, set to true. + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly + /// at the same time into a topic without peers), set to false. + /// + /// Default: false. + pub fn check_older_records_first_on_startup(&self) -> bool { + self.check_older_records_first_on_startup + } +} + +/// Top-level configuration combining all settings. +#[derive(Debug, Clone)] +pub struct Config { + bootstrap_config: BootstrapConfig, + publisher_config: PublisherConfig, + dht_config: DhtConfig, + + merge_config: MergeConfig, + + max_join_peer_count: usize, + timeouts: TimeoutConfig, +} + +impl Config { + /// Create a new `ConfigBuilder` with default values. + pub fn builder() -> ConfigBuilder { + ConfigBuilder { + config: Config::default(), + } + } + + /// Publisher strategy config. + /// + /// Default: PublisherConfig::default(). + pub fn publisher_config(&self) -> &PublisherConfig { + &self.publisher_config + } + + /// DHT operation settings. + /// + /// Default: DhtConfig::default(). + pub fn dht_config(&self) -> &DhtConfig { + &self.dht_config + } + + /// Bootstrap strategy settings. + /// + /// Default: BootstrapConfig::default(). + pub fn bootstrap_config(&self) -> &BootstrapConfig { + &self.bootstrap_config + } + + /// Max peers to join simultaneously. + /// + /// Minimum is 1. + /// + /// Default: 4. + pub fn max_join_peer_count(&self) -> usize { + self.max_join_peer_count + } + + /// Timeout settings. + /// + /// Default: TimeoutConfig::default(). + pub fn timeouts(&self) -> &TimeoutConfig { + &self.timeouts + } + + /// Merge strategy settings. + /// + /// Default: bubble and overlap merges enabled. + pub fn merge_config(&self) -> &MergeConfig { + &self.merge_config + } +} + +impl Default for Config { + fn default() -> Self { + Self { + merge_config: MergeConfig::default(), + bootstrap_config: BootstrapConfig::default(), + publisher_config: PublisherConfig::default(), + dht_config: DhtConfig::default(), + max_join_peer_count: 4, + timeouts: TimeoutConfig::default(), + } + } +} + +/// Builder for `Config`. +#[derive(Debug)] +pub struct ConfigBuilder { + config: Config, +} + +impl ConfigBuilder { + /// Merge strategy settings. + /// + /// Default: MergeConfig::default(). + pub fn merge_config(mut self, merge_config: MergeConfig) -> Self { + self.config.merge_config = merge_config; + self + } + + /// Publisher strategy config. + /// + /// Default: PublisherConfig::default(). + pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { + self.config.publisher_config = publisher_config; + self + } + + /// DHT operation settings. + /// + /// Default: DhtConfig::default(). + pub fn dht_config(mut self, dht_config: DhtConfig) -> Self { + self.config.dht_config = dht_config; + self + } + + /// Bootstrap strategy settings. + /// + /// Default: BootstrapConfig::default(). + pub fn bootstrap_config(mut self, bootstrap_config: BootstrapConfig) -> Self { + self.config.bootstrap_config = bootstrap_config; + self + } + + /// Max peers to join simultaneously. No-op if `max_join_peer_count` is zero. + /// + /// If `max_join_peer_count` is called only once with zero, default value prevails. + /// If `max_join_peer_count` is first called with a non-zero value, and then again with zero, the first set value is kept. + /// + /// Default: 4. + pub fn max_join_peer_count(mut self, max_peers: usize) -> Self { + if max_peers > 0 { + self.config.max_join_peer_count = max_peers; + } + self + } + + /// Timeout settings. + /// + /// Default: TimeoutConfig::default(). + pub fn timeouts(mut self, timeouts: TimeoutConfig) -> Self { + self.config.timeouts = timeouts; + self + } + + /// Build the `Config`. + /// + /// If `max_bootstrap_records` is zero, `PublisherConfig` is set to `Disabled`. + pub fn build(self) -> Config { + let mut config = self.config; + if config.bootstrap_config.max_bootstrap_records == 0 + && matches!(config.publisher_config, PublisherConfig::Enabled(_)) + { + // if max_bootstrap_records is zero, we don't publish, so disable publisher to avoid confusion + tracing::warn!( + "Publisher is enabled via PublisherConfig::Enabled(_) but BootstrapConfig.max_bootstrap_records is set to 0 (we effectively never publish). Overriding PublisherConfig to PublisherConfig::Disabled." + ); + config.publisher_config = PublisherConfig::Disabled; + } + + config + } +} diff --git a/src/crypto/keys.rs b/src/crypto/keys.rs index 4327bf5..07729f0 100644 --- a/src/crypto/keys.rs +++ b/src/crypto/keys.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use sha2::Digest; -use crate::crypto::RecordTopic; +use crate::TopicId; /// Trait for deriving time-rotated encryption keys. /// @@ -46,7 +46,7 @@ impl SecretRotation for DefaultSecretRotation { h.update(topic_hash); h.update(unix_minute.to_be_bytes()); h.update(initial_secret_hash); - h.finalize()[..32].try_into().unwrap() + h.finalize()[..32].try_into().expect("keys -> SecretRotation.derive() hash try into [..32] failed") } } @@ -96,13 +96,13 @@ impl RotationHandle { /// # Example /// /// ```ignore -/// let topic = RecordTopic::from_str("my-topic")?; +/// let topic = TopicId::from_str("my-topic")?; /// let unix_minute = crate::unix_minute(0); -/// let signing_key = signing_keypair(topic, unix_minute); +/// let signing_key = signing_keypair(&topic, unix_minute); /// ``` -pub fn signing_keypair(record_topic: RecordTopic, unix_minute: u64) -> ed25519_dalek::SigningKey { +pub fn signing_keypair(topic_id: &TopicId, unix_minute: u64) -> ed25519_dalek::SigningKey { let mut sign_keypair_hash = sha2::Sha512::new(); - sign_keypair_hash.update(record_topic.hash()); + sign_keypair_hash.update(topic_id.hash()); sign_keypair_hash.update(unix_minute.to_le_bytes()); let sign_keypair_seed: [u8; 32] = sign_keypair_hash.finalize()[..32] .try_into() @@ -117,12 +117,12 @@ pub fn signing_keypair(record_topic: RecordTopic, unix_minute: u64) -> ed25519_d /// # Example /// /// ```ignore -/// let topic = RecordTopic::from_str("my-topic")?; +/// let topic = TopicId::from_str("my-topic")?; /// let rotation = RotationHandle::default(); -/// let enc_key = encryption_keypair(topic, &rotation, initial_hash, 0); +/// let enc_key = encryption_keypair(&topic, &rotation, initial_hash, 0); /// ``` pub fn encryption_keypair( - record_topic: RecordTopic, + topic_id: &TopicId, secret_rotation_function: &RotationHandle, initial_secret_hash: [u8; 32], unix_minute: u64, @@ -130,17 +130,18 @@ pub fn encryption_keypair( let enc_keypair_seed = secret_rotation_function .0 - .derive(record_topic.hash(), unix_minute, initial_secret_hash); + .derive(topic_id.hash(), unix_minute, initial_secret_hash); ed25519_dalek::SigningKey::from_bytes(&enc_keypair_seed) } /// Derive DHT salt for mutable record lookups. /// -/// Salt = SHA512(topic_hash || unix_minute.to_le_bytes())[..32] +/// Salt = SHA512("salt" || topic_hash || unix_minute.to_le_bytes())[..32] /// Ensures records are stored in different DHT slots per minute. -pub fn salt(record_topic: RecordTopic, unix_minute: u64) -> [u8; 32] { +pub fn salt(topic_id: &TopicId, unix_minute: u64) -> [u8; 32] { let mut slot_hash = sha2::Sha512::new(); - slot_hash.update(record_topic.hash()); + slot_hash.update(b"salt"); + slot_hash.update(topic_id.hash()); slot_hash.update(unix_minute.to_le_bytes()); slot_hash.finalize()[..32] .try_into() diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index 599a247..a0f7fe4 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -10,4 +10,4 @@ pub use keys::{ DefaultSecretRotation, RotationHandle, SecretRotation, encryption_keypair, salt, signing_keypair, }; -pub use record::{EncryptedRecord, Record, RecordPublisher, RecordTopic}; +pub use record::{EncryptedRecord, Record, RecordPublisher, TopicId}; diff --git a/src/crypto/record.rs b/src/crypto/record.rs index 46272aa..23140bc 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -1,4 +1,7 @@ -use std::{collections::HashSet, str::FromStr, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + str::FromStr, +}; use anyhow::{Result, bail}; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; @@ -6,39 +9,70 @@ use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use ed25519_dalek_hpke::{Ed25519hpkeDecryption, Ed25519hpkeEncryption}; use serde::{Deserialize, Serialize}; use sha2::Digest; +use tokio_util::sync::CancellationToken; + +use crate::Config; /// Topic identifier derived from a string via SHA512 hashing. /// -/// Used as a stable identifier for peer discovery records. +/// Used as the stable identifier for gossip subscriptions and DHT records. /// /// # Example /// /// ```ignore -/// let topic = RecordTopic::from_str("chat-app-v1")?; +/// let topic_id = TopicId::new("chat-room-1".to_string()); /// ``` -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RecordTopic([u8; 32]); +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TopicId([u8; 32]); -impl FromStr for RecordTopic { +impl FromStr for TopicId { type Err = anyhow::Error; - fn from_str(s: &str) -> std::result::Result { - let mut hasher = sha2::Sha512::new(); - hasher.update(s.as_bytes()); - let hash: [u8; 32] = hasher.finalize()[..32] - .try_into() - .map_err(|_| anyhow::anyhow!("hashing failed"))?; - Ok(RecordTopic(hash)) + fn from_str(topic_name: &str) -> std::result::Result { + Ok(Self::new(topic_name.to_string())) + } +} + +impl From<&str> for TopicId { + fn from(topic_name: &str) -> Self { + Self::new(topic_name.to_string()) + } +} + +impl From for TopicId { + fn from(topic_name: String) -> Self { + Self::new(topic_name) + } +} +/// Treats `bytes` as a topic *name* and SHA-512 hashes them. +/// For a pre-computed 32-byte hash, use [`TopicId::from_hash`] instead. +impl From> for TopicId { + fn from(topic_name: Vec) -> Self { + Self::new(topic_name) } } -impl RecordTopic { +impl TopicId { + /// Create a new topic ID from a string. + /// + /// String is hashed with SHA512; the first 32 bytes produce the identifier. + pub fn new(topic_name: impl Into>) -> Self { + let mut topic_name_hash = sha2::Sha512::new(); + topic_name_hash.update(topic_name.into()); + + Self( + topic_name_hash.finalize()[..32] + .try_into() + .expect("hashing 'topic_name' failed"), + ) + } + /// Create from a pre-computed 32-byte hash. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { + pub fn from_hash(bytes: &[u8; 32]) -> Self { Self(*bytes) } - /// Get the raw 32-byte hash. + /// Get the hash bytes. pub fn hash(&self) -> [u8; 32] { self.0 } @@ -98,40 +132,93 @@ impl RecordContent { pub struct RecordPublisher { dht: crate::dht::Dht, - record_topic: RecordTopic, + config: crate::config::Config, + + topic_id: TopicId, pub_key: VerifyingKey, signing_key: SigningKey, secret_rotation: Option, initial_secret_hash: [u8; 32], } +/// Builder for `RecordPublisher`. +#[derive(Debug)] +pub struct RecordPublisherBuilder { + topic_id: TopicId, + signing_key: SigningKey, + secret_rotation: Option, + initial_secret: Vec, + config: crate::config::Config, +} + +impl RecordPublisherBuilder { + /// Set a custom secret rotation strategy. + pub fn secret_rotation(mut self, secret_rotation: crate::crypto::keys::RotationHandle) -> Self { + self.secret_rotation = Some(secret_rotation); + self + } + + /// Set the configuration. + pub fn config(mut self, config: crate::config::Config) -> Self { + self.config = config; + self + } + + /// Build the `RecordPublisher`. + pub fn build(self) -> RecordPublisher { + RecordPublisher::new( + self.topic_id, + self.signing_key, + self.secret_rotation, + self.initial_secret, + self.config, + ) + } +} + impl RecordPublisher { + /// Create a new `RecordPublisherBuilder`. + pub fn builder( + topic_id: impl Into, + signing_key: SigningKey, + initial_secret: impl Into>, + ) -> RecordPublisherBuilder { + RecordPublisherBuilder { + topic_id: topic_id.into(), + signing_key, + secret_rotation: None, + initial_secret: initial_secret.into(), + config: crate::config::Config::default(), + } + } + /// Create a new record publisher. /// /// # Arguments /// - /// * `record_topic` - Topic identifier - /// * `pub_key` - Ed25519 public key (verifying key) + /// * `topic_id` - Topic identifier /// * `signing_key` - Ed25519 secret key (signing key) /// * `secret_rotation` - Optional custom key rotation strategy /// * `initial_secret` - Initial secret for key derivation + /// * `config` - Configuration settings pub fn new( - record_topic: impl Into, - pub_key: VerifyingKey, + topic_id: impl Into, signing_key: SigningKey, secret_rotation: Option, - initial_secret: Vec, + initial_secret: impl Into>, + config: crate::config::Config, ) -> Self { let mut initial_secret_hash = sha2::Sha512::new(); - initial_secret_hash.update(initial_secret); + initial_secret_hash.update(initial_secret.into()); let initial_secret_hash: [u8; 32] = initial_secret_hash.finalize()[..32] .try_into() .expect("hashing failed"); Self { - dht: crate::dht::Dht::new(), - record_topic: record_topic.into(), - pub_key, + dht: crate::dht::Dht::new(config.dht_config()), + config, + topic_id: topic_id.into(), + pub_key: signing_key.verifying_key(), signing_key, secret_rotation, initial_secret_hash, @@ -150,9 +237,8 @@ impl RecordPublisher { record_content: impl Serialize + Deserialize<'a>, ) -> Result { Record::sign( - self.record_topic.hash(), + self.topic_id.hash(), unix_minute, - self.pub_key.to_bytes(), record_content, &self.signing_key, ) @@ -163,14 +249,14 @@ impl RecordPublisher { self.pub_key } - /// Get the record topic. - pub fn record_topic(&self) -> RecordTopic { - self.record_topic + /// Get TopicId. + pub fn topic_id(&self) -> &TopicId { + &self.topic_id } /// Get the signing key. - pub fn signing_key(&self) -> ed25519_dalek::SigningKey { - self.signing_key.clone() + pub fn signing_key(&self) -> &ed25519_dalek::SigningKey { + &self.signing_key } /// Get the secret rotation handle if set. @@ -182,132 +268,164 @@ impl RecordPublisher { pub fn initial_secret_hash(&self) -> [u8; 32] { self.initial_secret_hash } + + /// Get the configuration. + pub fn config(&self) -> &Config { + &self.config + } } impl RecordPublisher { /// Publish a record to the DHT if slot capacity allows. /// /// Checks existing record count for this time slot and skips publishing if - /// `MAX_BOOTSTRAP_RECORDS` limit reached. - pub async fn publish_record(&self, record: Record) -> Result<()> { - let records = self - .get_records(record.unix_minute()) - .await - .iter() - .cloned() - .collect::>(); - - tracing::debug!( - "RecordPublisher: found {} existing records for unix_minute {}", - records.len(), - record.unix_minute() - ); - - if records.len() >= crate::MAX_BOOTSTRAP_RECORDS { + /// `self.config.bootstrap_config().max_bootstrap_records()` limit reached. + pub async fn publish_record(&self, record: Record, cancel_token: CancellationToken) -> Result<()> { + self.publish_record_cached_records(record, None, cancel_token).await + } + + /// Publish a record to the DHT (using cached get_records) if slot capacity allows. + /// + /// Checks existing record count for this time slot and skips publishing if + /// `self.config.bootstrap_config().max_bootstrap_records()` limit reached. + pub async fn publish_record_cached_records( + &self, + record: Record, + cached_records: Option>, + cancel_token: CancellationToken, + ) -> Result<()> { + let publish_fut = async { + let records = match cached_records { + Some(records) => records, + None => self.get_records(record.unix_minute(), cancel_token.clone()).await?, + }; + tracing::debug!( - "RecordPublisher: max records reached ({}), skipping publish", - crate::MAX_BOOTSTRAP_RECORDS + "RecordPublisher: found {} existing records for unix_minute {}", + records.len(), + record.unix_minute() ); - return Ok(()); - } - // Publish own records - let sign_key = crate::crypto::keys::signing_keypair(self.record_topic, record.unix_minute); - let salt = crate::crypto::keys::salt(self.record_topic, record.unix_minute); - let encryption_key = crate::crypto::keys::encryption_keypair( - self.record_topic, - &self.secret_rotation.clone().unwrap_or_default(), - self.initial_secret_hash, - record.unix_minute, - ); - let encrypted_record = record.encrypt(&encryption_key); - - tracing::debug!( - "RecordPublisher: publishing record to DHT for unix_minute {}", - record.unix_minute() - ); - - self.dht - .put_mutable( - sign_key.clone(), - sign_key.verifying_key(), - Some(salt.to_vec()), - encrypted_record.to_bytes().to_vec(), - Some(3), - Duration::from_secs(10), - ) - .await?; - - tracing::debug!("RecordPublisher: successfully published to DHT"); - Ok(()) + if records.len() >= self.config.bootstrap_config().max_bootstrap_records() { + tracing::debug!( + "RecordPublisher: max records reached ({}), skipping publish", + self.config.bootstrap_config().max_bootstrap_records() + ); + return Ok(()); + } + + // Publish own records + let sign_key = crate::crypto::keys::signing_keypair(self.topic_id(), record.unix_minute); + let salt = crate::crypto::keys::salt(self.topic_id(), record.unix_minute); + let encryption_key = crate::crypto::keys::encryption_keypair( + self.topic_id(), + &self.secret_rotation.clone().unwrap_or_default(), + self.initial_secret_hash, + record.unix_minute, + ); + let encrypted_record = record.encrypt(&encryption_key); + let next_seq_num = i64::MAX; + + tracing::debug!( + "RecordPublisher: publishing record to DHT for unix_minute {}", + record.unix_minute() + ); + + self.dht + .put_mutable( + sign_key.clone(), + Some(salt.to_vec()), + encrypted_record.to_bytes()?, + next_seq_num, + ) + .await?; + + tracing::debug!("RecordPublisher: successfully published to DHT"); + Ok(()) + }; + + tokio::select! { + _ = cancel_token.cancelled() => { + anyhow::bail!("publish cancelled"); + } + res = publish_fut => { + res + } + } } /// Retrieve all verified records for a given time slot from the DHT. /// /// Filters out records from this publisher's own node ID. - pub async fn get_records(&self, unix_minute: u64) -> HashSet { - tracing::debug!( - "RecordPublisher: fetching records from DHT for unix_minute {}", - unix_minute - ); - - let topic_sign = crate::crypto::keys::signing_keypair(self.record_topic, unix_minute); - let encryption_key = crate::crypto::keys::encryption_keypair( - self.record_topic, - &self.secret_rotation.clone().unwrap_or_default(), - self.initial_secret_hash, - unix_minute, - ); - let salt = crate::crypto::keys::salt(self.record_topic, unix_minute); - - // Get records, decrypt and verify - let records_iter = self - .dht - .get( - topic_sign.verifying_key(), - Some(salt.to_vec()), - None, - Duration::from_secs(10), - ) - .await - .unwrap_or_default(); - - tracing::debug!( - "RecordPublisher: received {} raw records from DHT", - records_iter.len() - ); - - let verified_records = records_iter - .iter() - .filter_map( - |record| match EncryptedRecord::from_bytes(record.value().to_vec()) { - Ok(encrypted_record) => match encrypted_record.decrypt(&encryption_key) { - Ok(record) => match record.verify(&self.record_topic.hash(), unix_minute) { - Ok(_) => match record.node_id().eq(self.pub_key.as_bytes()) { - true => { - tracing::debug!("RecordPublisher: filtered out self"); - None - } - false => Some(record), - }, - Err(_) => None, - }, - Err(_) => None, - }, - Err(_) => None, - }, - ) - .collect::>(); - - tracing::debug!( - "RecordPublisher: verified {} records (filtered self)", - verified_records.len() - ); - verified_records + /// Dedup's records based on pub_key, keeping the highest sequence number per pub_key. + pub async fn get_records(&self, unix_minute: u64, cancel_token: CancellationToken) -> Result> { + let get_fut = async { + tracing::debug!( + "RecordPublisher: fetching records from DHT for unix_minute {}", + unix_minute + ); + + let topic_sign = crate::crypto::keys::signing_keypair(self.topic_id(), unix_minute); + let encryption_key = crate::crypto::keys::encryption_keypair( + self.topic_id(), + &self.secret_rotation.clone().unwrap_or_default(), + self.initial_secret_hash, + unix_minute, + ); + let salt = crate::crypto::keys::salt(self.topic_id(), unix_minute); + + // Get records, decrypt and verify + let records_iter = self + .dht + .get(topic_sign.verifying_key(), Some(salt.to_vec()), None) + .await?; + + tracing::debug!( + "RecordPublisher: received {} raw records from DHT", + records_iter.len() + ); + + let mut dedubed_records = HashMap::new(); + for item in records_iter { + if let Ok(encrypted_record) = EncryptedRecord::from_bytes(item.value().to_vec()) + && let Ok(record) = encrypted_record.decrypt(&encryption_key) + && record.verify(&self.topic_id.hash(), unix_minute).is_ok() + && !record.pub_key().eq(self.pub_key.as_bytes()) + { + let pub_key = record.pub_key(); + match dedubed_records.get(&pub_key) { + Some((seq, _)) if *seq >= item.seq() => {} + _ => { + dedubed_records.insert(pub_key, (item.seq(), record)); + } + } + } + } + tracing::debug!( + "RecordPublisher: verified {} records (filtered self)", + dedubed_records.len() + ); + + Ok(dedubed_records + .into_values() + .map(|(_, record)| record) + .collect::>()) + }; + + tokio::select! { + _ = cancel_token.cancelled() => { + anyhow::bail!("get_records cancelled"); + } + res = get_fut => { + res + } + } } } impl EncryptedRecord { + const MAX_SIZE: usize = 2048; + /// Decrypt using an Ed25519 HPKE private key. pub fn decrypt(&self, decryption_key: &ed25519_dalek::SigningKey) -> Result { let one_time_key_bytes: [u8; 32] = decryption_key @@ -322,19 +440,38 @@ impl EncryptedRecord { } /// Serialize to bytes (length-prefixed format). - pub fn to_bytes(&self) -> Vec { + pub fn to_bytes(&self) -> Result> { let mut buf = Vec::new(); let encrypted_record_len = self.encrypted_record.len() as u32; buf.extend_from_slice(&encrypted_record_len.to_le_bytes()); buf.extend_from_slice(&self.encrypted_record); buf.extend_from_slice(&self.encrypted_decryption_key); - buf + + if buf.len() > Self::MAX_SIZE { + bail!( + "EncryptedRecord serialization exceeds maximum size, the max is set generously so this should never happen, if so your code is using it manually" + ); + } + Ok(buf) } /// Deserialize from bytes. pub fn from_bytes(buf: Vec) -> Result { + if buf.len() < 4 { + bail!("buffer too short for EncryptedRecord deserialization") + } let (encrypted_record_len, buf) = buf.split_at(4); let encrypted_record_len = u32::from_le_bytes(encrypted_record_len.try_into()?); + const ENCRYPTED_KEY_LENGTH: usize = 88; + let expected_payload_len = encrypted_record_len + .checked_add(ENCRYPTED_KEY_LENGTH as u32) + .ok_or_else(|| anyhow::anyhow!("encrypted record length overflow"))?; + if encrypted_record_len > Self::MAX_SIZE as u32 { + bail!("encrypted record length exceeds maximum allowed size") + } + if buf.len() != expected_payload_len as usize { + bail!("buffer length does not match expected encrypted record length") + } let (encrypted_record, encrypted_decryption_key) = buf.split_at(encrypted_record_len as usize); @@ -350,7 +487,6 @@ impl Record { pub fn sign<'a>( topic: [u8; 32], unix_minute: u64, - node_id: [u8; 32], record_content: impl Serialize + Deserialize<'a>, signing_key: &ed25519_dalek::SigningKey, ) -> anyhow::Result { @@ -358,14 +494,13 @@ impl Record { let mut signature_data = Vec::new(); signature_data.extend_from_slice(&topic); signature_data.extend_from_slice(&unix_minute.to_le_bytes()); - signature_data.extend_from_slice(&node_id); + signature_data.extend_from_slice(&signing_key.verifying_key().to_bytes()); signature_data.extend(&record_content.clone().0); - let signing_key = signing_key.clone(); let signature = signing_key.sign(&signature_data); Ok(Self { topic, unix_minute, - pub_key: node_id, + pub_key: signing_key.verifying_key().to_bytes(), content: record_content, signature: signature.to_bytes(), }) @@ -373,9 +508,12 @@ impl Record { /// Deserialize from bytes. pub fn from_bytes(buf: Vec) -> Result { + if buf.len() < 32 + 8 + 32 + 64 { + bail!("buffer too short for Record deserialization") + } let (topic, buf) = buf.split_at(32); let (unix_minute, buf) = buf.split_at(8); - let (node_id, buf) = buf.split_at(32); + let (pub_key, buf) = buf.split_at(32); let (record_content, buf) = buf.split_at(buf.len() - 64); let (signature, buf) = buf.split_at(64); @@ -387,7 +525,7 @@ impl Record { Ok(Self { topic: topic.try_into()?, unix_minute: u64::from_le_bytes(unix_minute.try_into()?), - pub_key: node_id.try_into()?, + pub_key: pub_key.try_into()?, content: RecordContent(record_content.to_vec()), signature: signature.try_into()?, }) @@ -416,9 +554,9 @@ impl Record { let record_bytes = self.to_bytes(); let signature_data = record_bytes[..record_bytes.len() - 64].to_vec(); let signature = ed25519_dalek::Signature::from_bytes(&self.signature); - let node_id = ed25519_dalek::VerifyingKey::from_bytes(&self.pub_key)?; + let pub_key = ed25519_dalek::VerifyingKey::from_bytes(&self.pub_key)?; - node_id.verify_strict(signature_data.as_slice(), &signature)?; + pub_key.verify_strict(signature_data.as_slice(), &signature)?; Ok(()) } @@ -452,8 +590,8 @@ impl Record { self.unix_minute } - /// Get the node ID (publisher's public key). - pub fn node_id(&self) -> [u8; 32] { + /// Get the pub_key (publisher's public key). + pub fn pub_key(&self) -> [u8; 32] { self.pub_key } diff --git a/src/dht.rs b/src/dht.rs index dc3f421..5b6c91c 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -5,13 +5,13 @@ use std::time::Duration; -use actor_helper::{Action, Actor, Handle, Receiver, act}; +use actor_helper::{Handle, act}; use anyhow::{Context, Result, bail}; use ed25519_dalek::VerifyingKey; use futures_lite::StreamExt; use mainline::{MutableItem, SigningKey}; -const RETRY_DEFAULT: usize = 3; +use crate::config::DhtConfig; /// DHT client wrapper with actor-based concurrency. /// @@ -22,25 +22,24 @@ pub struct Dht { api: Handle, } -#[derive(Debug)] +#[derive(Debug, Default)] struct DhtActor { - rx: Receiver>, dht: Option, + config: crate::config::DhtConfig, } impl Dht { /// Create a new DHT client. /// /// Spawns a background actor for handling DHT operations. - pub fn new() -> Self { - let (api, rx) = Handle::channel(); - - tokio::spawn(async move { - let mut actor = DhtActor { rx, dht: None }; - let _ = actor.run().await; - }); - - Self { api } + pub fn new(dht_config: &DhtConfig) -> Self { + Self { + api: Handle::spawn(DhtActor { + dht: None, + config: dht_config.clone(), + }) + .0, + } } /// Retrieve mutable records from the DHT. @@ -50,16 +49,14 @@ impl Dht { /// * `pub_key` - Ed25519 public key for the record /// * `salt` - Optional salt for record lookup /// * `more_recent_than` - Sequence number filter (get records newer than this) - /// * `timeout` - Maximum time to wait for results pub async fn get( &self, pub_key: VerifyingKey, salt: Option>, more_recent_than: Option, - timeout: Duration, ) -> Result> { self.api - .call(act!(actor => actor.get(pub_key, salt, more_recent_than, timeout))) + .call(act!(actor => actor.get(pub_key, salt, more_recent_than))) .await } @@ -68,40 +65,19 @@ impl Dht { /// # Arguments /// /// * `signing_key` - Ed25519 secret key for signing - /// * `pub_key` - Ed25519 public key (used for routing) /// * `salt` - Optional salt for record slot /// * `data` - Record value to publish - /// * `retry_count` - Number of retry attempts (default: 3) - /// * `timeout` - Per-request timeout + /// * `next_unix_minute_seq` - Sequence number for the record (per unix_minute) pub async fn put_mutable( &self, signing_key: SigningKey, - pub_key: VerifyingKey, salt: Option>, data: Vec, - retry_count: Option, - timeout: Duration, + next_unix_minute_seq: i64, ) -> Result<()> { - self.api.call(act!(actor => actor.put_mutable(signing_key, pub_key, salt, data, retry_count, timeout))).await - } -} - -impl Default for Dht { - fn default() -> Self { - Self::new() - } -} - -impl Actor for DhtActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; - } - else => break Ok(()), - } - } + self.api + .call(act!(actor => actor.put_mutable(signing_key, salt, data, next_unix_minute_seq))) + .await } } @@ -111,73 +87,88 @@ impl DhtActor { pub_key: VerifyingKey, salt: Option>, more_recent_than: Option, - timeout: Duration, ) -> Result> { if self.dht.is_none() { self.reset().await?; } let dht = self.dht.as_mut().context("DHT not initialized")?; - Ok(tokio::time::timeout( - timeout, + match tokio::time::timeout( + self.config.get_timeout(), dht.get_mutable(pub_key.as_bytes(), salt.as_deref(), more_recent_than) .collect::>(), ) - .await?) + .await + { + Ok(items) => Ok(items), + Err(_) => { + tracing::warn!("DHT get operation timed out"); + bail!("DHT get operation timed out") + } + } } pub async fn put_mutable( &mut self, signing_key: SigningKey, - pub_key: VerifyingKey, salt: Option>, data: Vec, - retry_count: Option, - timeout: Duration, + next_unix_minute_seq: i64, ) -> Result<()> { if self.dht.is_none() { self.reset().await?; } - for i in 0..retry_count.unwrap_or(RETRY_DEFAULT) { + for i in 0..1 + self.config.retries() { let dht = self.dht.as_mut().context("DHT not initialized")?; - let most_recent_result = tokio::time::timeout( - timeout, - dht.get_mutable_most_recent(pub_key.as_bytes(), salt.as_deref()), - ) - .await?; - - let item = if let Some(mut_item) = most_recent_result { - MutableItem::new( - signing_key.clone(), - &data, - mut_item.seq() + 1, - salt.as_deref(), - ) - } else { - MutableItem::new(signing_key.clone(), &data, 0, salt.as_deref()) - }; + let item = MutableItem::new( + signing_key.clone(), + &data, + next_unix_minute_seq, + salt.as_deref(), + ); let put_result = match tokio::time::timeout( - Duration::from_secs(10), + self.config.put_timeout(), dht.put_mutable(item.clone(), Some(item.seq())), ) .await { - Ok(result) => result.ok(), + Ok(result) => match result { + Ok(id) => Some(id), + Err(err) => { + tracing::warn!("DHT put_mutable operation failed: {err:?}"); + None + } + }, Err(_) => None, }; if put_result.is_some() { break; - } else if i == retry_count.unwrap_or(RETRY_DEFAULT) - 1 { + } else if i == self.config.retries() { bail!("failed to publish record") } self.reset().await?; - tokio::time::sleep(Duration::from_millis(rand::random::() % 2000)).await; + let jitter = if self.config.max_retry_jitter() > Duration::ZERO { + Duration::from_nanos( + (rand::random::() % self.config.max_retry_jitter().as_nanos()) as u64, + ) + } else { + Duration::ZERO + }; + let retry_interval = self.config.base_retry_interval() + jitter; + + tracing::debug!( + "DHTActor: put_mutable attempt {}/{} failed, retrying in {}ms", + i + 1, + 1 + self.config.retries(), + retry_interval.as_millis() + ); + tokio::time::sleep(retry_interval).await; } Ok(()) } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 7c28945..d014354 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -1,8 +1,8 @@ //! Bubble detection: merge isolated peer groups in the same topic. //! -//! If local peer count < 4, extract suggested peers from DHT records and join them. +//! If local peer count < `min_neighbors`, extract suggested peers from DHT records and join them. -use actor_helper::{Action, Actor, Handle, Receiver}; +use actor_helper::{Action, Handle, Receiver}; use iroh::EndpointId; use std::{collections::HashSet, time::Duration}; @@ -11,7 +11,7 @@ use anyhow::Result; /// Detects and merges small isolated peer groups (bubbles). /// -/// Triggers when local peer count < 4 and DHT records exist. Extracts `active_peers` +/// Triggers when local peer count < `min_neighbors` and DHT records exist. Extracts `active_peers` /// from DHT records and joins them. #[derive(Debug, Clone)] pub struct BubbleMerge { @@ -20,57 +20,83 @@ pub struct BubbleMerge { #[derive(Debug)] struct BubbleMergeActor { - rx: Receiver>, - record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, gossip_sender: GossipSender, ticker: tokio::time::Interval, + cancel_token: tokio_util::sync::CancellationToken, + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, } impl BubbleMerge { /// Create a new bubble merge detector. /// /// Spawns a background task that periodically checks cluster size. + #[allow(clippy::too_many_arguments)] pub fn new( record_publisher: RecordPublisher, gossip_sender: GossipSender, gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, ) -> Result { - let (api, rx) = Handle::channel(); - - let mut ticker = tokio::time::interval(Duration::from_secs(10)); + let base_interval = base_interval.max(Duration::from_secs(1)); + let mut ticker = tokio::time::interval(base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - tokio::spawn(async move { - let mut actor = BubbleMergeActor { - rx, + let api = Handle::spawn_with( + BubbleMergeActor { record_publisher, gossip_receiver, gossip_sender, ticker, - }; - let _ = actor.run().await; - }); + cancel_token, + max_join_peers, + base_interval, + max_jitter, + min_neighbors, + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { _api: api }) } } -impl Actor for BubbleMergeActor { - async fn run(&mut self) -> Result<()> { +impl BubbleMergeActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("BubbleMerge: starting bubble merge actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } _ = self.ticker.tick() => { tracing::debug!("BubbleMerge: tick fired, checking for bubbles"); - let _ = self.merge().await; - let next_interval = rand::random::() % 50; - tracing::debug!("BubbleMerge: next check in {}s", next_interval); - self.ticker.reset_after(Duration::from_secs(next_interval)); + if let Err(e) = self.merge().await { + tracing::warn!("BubbleMerge: error during merge: {:?}", e); + } + let jitter = if self.max_jitter > Duration::ZERO { + Duration::from_nanos((rand::random::() % self.max_jitter.as_nanos()) as u64) + } else { + Duration::ZERO + }; + let next_interval = self.base_interval + jitter; + tracing::debug!("BubbleMerge: next check in {}ms", next_interval.as_millis()); + self.ticker.reset_after(next_interval); + } + _ = self.cancel_token.cancelled() => { + break Ok(()); } else => break Ok(()), } @@ -82,26 +108,34 @@ impl BubbleMergeActor { // Cluster size as bubble indicator async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1).await; - records.extend(self.record_publisher.get_records(unix_minute).await); + let mut records = self + .record_publisher + .get_records(unix_minute - 1, self.cancel_token.clone()) + .await?; + records.extend( + self.record_publisher + .get_records(unix_minute, self.cancel_token.clone()) + .await?, + ); - let neighbors = self.gossip_receiver.neighbors().await; + let neighbors = self.gossip_receiver.neighbors().await?; tracing::debug!( "BubbleMerge: checking with {} neighbors and {} records", neighbors.len(), records.len() ); - if neighbors.len() < 4 && !records.is_empty() { + if neighbors.len() < self.min_neighbors && !records.is_empty() { tracing::debug!( - "BubbleMerge: detected small bubble ({} neighbors < 4), attempting merge", - neighbors.len() + "BubbleMerge: detected small bubble ({} neighbors < {}), attempting merge", + neighbors.len(), + self.min_neighbors ); - let node_ids = records + let self_pub_key = EndpointId::from_verifying_key(self.record_publisher.pub_key()); + let pub_keys = records .iter() .flat_map(|record| { - let mut endpoint_ids = if let Ok(content) = - record.content::() + let mut pub_keys = if let Ok(content) = record.content::() { content .active_peers @@ -109,8 +143,8 @@ impl BubbleMergeActor { .filter_map(|&active_peer| { if active_peer == [0; 32] || neighbors.contains(&active_peer) - || active_peer.eq(record.node_id().to_vec().as_slice()) - || active_peer.eq(self.record_publisher.pub_key().as_bytes()) + || active_peer == record.pub_key() + || active_peer.eq(self_pub_key.as_bytes()) { None } else { @@ -121,28 +155,29 @@ impl BubbleMergeActor { } else { vec![] }; - if let Ok(endpoint_id) = EndpointId::from_bytes(&record.node_id()) - && endpoint_id - != EndpointId::from_verifying_key(self.record_publisher.pub_key()) + if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) + && !pub_key.eq(&self_pub_key) + && !neighbors.contains(&pub_key) { - endpoint_ids.push(endpoint_id); + pub_keys.push(pub_key); } - endpoint_ids + pub_keys }) .collect::>(); - tracing::debug!( - "BubbleMerge: found {} potential peers to join", - node_ids.len() - ); + if !pub_keys.is_empty() { + tracing::debug!( + "BubbleMerge: found {} potential peers to join", + pub_keys.len() + ); - if !node_ids.is_empty() { self.gossip_sender .join_peers( - node_ids.iter().cloned().collect::>(), - Some(super::MAX_JOIN_PEERS_COUNT), + pub_keys.iter().cloned().collect::>(), + Some(self.max_join_peers), ) .await?; + tracing::debug!("BubbleMerge: join_peers request sent"); } } else { diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index ea8c2bd..e9e3eb0 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -1,6 +1,6 @@ //! Split-brain detection via message hash overlap in DHT records. -use actor_helper::{Action, Actor, Handle, Receiver}; +use actor_helper::{Action, Handle, Receiver}; use iroh::EndpointId; use std::{collections::HashSet, time::Duration}; @@ -9,7 +9,7 @@ use anyhow::Result; /// Detects network partitions by comparing message hashes across DHT records. /// -/// Joins peers when their published hashes match local message history. +/// Joins peers whose published hashes do not overlap with local message history. #[derive(Debug, Clone)] pub struct MessageOverlapMerge { _api: Handle, @@ -17,12 +17,15 @@ pub struct MessageOverlapMerge { #[derive(Debug)] struct MessageOverlapMergeActor { - rx: Receiver>, - record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, gossip_sender: GossipSender, ticker: tokio::time::Interval, + cancel_token: tokio_util::sync::CancellationToken, + + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, } impl MessageOverlapMerge { @@ -31,41 +34,60 @@ impl MessageOverlapMerge { record_publisher: RecordPublisher, gossip_sender: GossipSender, gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, ) -> Result { - let (api, rx) = Handle::channel(); - - let mut ticker = tokio::time::interval(Duration::from_secs(10)); + let base_interval = base_interval.max(Duration::from_secs(1)); + let mut ticker = tokio::time::interval(base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - tokio::spawn(async move { - let mut actor = MessageOverlapMergeActor { - rx, + let api = Handle::spawn_with( + MessageOverlapMergeActor { record_publisher, gossip_receiver, gossip_sender, ticker, - }; - let _ = actor.run().await; - }); - + cancel_token, + max_join_peers, + base_interval, + max_jitter, + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { _api: api }) } } -impl Actor for MessageOverlapMergeActor { - async fn run(&mut self) -> Result<()> { +impl MessageOverlapMergeActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("MessageOverlapMerge: starting message overlap merge actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } _ = self.ticker.tick() => { tracing::debug!("MessageOverlapMerge: tick fired, checking for split-brain"); - let _ = self.merge().await; - let next_interval = rand::random::() % 50; - tracing::debug!("MessageOverlapMerge: next check in {}s", next_interval); - self.ticker.reset_after(Duration::from_secs(next_interval)); + if let Err(e) = self.merge().await { + tracing::warn!("MessageOverlapMerge: error during merge: {:?}", e); + } + let jitter = if self.max_jitter > Duration::ZERO { + Duration::from_nanos((rand::random::() % self.max_jitter.as_nanos()) as u64) + } else { + Duration::ZERO + }; + let next_interval = self.base_interval + jitter; + tracing::debug!("MessageOverlapMerge: next check in {}ms", next_interval.as_millis()); + self.ticker.reset_after(next_interval); + } + _ = self.cancel_token.cancelled() => { + break Ok(()); } else => break Ok(()), } @@ -76,10 +98,17 @@ impl Actor for MessageOverlapMergeActor { impl MessageOverlapMergeActor { async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1).await; - records.extend(self.record_publisher.get_records(unix_minute).await); + let mut records = self + .record_publisher + .get_records(unix_minute - 1, self.cancel_token.clone()) + .await?; + records.extend( + self.record_publisher + .get_records(unix_minute, self.cancel_token.clone()) + .await?, + ); - let local_hashes = self.gossip_receiver.last_message_hashes().await; + let local_hashes = self.gossip_receiver.last_message_hashes().await?; tracing::debug!( "MessageOverlapMerge: checking {} records with {} local message hashes", records.len(), @@ -92,10 +121,16 @@ impl MessageOverlapMergeActor { .iter() .filter(|record| { if let Ok(content) = record.content::() { - content.last_message_hashes.iter().any(|last_message_hash| { - *last_message_hash != [0; 32] - && last_message_hashes.contains(last_message_hash) - }) + let remote_hashes = content + .last_message_hashes + .iter() + .filter(|last_message_hash| **last_message_hash != [0; 32]) + .collect::>(); + + !remote_hashes.is_empty() + && remote_hashes.iter().all(|last_message_hash| { + !last_message_hashes.contains(*last_message_hash) + }) } else { false } @@ -103,47 +138,56 @@ impl MessageOverlapMergeActor { .collect::>(); tracing::debug!( - "MessageOverlapMerge: found {} peers with overlapping message hashes", + "MessageOverlapMerge: found {} peers with no overlapping message hashes", peers_to_join.len() ); if !peers_to_join.is_empty() { - let node_ids = peers_to_join + let active_neighbors = self.gossip_receiver.neighbors().await?; + let self_pub_key = EndpointId::from_verifying_key(self.record_publisher.pub_key()); + let pub_keys = peers_to_join .iter() .flat_map(|&record| { let mut peers = vec![]; - if let Ok(node_id) = EndpointId::from_bytes(&record.node_id()) { - peers.push(node_id); + if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) + && pub_key != self_pub_key + { + peers.push(pub_key); } if let Ok(content) = record.content::() { for active_peer in content.active_peers { if active_peer == [0; 32] { continue; } - if let Ok(node_id) = EndpointId::from_bytes(&active_peer) { - peers.push(node_id); + if let Ok(pub_key) = EndpointId::from_bytes(&active_peer) + && pub_key != self_pub_key + { + peers.push(pub_key); } } } peers }) + .filter(|pub_key| !active_neighbors.contains(pub_key)) .collect::>(); - tracing::debug!( - "MessageOverlapMerge: attempting to join {} node_ids with overlapping messages", - node_ids.len() - ); - - self.gossip_sender - .join_peers( - node_ids.iter().cloned().collect::>(), - Some(super::MAX_JOIN_PEERS_COUNT), - ) - .await?; - - tracing::debug!( - "MessageOverlapMerge: join_peers request sent for split-brain recovery" - ); + if !pub_keys.is_empty() { + tracing::debug!( + "MessageOverlapMerge: attempting to join {} pub_keys with no overlapping messages", + pub_keys.len() + ); + + self.gossip_sender + .join_peers( + pub_keys.iter().cloned().collect::>(), + Some(self.max_join_peers), + ) + .await?; + + tracing::debug!( + "MessageOverlapMerge: join_peers request sent for split-brain recovery" + ); + } } } else { tracing::debug!( diff --git a/src/gossip/merge/mod.rs b/src/gossip/merge/mod.rs index b791400..87622a5 100644 --- a/src/gossip/merge/mod.rs +++ b/src/gossip/merge/mod.rs @@ -1,14 +1,12 @@ //! Peer merging strategies for recovering from network partitions. //! //! Two complementary strategies detect and heal split-brain scenarios: -//! - **Bubble Merge**: Joins small clusters (< 4 peers) with peers advertised in DHT -//! - **Message Overlap**: Detects when isolated clusters share common message hashes, -//! indicating they have seen the same messages and can be merged +//! - **Bubble Merge**: Joins small clusters (< `min_neighbors` peers, as configured in `BubbleMergeConfig::Enabled`) with peers advertised in DHT +//! - **Message Overlap**: Detects when isolated clusters have no common message hashes, +//! indicating a network partition that should be healed by merging mod bubble; mod message_overlap; pub use bubble::BubbleMerge; -pub use message_overlap::MessageOverlapMerge; - -pub const MAX_JOIN_PEERS_COUNT: usize = 30; +pub use message_overlap::MessageOverlapMerge; \ No newline at end of file diff --git a/src/gossip/mod.rs b/src/gossip/mod.rs index fb79200..043083f 100644 --- a/src/gossip/mod.rs +++ b/src/gossip/mod.rs @@ -13,9 +13,9 @@ pub use merge::{BubbleMerge, MessageOverlapMerge}; pub use receiver::GossipReceiver; pub use sender::GossipSender; use serde::{Deserialize, Serialize}; -pub use topic::{Bootstrap, Publisher, Topic, TopicId}; +pub use topic::{Bootstrap, Publisher, Topic}; -use crate::RecordPublisher; +use crate::{MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher}; /// Record content for peer discovery. /// @@ -23,10 +23,10 @@ use crate::RecordPublisher; /// and recently seen message hashes for cluster merging. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GossipRecordContent { - /// Fixed array of 5 peer node IDs (as 32-byte arrays, empty slots are zero-filled) - pub active_peers: [[u8; 32]; 5], - /// Fixed array of 5 recent message hashes for overlap detection (empty slots are zero-filled) - pub last_message_hashes: [[u8; 32]; 5], + /// Fixed array of MAX_RECORD_PEERS peer node IDs (as 32-byte arrays, empty slots are zero-filled) + pub active_peers: [[u8; 32]; MAX_RECORD_PEERS], + /// Fixed array of MAX_MESSAGE_HASHES recent message hashes for overlap detection (empty slots are zero-filled) + pub last_message_hashes: [[u8; 32]; MAX_MESSAGE_HASHES], } /// Extension trait for iroh Gossip enabling auto-discovery. diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 1f11adb..bdb5d92 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -1,167 +1,247 @@ //! Actor-based wrapper for iroh-gossip message receiving. -use std::collections::{HashSet, VecDeque}; +use std::{ + collections::{HashSet, VecDeque}, + fmt::Display, + sync::Arc, +}; -use actor_helper::{Action, Actor, Handle, Receiver, act, act_ok}; +use actor_helper::{Action, Handle, Receiver, act_ok}; use anyhow::Result; use futures_lite::StreamExt; use iroh::EndpointId; use sha2::Digest; +use crate::{MAX_MESSAGE_HASHES, Topic}; + /// Gossip receiver that collects incoming messages and neighbor info. /// /// Tracks SHA512 message hashes (first 32 bytes) for overlap detection and provides /// neighbor list for topology analysis. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct GossipReceiver { api: Handle, - _gossip: iroh_gossip::net::Gossip, + pub(crate) _topic_keep_alive: Option>, + _next_channel_sender: tokio::sync::broadcast::WeakSender>, + next_channel_receiver: tokio::sync::broadcast::Receiver>, + _join_channel_sender: tokio::sync::broadcast::WeakSender>, + join_channel_receiver: tokio::sync::broadcast::Receiver>, +} + +impl Clone for GossipReceiver { + fn clone(&self) -> Self { + let next_rx = match self._next_channel_sender.upgrade() { + Some(sender) => sender.subscribe(), + None => { + let (tx, rx) = tokio::sync::broadcast::channel(1); + drop(tx); + rx + } + }; + let join_rx = match self._join_channel_sender.upgrade() { + Some(sender) => sender.subscribe(), + None => { + let (tx, rx) = tokio::sync::broadcast::channel(1); + drop(tx); + rx + } + }; + Self { + api: self.api.clone(), + _topic_keep_alive: self._topic_keep_alive.clone(), + _next_channel_sender: self._next_channel_sender.clone(), + next_channel_receiver: next_rx, + _join_channel_sender: self._join_channel_sender.clone(), + join_channel_receiver: join_rx, + } + } } +/// Internal actor for gossip receive operations. #[derive(Debug)] pub struct GossipReceiverActor { - rx: Receiver>, gossip_receiver: iroh_gossip::api::GossipReceiver, - last_message_hashes: Vec<[u8; 32]>, - msg_queue: VecDeque>>, - waiters: VecDeque< - tokio::sync::oneshot::Sender< - Option>, - >, - >, - _gossip: iroh_gossip::net::Gossip, + last_message_hashes: VecDeque<[u8; 32]>, + cancel_token: tokio_util::sync::CancellationToken, + next_channel_sender: tokio::sync::broadcast::Sender>, + join_channel_sender: tokio::sync::broadcast::Sender>, +} + +#[derive(Debug)] +pub enum ChannelError { + Closed, + Lagged(u64), +} + +impl From for ChannelError { + fn from(err: tokio::sync::broadcast::error::RecvError) -> Self { + match err { + tokio::sync::broadcast::error::RecvError::Closed => ChannelError::Closed, + tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { + ChannelError::Lagged(skipped) + } + } + } +} + +impl std::error::Error for ChannelError {} +impl Display for ChannelError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ChannelError::Closed => write!(f, "channel closed"), + ChannelError::Lagged(skipped) => { + write!(f, "channel lagged, skipped {} messages", skipped) + } + } + } } impl GossipReceiver { /// Create a new gossip receiver from an iroh topic receiver. pub fn new( gossip_receiver: iroh_gossip::api::GossipReceiver, - gossip: iroh_gossip::net::Gossip, + cancel_token: tokio_util::sync::CancellationToken, ) -> Self { - let (api, rx) = Handle::channel(); - tokio::spawn({ - let gossip = gossip.clone(); - async move { - let mut actor = GossipReceiverActor { - rx, - gossip_receiver, - last_message_hashes: Vec::new(), - msg_queue: VecDeque::new(), - waiters: VecDeque::new(), - _gossip: gossip.clone(), - }; - let _ = actor.run().await; - } - }); + let (next_tx, next_rx) = tokio::sync::broadcast::channel(64); + let (join_tx, join_rx) = tokio::sync::broadcast::channel(64); + let api = Handle::spawn_with( + GossipReceiverActor { + gossip_receiver, + last_message_hashes: VecDeque::with_capacity(MAX_MESSAGE_HASHES), + cancel_token, + next_channel_sender: next_tx.clone(), + join_channel_sender: join_tx.clone(), + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Self { api, - _gossip: gossip.clone(), + _topic_keep_alive: None, + next_channel_receiver: next_rx, + _next_channel_sender: next_tx.downgrade(), + join_channel_receiver: join_rx, + _join_channel_sender: join_tx.downgrade(), } } /// Get the set of currently connected neighbor node IDs. - pub async fn neighbors(&self) -> HashSet { + pub async fn neighbors(&self) -> Result> { self.api .call(act_ok!(actor => async move { actor.gossip_receiver.neighbors().collect::>() })) .await - .expect("actor stopped") } /// Check if the local node has joined the topic. - pub async fn is_joined(&self) -> bool { + pub async fn is_joined(&self) -> Result { self.api .call(act_ok!(actor => async move { actor.gossip_receiver.is_joined() })) .await - .expect("actor stopped") } /// Receive the next gossip event. - /// - /// Returns `None` if the receiver is closed. - pub async fn next( - &self, - ) -> Option> { - let (tx, rx) = tokio::sync::oneshot::channel(); - self.api - .call(act!(actor => actor.register_next(tx))) - .await - .ok()?; - rx.await.ok()? + pub async fn next(&mut self) -> Result { + match self.next_channel_receiver.recv().await { + Ok(event) => match event { + Some(event) => Ok(event), + None => Err(ChannelError::Closed), + }, + Err(err) => Err(err.into()), + } + } + + /// Waits for a NeighborUp or a message Received event then returns `Ok(())`. + pub async fn joined(&mut self) -> Result<(), ChannelError> { + if self.is_joined().await.map_err(|_| ChannelError::Closed)? { + return Ok(()); + } + match self.join_channel_receiver.recv().await { + Ok(event) => match event { + Some(event) => Ok(event), + None => Err(ChannelError::Closed), + }, + Err(err) => Err(err.into()), + } } /// Get SHA512 hashes (first 32 bytes) of recently received messages. /// /// Used for detecting message overlap during network partition recovery. - pub async fn last_message_hashes(&self) -> Vec<[u8; 32]> { + pub async fn last_message_hashes(&self) -> Result> { self.api .call(act_ok!(actor => async move { actor.last_message_hashes.clone() })) .await - .expect("void") } } -impl Actor for GossipReceiverActor { - async fn run(&mut self) -> Result<()> { +impl GossipReceiverActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("GossipReceiver: starting gossip receiver actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } raw_event = self.gossip_receiver.next() => { - self.msg_queue.push_front(raw_event); - - if let Some(waiter) = self.waiters.pop_back() { - if let Some(event) = self.msg_queue.pop_back() { - let _ = waiter.send(event); - } else { - let _ = waiter.send(None); - // this should never happen + let event = match raw_event { + None => { + tracing::debug!("GossipReceiver: gossip receiver closed"); + self.join_channel_sender.send(None).ok(); + self.next_channel_sender.send(None).ok(); + self.cancel_token.cancel(); + break Ok(()); } - } - if let Some(Some(Ok(event))) = self.msg_queue.front() { - match event { - iroh_gossip::api::Event::Received(msg) => { - tracing::debug!("GossipReceiver: received message from {:?}", msg.delivered_from); - let mut hash = sha2::Sha512::new(); - hash.update(msg.content.clone()); - if let Ok(lmh) = hash.finalize()[..32].try_into() { - self.last_message_hashes.push(lmh); + Some(Err(err)) => { + tracing::warn!("GossipReceiver: error receiving gossip event: {err}"); + self.next_channel_sender.send(None).ok(); + self.join_channel_sender.send(None).ok(); + self.cancel_token.cancel(); + break Ok(()); + } + Some(Ok(ref event)) => { + match event { + iroh_gossip::api::Event::Received(msg) => { + tracing::debug!("GossipReceiver: received message from {:?}", msg.delivered_from); + let mut hash = sha2::Sha512::new(); + hash.update(&msg.content); + if let Ok(lmh) = hash.finalize()[..32].try_into() { + if self.last_message_hashes.len() == MAX_MESSAGE_HASHES { + self.last_message_hashes.pop_front(); + } + self.last_message_hashes.push_back(lmh); + } + self.join_channel_sender.send(Some(())).ok(); } - } - iroh_gossip::api::Event::NeighborUp(node_id) => { - tracing::debug!("GossipReceiver: neighbor UP: {}", node_id); - } - iroh_gossip::api::Event::NeighborDown(node_id) => { - tracing::debug!("GossipReceiver: neighbor DOWN: {}", node_id); - } - iroh_gossip::api::Event::Lagged => { - tracing::debug!("GossipReceiver: event stream lagged"); - } + iroh_gossip::api::Event::NeighborUp(pub_key) => { + tracing::debug!("GossipReceiver: neighbor UP: {}", pub_key); + self.join_channel_sender.send(Some(())).ok(); + } + iroh_gossip::api::Event::NeighborDown(pub_key) => { + tracing::debug!("GossipReceiver: neighbor DOWN: {}", pub_key); + } + iroh_gossip::api::Event::Lagged => { + tracing::debug!("GossipReceiver: event stream lagged"); + } + }; + event.clone() } - } + }; + + self.next_channel_sender.send(Some(event)).ok(); + } + _ = self.cancel_token.cancelled() => { + self.join_channel_sender.send(None).ok(); + self.next_channel_sender.send(None).ok(); + break Ok(()); } else => break Ok(()), } } } } - -impl GossipReceiverActor { - pub async fn register_next( - &mut self, - waiter: tokio::sync::oneshot::Sender< - Option>, - >, - ) -> Result<()> { - if let Some(event) = self.msg_queue.pop_back() { - let _ = waiter.send(event); - } else { - self.waiters.push_front(waiter); - } - Ok(()) - } -} diff --git a/src/gossip/sender.rs b/src/gossip/sender.rs index f4db8ab..f9f4939 100644 --- a/src/gossip/sender.rs +++ b/src/gossip/sender.rs @@ -1,10 +1,14 @@ //! Actor-based wrapper for iroh-gossip broadcast operations. -use actor_helper::{Action, Actor, Handle, Receiver, act}; +use std::sync::Arc; + +use actor_helper::{Handle, act}; use anyhow::Result; use iroh::EndpointId; use rand::seq::SliceRandom; +use crate::{TimeoutConfig, Topic}; + /// Gossip sender that broadcasts messages to peers. /// /// Provides methods for broadcasting to all peers or just direct neighbors, @@ -12,48 +16,40 @@ use rand::seq::SliceRandom; #[derive(Debug, Clone)] pub struct GossipSender { api: Handle, - _gossip: iroh_gossip::net::Gossip, + pub(crate) _topic_keep_alive: Option>, + pub(crate) timeout_config: TimeoutConfig, } +/// Internal actor for gossip send operations. #[derive(Debug)] pub struct GossipSenderActor { - rx: Receiver>, gossip_sender: iroh_gossip::api::GossipSender, - _gossip: iroh_gossip::net::Gossip, } impl GossipSender { /// Create a new gossip sender from an iroh topic sender. - pub fn new( - gossip_sender: iroh_gossip::api::GossipSender, - gossip: iroh_gossip::net::Gossip, - ) -> Self { - let (api, rx) = Handle::channel(); - tokio::spawn({ - let gossip = gossip.clone(); - async move { - let mut actor = GossipSenderActor { - rx, - gossip_sender, - _gossip: gossip.clone(), - }; - let _ = actor.run().await; - } - }); + pub fn new(gossip_sender: iroh_gossip::api::GossipSender, timeout_config: TimeoutConfig) -> Self { + let api = Handle::spawn(GossipSenderActor { gossip_sender }).0; Self { api, - _gossip: gossip, + _topic_keep_alive: None, + timeout_config, } } /// Broadcast a message to all peers in the topic. pub async fn broadcast(&self, data: Vec) -> Result<()> { tracing::debug!("GossipSender: broadcasting message ({} bytes)", data.len()); + let broadcast_timeout = self.timeout_config.broadcast_timeout(); self.api .call(act!(actor => async move { - actor.gossip_sender - .broadcast(data.into()).await.map_err(|e| anyhow::anyhow!(e)) + tokio::time::timeout( + broadcast_timeout, + actor.gossip_sender.broadcast(data.into()) + ).await + .map_err(|_| anyhow::anyhow!("broadcast timed out"))? + .map_err(|e| anyhow::anyhow!(e)) })) .await } @@ -64,9 +60,15 @@ impl GossipSender { "GossipSender: broadcasting to neighbors ({} bytes)", data.len() ); + let broadcast_neighbors_timeout = self.timeout_config.broadcast_neighbors_timeout(); self.api .call(act!(actor => async move { - actor.gossip_sender.broadcast_neighbors(data.into()).await.map_err(|e| anyhow::anyhow!(e)) + tokio::time::timeout( + broadcast_neighbors_timeout, + actor.gossip_sender.broadcast_neighbors(data.into()) + ).await + .map_err(|_| anyhow::anyhow!("broadcast_neighbors timed out"))? + .map_err(|e| anyhow::anyhow!(e)) })) .await } @@ -86,26 +88,16 @@ impl GossipSender { tracing::debug!("GossipSender: joining {} peers", peers.len()); + let join_peers_timeout = self.timeout_config.join_peer_timeout(); self.api .call(act!(actor => async move { - actor.gossip_sender - .join_peers(peers) - .await + tokio::time::timeout( + join_peers_timeout, + actor.gossip_sender.join_peers(peers) + ).await + .map_err(|_| anyhow::anyhow!("join_peers timed out"))? .map_err(|e| anyhow::anyhow!(e)) })) .await } } - -impl Actor for GossipSenderActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; - } - else => break Ok(()), - } - } - } -} diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 3ff6dcf..62f7b82 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -1,14 +1,16 @@ //! Bootstrap process for discovering and joining peers via DHT. -use std::{collections::HashSet, time::Duration}; +use std::collections::HashSet; -use actor_helper::{Action, Actor, Handle, Receiver, act, act_ok}; +use actor_helper::{Handle, act, act_ok}; use anyhow::Result; use iroh::EndpointId; use tokio::time::sleep; +use tokio_util::sync::CancellationToken; use crate::{ - GossipSender, + GossipSender, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher, + config::BootstrapConfig, crypto::Record, gossip::{GossipRecordContent, receiver::GossipReceiver}, }; @@ -24,12 +26,11 @@ pub struct Bootstrap { #[derive(Debug)] struct BootstrapActor { - rx: Receiver>, - record_publisher: crate::crypto::RecordPublisher, - gossip_sender: GossipSender, gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, + config: BootstrapConfig, } impl Bootstrap { @@ -37,30 +38,30 @@ impl Bootstrap { pub async fn new( record_publisher: crate::crypto::RecordPublisher, gossip: iroh_gossip::net::Gossip, + cancel_token: tokio_util::sync::CancellationToken, + timeout_config: crate::config::TimeoutConfig, + bootstrap_config: BootstrapConfig, ) -> Result { let gossip_topic: iroh_gossip::api::GossipTopic = gossip .subscribe( - iroh_gossip::proto::TopicId::from(record_publisher.record_topic().hash()), + iroh_gossip::proto::TopicId::from(record_publisher.topic_id().hash()), vec![], ) .await?; let (gossip_sender, gossip_receiver) = gossip_topic.split(); let (gossip_sender, gossip_receiver) = ( - GossipSender::new(gossip_sender, gossip.clone()), - GossipReceiver::new(gossip_receiver, gossip.clone()), + GossipSender::new(gossip_sender, timeout_config), + GossipReceiver::new(gossip_receiver, cancel_token.clone()), ); - let (api, rx) = Handle::channel(); - - tokio::spawn(async move { - let mut actor = BootstrapActor { - rx, - record_publisher, - gossip_sender, - gossip_receiver, - }; - let _ = actor.run().await; - }); + let api = Handle::spawn(BootstrapActor { + record_publisher, + gossip_sender, + gossip_receiver, + cancel_token, + config: bootstrap_config, + }) + .0; Ok(Self { api }) } @@ -68,7 +69,7 @@ impl Bootstrap { /// Start the bootstrap process. /// /// Returns a receiver that signals completion when the node has joined the topic (has at least one neighbor). - pub async fn bootstrap(&self) -> Result> { + pub async fn bootstrap(&self) -> Result>> { self.api.call(act!(actor=> actor.start_bootstrap())).await } @@ -87,100 +88,159 @@ impl Bootstrap { } } -impl Actor for BootstrapActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; - } - else => break Ok(()), - } - } - } -} - impl BootstrapActor { - pub async fn start_bootstrap(&mut self) -> Result> { + pub async fn start_bootstrap(&mut self) -> Result>> { let (sender, receiver) = tokio::sync::oneshot::channel(); tokio::spawn({ let mut last_published_unix_minute = 0; - let (gossip_sender, gossip_receiver) = + let (gossip_sender, mut gossip_receiver) = (self.gossip_sender.clone(), self.gossip_receiver.clone()); let record_publisher = self.record_publisher.clone(); + let cancel_token = self.cancel_token.clone(); + let bootstrap_config = self.config.clone(); + let mut is_joined_ret = false; + + if self.config.publish_record_on_startup() { + let unix_minute = crate::unix_minute(0); + tracing::debug!("Bootstrap: initial startup record publish {}", unix_minute); + last_published_unix_minute = if self.config.check_older_records_first_on_startup() { + 0 + } else { + unix_minute + }; + let record_creator = record_publisher.clone(); + let record_content = GossipRecordContent { + active_peers: [[0; 32]; MAX_RECORD_PEERS], + last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], + }; + if let Ok(record) = Record::sign( + record_publisher.topic_id().hash(), + unix_minute, + record_content, + record_publisher.signing_key(), + ) { + publish_record_fire_and_forget( + record_creator, + record, + None, + cancel_token.clone(), + ); + } + } + async move { tracing::debug!("Bootstrap: starting bootstrap process"); - loop { + 'bootstrap: while !cancel_token.is_cancelled() { // Check if we are connected to at least one node - if gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!("Bootstrap: already joined, exiting bootstrap loop"); + is_joined_ret = true; + break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); break; } - // On the first try we check the prev unix minute, after that the current one - let unix_minute = crate::unix_minute(if last_published_unix_minute == 0 { - -1 + let current_unix_minute = crate::unix_minute(0); + + let mut use_cached_next = true; + // last_published_unix_minute == 0 means first run + let unix_minute_offset = if last_published_unix_minute == 0 + && bootstrap_config.check_older_records_first_on_startup() + { + use_cached_next = false; + 1 } else { 0 - }); + }; // Unique, verified records for the unix minute - let mut records = record_publisher.get_records(unix_minute - 1).await; - records.extend(record_publisher.get_records(unix_minute).await); + let mut records = record_publisher + .get_records( + current_unix_minute.saturating_sub(unix_minute_offset + 1), + cancel_token.clone(), + ) + .await + .unwrap_or_default(); + let current_records = record_publisher + .get_records( + current_unix_minute.saturating_sub(unix_minute_offset), + cancel_token.clone(), + ) + .await + .unwrap_or_default(); + records.extend(current_records.clone()); tracing::debug!( "Bootstrap: fetched {} records for unix_minute {}", records.len(), - unix_minute + current_unix_minute ); // If there are no records, invoke the publish_proc (the publishing procedure) // continue the loop after if records.is_empty() { - if unix_minute != last_published_unix_minute { + if current_unix_minute != last_published_unix_minute { tracing::debug!( "Bootstrap: no records found, publishing own record for unix_minute {}", - unix_minute + current_unix_minute ); - last_published_unix_minute = unix_minute; + last_published_unix_minute = current_unix_minute; let record_creator = record_publisher.clone(); let record_content = GossipRecordContent { - active_peers: [[0; 32]; 5], - last_message_hashes: [[0; 32]; 5], + active_peers: [[0; 32]; MAX_RECORD_PEERS], + last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], }; if let Ok(record) = Record::sign( - record_publisher.record_topic().hash(), - unix_minute, - record_publisher.pub_key().to_bytes(), + record_publisher.topic_id().hash(), + current_unix_minute, record_content, - &record_publisher.signing_key(), + record_publisher.signing_key(), ) { - tokio::spawn(async move { - let _ = record_creator.publish_record(record).await; - }); + publish_record_fire_and_forget( + record_creator, + record, + if use_cached_next { + Some(current_records.clone()) + } else { + None + }, + cancel_token.clone(), + ); } } - sleep(Duration::from_millis(100)).await; + tokio::select! { + _ = sleep(bootstrap_config.no_peers_retry_interval()) => {} + _ = gossip_receiver.joined() => continue, + _ = cancel_token.cancelled() => break, + } continue; } // We found records - // Collect node ids from active_peers and record.node_id (of publisher) + // Collect node ids from active_peers and record.pub_key (of publisher) let bootstrap_nodes = records .iter() .flat_map(|record| { - let mut v = vec![record.node_id()]; + // records are already filtered by self entry + let mut v = vec![record.pub_key()]; + if let Ok(record_content) = record.content::() { for peer in record_content.active_peers { - if peer != [0; 32] { + if peer != [0; 32] + && !peer.eq(record_publisher.pub_key().as_bytes()) + { v.push(peer); } } } v }) - .filter_map(|node_id| EndpointId::from_bytes(&node_id).ok()) + .filter_map(|pub_key| EndpointId::from_bytes(&pub_key).ok()) .collect::>(); tracing::debug!( @@ -191,22 +251,44 @@ impl BootstrapActor { // Maybe in the meantime someone connected to us via one of our published records // we don't want to disrup the gossip rotations any more then we have to // so we check again before joining new peers - if gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!("Bootstrap: joined while processing records, exiting"); + is_joined_ret = true; + break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); break; } - // Instead of throwing everything into join_peers() at once we go node_id by node_id + // Instead of throwing everything into join_peers() at once we go pub_key by pub_key // again to disrupt as little nodes peer neighborhoods as possible. - for node_id in bootstrap_nodes.iter() { - match gossip_sender.join_peers(vec![*node_id], None).await { + for pub_key in bootstrap_nodes.iter() { + match gossip_sender.join_peers(vec![*pub_key], None).await { Ok(_) => { - tracing::debug!("Bootstrap: attempted to join peer {}", node_id); - sleep(Duration::from_millis(100)).await; - if gossip_receiver.is_joined().await { + tracing::debug!("Bootstrap: attempted to join peer {}", pub_key); + + tokio::select! { + _ = sleep(bootstrap_config.per_peer_join_settle_time()) => {} + _ = gossip_receiver.joined() => {}, + _ = cancel_token.cancelled() => break 'bootstrap, + } + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!( "Bootstrap: successfully joined via peer {}", - node_id + pub_key + ); + is_joined_ret = true; + break; + } else if let Err(e) = is_joined { + tracing::debug!( + "Bootstrap: error checking join status: {:?}", + e ); break; } @@ -214,7 +296,7 @@ impl BootstrapActor { Err(e) => { tracing::debug!( "Bootstrap: failed to join peer {}: {:?}", - node_id, + pub_key, e ); continue; @@ -224,51 +306,101 @@ impl BootstrapActor { // If we are still not connected to anyone: // give it the default iroh-gossip connection timeout before the final is_joined() check - if !gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined + && !is_joined + { tracing::debug!( - "Bootstrap: not joined yet, waiting 500ms before final check" + "Bootstrap: not joined yet, waiting {:?} before final check", + bootstrap_config.join_confirmation_wait_time() ); - sleep(Duration::from_millis(500)).await; + tokio::select! { + _ = sleep(bootstrap_config.join_confirmation_wait_time()) => {} + _ = gossip_receiver.joined() => {}, + _ = cancel_token.cancelled() => break, + } + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); + break; } // If we are connected: return - if gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!("Bootstrap: successfully joined after final wait"); + is_joined_ret = true; + break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); break; } else { tracing::debug!("Bootstrap: still not joined after attempting all peers"); // If we are not connected: check if we should publish a record this minute - if unix_minute != last_published_unix_minute { + if current_unix_minute != last_published_unix_minute { tracing::debug!( "Bootstrap: publishing fallback record for unix_minute {}", - unix_minute + current_unix_minute ); - last_published_unix_minute = unix_minute; + last_published_unix_minute = current_unix_minute; let record_creator = record_publisher.clone(); if let Ok(record) = Record::sign( - record_publisher.record_topic().hash(), - unix_minute, - record_publisher.pub_key().to_bytes(), + record_publisher.topic_id().hash(), + current_unix_minute, GossipRecordContent { - active_peers: [[0; 32]; 5], - last_message_hashes: [[0; 32]; 5], + active_peers: [[0; 32]; MAX_RECORD_PEERS], + last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], }, - &record_publisher.signing_key(), + record_publisher.signing_key(), ) { - tokio::spawn(async move { - let _ = record_creator.publish_record(record).await; - }); + publish_record_fire_and_forget( + record_creator, + record, + if use_cached_next { + Some(current_records) + } else { + None + }, + cancel_token.clone(), + ); } } - sleep(Duration::from_millis(100)).await; - continue; + tokio::select! { + _ = sleep(bootstrap_config.discovery_poll_interval()) => continue, + _ = gossip_receiver.joined() => continue, + _ = cancel_token.cancelled() => break, + } } } - tracing::debug!("Bootstrap: completed successfully"); - let _ = sender.send(()); + tracing::debug!("Bootstrap: exited"); + + if is_joined_ret { + let _ = sender.send(Ok(())); + } else { + let _ = sender.send(Err(anyhow::anyhow!( + "Bootstrap process failed or was cancelled" + ))); + } } }); Ok(receiver) } } + +fn publish_record_fire_and_forget( + record_publisher: RecordPublisher, + record: Record, + cached_records: Option>, + cancel_token: CancellationToken, +) { + tokio::spawn(async move { + if let Err(err) = record_publisher + .publish_record_cached_records(record, cached_records, cancel_token) + .await + { + tracing::warn!("Failed to publish record: {:?}", err); + } + }); +} diff --git a/src/gossip/topic/mod.rs b/src/gossip/topic/mod.rs index 2037d53..a35faa2 100644 --- a/src/gossip/topic/mod.rs +++ b/src/gossip/topic/mod.rs @@ -7,4 +7,4 @@ mod topic; pub use bootstrap::Bootstrap; pub use publisher::Publisher; -pub use topic::{Topic, TopicId}; +pub use topic::Topic; diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 18256e3..621dc8e 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -1,15 +1,15 @@ //! Background publisher that updates DHT records with active peer info. -use actor_helper::{Action, Actor, Handle, Receiver}; +use actor_helper::{Action, Handle, Receiver}; use std::time::Duration; -use crate::{GossipReceiver, RecordPublisher}; +use crate::{GossipReceiver, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher}; use anyhow::Result; /// Periodically publishes node state to DHT for peer discovery. /// -/// Publishes a record after an initial 10s delay, then repeatedly with -/// randomized 0-49s intervals, containing this node's active neighbor list +/// Publishes a record after an initial delay initial_delay, then repeatedly with +/// randomized base_interval + rand(0 to max_jitter) intervals, containing this node's active neighbor list /// and message hashes for bubble detection and merging. #[derive(Debug, Clone)] pub struct Publisher { @@ -18,50 +18,74 @@ pub struct Publisher { #[derive(Debug)] struct PublisherActor { - rx: Receiver>, - record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, ticker: tokio::time::Interval, + cancel_token: tokio_util::sync::CancellationToken, + base_interval: Duration, + max_jitter: Duration, } impl Publisher { /// Create a new background publisher. /// /// Spawns a background task that periodically publishes records. - pub fn new(record_publisher: RecordPublisher, gossip_receiver: GossipReceiver) -> Result { - let (api, rx) = Handle::channel(); - - tokio::spawn(async move { - let mut ticker = tokio::time::interval(Duration::from_secs(10)); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let mut actor = PublisherActor { - rx, + pub fn new( + record_publisher: RecordPublisher, + gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, + initial_delay: Duration, + base_interval: Duration, + max_jitter: Duration, + ) -> Result { + let base_interval = base_interval.max(Duration::from_secs(1)); + let mut ticker = + tokio::time::interval_at(tokio::time::Instant::now() + initial_delay, base_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let api = Handle::spawn_with( + PublisherActor { record_publisher, gossip_receiver, ticker, - }; - let _ = actor.run().await; - }); + cancel_token, + base_interval, + max_jitter, + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { _api: api }) } } -impl Actor for PublisherActor { - async fn run(&mut self) -> Result<()> { +impl PublisherActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("Publisher: starting publisher actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } _ = self.ticker.tick() => { tracing::debug!("Publisher: tick fired, attempting to publish"); - let _ = self.publish().await; - let next_interval = rand::random::() % 50; - tracing::debug!("Publisher: next publish in {}s", next_interval); - self.ticker.reset_after(Duration::from_secs(next_interval)); + if let Err(e) = self.publish().await { + tracing::warn!("Publisher: failed to publish record: {:?}", e); + } + let jitter = if self.max_jitter > Duration::ZERO { + Duration::from_nanos((rand::random::() % self.max_jitter.as_nanos()) as u64) + } else { + Duration::ZERO + }; + let next_interval = self.base_interval + jitter; + tracing::debug!("Publisher: next publish in {}ms", next_interval.as_millis()); + self.ticker.reset_after(next_interval); + } + _ = self.cancel_token.cancelled() => { + break Ok(()); } else => break Ok(()), } @@ -76,17 +100,19 @@ impl PublisherActor { let active_peers = self .gossip_receiver .neighbors() - .await + .await? .iter() - .filter_map(|pub_key| TryInto::<[u8; 32]>::try_into(pub_key.as_slice()).ok()) + .filter_map(|pub_key| pub_key.as_bytes().as_ref().try_into().ok()) + .take(MAX_RECORD_PEERS) .collect::>(); let last_message_hashes = self .gossip_receiver .last_message_hashes() - .await + .await? .iter() - .filter_map(|hash| TryInto::<[u8; 32]>::try_into(hash.as_slice()).ok()) + .filter_map(|hash| hash.as_ref().try_into().ok()) + .take(MAX_MESSAGE_HASHES) .collect::>(); tracing::debug!( @@ -97,8 +123,16 @@ impl PublisherActor { ); let record_content = crate::gossip::GossipRecordContent { - active_peers: active_peers.as_slice().try_into()?, - last_message_hashes: last_message_hashes.as_slice().try_into()?, + active_peers: { + let mut peers = [Default::default(); MAX_RECORD_PEERS]; + peers[..active_peers.len()].copy_from_slice(&active_peers); + peers + }, + last_message_hashes: { + let mut hashes = [Default::default(); MAX_MESSAGE_HASHES]; + hashes[..last_message_hashes.len()].copy_from_slice(&last_message_hashes); + hashes + }, }; tracing::debug!("Publisher: created record content: {:?}", record_content); @@ -108,12 +142,15 @@ impl PublisherActor { .new_record(unix_minute, record_content); tracing::debug!("Publisher: created new record: {:?}", res); let record = res?; - let result = self.record_publisher.publish_record(record).await; + let result = self + .record_publisher + .publish_record(record, self.cancel_token.clone()) + .await; - if result.is_ok() { - tracing::debug!("Publisher: successfully published record"); + if let Err(ref e) = result { + tracing::debug!("Publisher: failed to publish record: {:?}", e); } else { - tracing::debug!("Publisher: failed to publish record: {:?}", result); + tracing::debug!("Publisher: successfully published record"); } result diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 7ea0f88..4daab69 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -1,65 +1,18 @@ //! Main topic handle combining bootstrap, publishing, and merging. +use std::sync::{Arc, Weak}; + use crate::{ - GossipSender, - crypto::RecordTopic, + BubbleMergeConfig, Config, GossipSender, MessageOverlapMergeConfig, + config::PublisherConfig, gossip::{ merge::{BubbleMerge, MessageOverlapMerge}, topic::{bootstrap::Bootstrap, publisher::Publisher}, }, }; -use actor_helper::{Action, Actor, Handle, Receiver, act, act_ok}; +use actor_helper::{Handle, act, act_ok}; use anyhow::Result; -use sha2::Digest; - -/// Topic identifier derived from a string via SHA512 hashing. -/// -/// Used as the stable identifier for gossip subscriptions and DHT records. -/// -/// # Example -/// -/// ```ignore -/// let topic_id = TopicId::new("chat-room-1".to_string()); -/// ``` -#[derive(Debug, Clone)] -pub struct TopicId { - _raw: String, - hash: [u8; 32], -} - -impl From for RecordTopic { - fn from(val: TopicId) -> Self { - RecordTopic::from_bytes(&val.hash) - } -} - -impl TopicId { - /// Create a new topic ID from a string. - /// - /// String is hashed with SHA512; the first 32 bytes produce the identifier. - pub fn new(raw: String) -> Self { - let mut raw_hash = sha2::Sha512::new(); - raw_hash.update(raw.as_bytes()); - - Self { - _raw: raw, - hash: raw_hash.finalize()[..32] - .try_into() - .expect("hashing 'raw' failed"), - } - } - - /// Get the hash bytes. - pub fn hash(&self) -> [u8; 32] { - self.hash - } - - /// Get the original string. - #[allow(dead_code)] - pub fn raw(&self) -> &str { - &self._raw - } -} +use tokio_util::sync::CancellationToken; /// Handle to a joined gossip topic with auto-discovery. /// @@ -67,17 +20,24 @@ impl TopicId { /// Can be split into sender and receiver for message exchange. #[derive(Debug, Clone)] pub struct Topic { - api: Handle, + api: Arc>, + cancel_token: CancellationToken, } #[derive(Debug)] struct TopicActor { - rx: Receiver>, bootstrap: Bootstrap, publisher: Option, bubble_merge: Option, message_overlap_merge: Option, record_publisher: crate::crypto::RecordPublisher, + cancel_token: CancellationToken, +} + +impl Drop for TopicActor { + fn drop(&mut self) { + self.cancel_token.cancel(); + } } impl Topic { @@ -98,54 +58,77 @@ impl Topic { async_bootstrap ); - let (api, rx) = Handle::channel(); - - let bootstrap = Bootstrap::new(record_publisher.clone(), gossip.clone()).await?; + let cancel_token = CancellationToken::new(); + let bootstrap = Bootstrap::new( + record_publisher.clone(), + gossip.clone(), + cancel_token.clone(), + record_publisher.config().timeouts().clone(), + record_publisher.config().bootstrap_config().clone(), + ) + .await?; tracing::debug!("Topic: bootstrap instance created"); + let api = Arc::new( + Handle::spawn(TopicActor { + bootstrap: bootstrap.clone(), + record_publisher: record_publisher.clone(), + publisher: None, + bubble_merge: None, + message_overlap_merge: None, + cancel_token: cancel_token.clone(), + }) + .0, + ); + + let bootstrap_done = bootstrap.bootstrap().await?; + let config = record_publisher.config().clone(); tokio::spawn({ - let bootstrap = bootstrap.clone(); + let api = Arc::downgrade(&api); + let config = config.clone(); + let cancel_token = cancel_token.clone(); async move { - tracing::debug!("Topic: starting topic actor"); - let mut actor = TopicActor { - rx, - bootstrap: bootstrap.clone(), - record_publisher, - publisher: None, - bubble_merge: None, - message_overlap_merge: None, - }; - let _ = actor.run().await; + if let Err(err) = wait_for_bootstrap(bootstrap_done, cancel_token.clone()).await { + tracing::warn!("bootstrap failed: {}", err); + return; + } + + if async_bootstrap { + tracing::debug!("Bootstrap completed, now spawning workers"); + if let Err(err) = spawn_workers(api, config, cancel_token.clone()).await { + cancel_token.cancel(); + tracing::warn!("failed to spawn workers: {}", err); + } + } } }); - let bootstrap_done = bootstrap.bootstrap().await?; if !async_bootstrap { tracing::debug!("Topic: waiting for bootstrap to complete"); - bootstrap_done.await?; + bootstrap.gossip_receiver().await?.joined().await?; + if let Err(err) = + spawn_workers(Arc::downgrade(&api), config, cancel_token.clone()).await + { + tracing::warn!("failed to spawn workers: {}", err); + cancel_token.cancel(); + return Err(anyhow::anyhow!("failed to spawn workers: {}", err)); + } tracing::debug!("Topic: bootstrap completed"); } else { tracing::debug!("Topic: bootstrap started asynchronously"); } - tracing::debug!("Topic: starting publisher"); - let _ = api.call(act!(actor => actor.start_publishing())).await; - - tracing::debug!("Topic: starting bubble merge"); - let _ = api.call(act!(actor => actor.start_bubble_merge())).await; - - tracing::debug!("Topic: starting message overlap merge"); - let _ = api - .call(act!(actor => actor.start_message_overlap_merge())) - .await; - - tracing::debug!("Topic: fully initialized"); - Ok(Self { api }) + Ok(Self { api, cancel_token }) } /// Split into sender and receiver for message exchange. pub async fn split(&self) -> Result<(GossipSender, crate::gossip::receiver::GossipReceiver)> { - Ok((self.gossip_sender().await?, self.gossip_receiver().await?)) + let topic_ref = Arc::new(self.clone()); + let mut sender = self.gossip_sender().await?; + let mut receiver = self.gossip_receiver().await?; + sender._topic_keep_alive = Some(topic_ref.clone()); + receiver._topic_keep_alive = Some(topic_ref); + Ok((sender, receiver)) } /// Get the gossip sender for this topic. @@ -168,54 +151,326 @@ impl Topic { .call(act_ok!(actor => async move { actor.record_publisher.clone() })) .await } + + #[allow(dead_code)] + pub(crate) fn cancel_token(&self) -> CancellationToken { + self.cancel_token.clone() + } } -impl Actor for TopicActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - let _ = action(self).await; +async fn wait_for_bootstrap( + bootstrap_done: tokio::sync::oneshot::Receiver>, + cancel_token: CancellationToken, +) -> Result<()> { + if let Ok(Ok(_)) = bootstrap_done.await { + Ok(()) + } else { + tracing::error!("Topic: bootstrap failed or cancelled, shutting down topic"); + cancel_token.cancel(); + Err(anyhow::anyhow!("bootstrap failed or cancelled")) + } +} + +async fn spawn_workers( + api: Weak>, + config: Config, + cancel_token: CancellationToken, +) -> Result<()> { + if !cancel_token.is_cancelled() { + if matches!(config.publisher_config(), PublisherConfig::Enabled(_)) { + tracing::debug!("Topic: starting publisher"); + match api.upgrade() { + Some(api) => { + if let Err(err) = api.call(act!(actor => actor.start_publishing())).await { + return Err(anyhow::anyhow!("failed to start publisher: {err}")); + } + } + None => { + return Err(anyhow::anyhow!( + "failed to start publisher, topic actor dropped" + )); + } + } + } + + if matches!( + config.merge_config().bubble_merge(), + BubbleMergeConfig::Enabled(_) + ) { + tracing::debug!("Topic: starting bubble merge"); + match api.upgrade() { + Some(api) => { + if let Err(err) = api.call(act!(actor => actor.start_bubble_merge())).await { + return Err(anyhow::anyhow!("failed to start bubble merge: {err}")); + } + } + None => { + return Err(anyhow::anyhow!( + "failed to start bubble merge, topic actor dropped" + )); } - else => break Ok(()), } } + + if matches!( + config.merge_config().message_overlap_merge(), + MessageOverlapMergeConfig::Enabled(_) + ) { + tracing::debug!("Topic: starting message overlap merge"); + match api.upgrade() { + Some(api) => { + if let Err(err) = api + .call(act!(actor => actor.start_message_overlap_merge())) + .await + { + return Err(anyhow::anyhow!( + "failed to start message overlap merge: {err}" + )); + } + } + None => { + return Err(anyhow::anyhow!( + "failed to start message overlap merge, topic actor dropped" + )); + } + } + } + tracing::debug!("Topic: spawn_worker finished"); + Ok(()) + } else { + tracing::warn!("Topic: cancelled before workers could be spawned"); + Err(anyhow::anyhow!("cancelled before workers could be spawned")) } } impl TopicActor { pub async fn start_publishing(&mut self) -> Result<()> { - tracing::debug!("TopicActor: initializing publisher"); - let publisher = Publisher::new( - self.record_publisher.clone(), - self.bootstrap.gossip_receiver().await?, - )?; - self.publisher = Some(publisher); - tracing::debug!("TopicActor: publisher started"); + if let PublisherConfig::Enabled(config) = self.record_publisher.config().publisher_config() + { + tracing::debug!("TopicActor: initializing publisher"); + let publisher = async { + Publisher::new( + self.record_publisher.clone(), + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + config.initial_delay(), + config.base_interval(), + config.max_jitter(), + ) + }; + + match publisher.await { + Ok(publisher) => { + self.publisher = Some(publisher); + tracing::debug!("TopicActor: publisher started"); + } + Err(err) => { + if config.fail_topic_creation_on_publishing_startup_failure() { + return Err(anyhow::anyhow!("failed to start publisher: {}", err)); + } else { + tracing::warn!( + "TopicActor: failed to start publisher: {}, but continuing because Publisher.fail_topic_creation_on_publishing_startup_failure is false", + err + ); + } + } + } + } Ok(()) } pub async fn start_bubble_merge(&mut self) -> Result<()> { - tracing::debug!("TopicActor: initializing bubble merge"); - let bubble_merge = BubbleMerge::new( - self.record_publisher.clone(), - self.bootstrap.gossip_sender().await?, - self.bootstrap.gossip_receiver().await?, - )?; - self.bubble_merge = Some(bubble_merge); - tracing::debug!("TopicActor: bubble merge started"); + if let BubbleMergeConfig::Enabled(config) = + self.record_publisher.config().merge_config().bubble_merge() + { + tracing::debug!("TopicActor: initializing bubble merge"); + let bubble_merge = async { + BubbleMerge::new( + self.record_publisher.clone(), + self.bootstrap.gossip_sender().await?, + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + self.record_publisher.config().max_join_peer_count(), + config.base_interval(), + config.max_jitter(), + config.min_neighbors(), + ) + }; + + match bubble_merge.await { + Ok(bubble_merge) => { + self.bubble_merge = Some(bubble_merge); + tracing::debug!("TopicActor: bubble merge started"); + } + Err(err) => { + if config.fail_topic_creation_on_merge_startup_failure() { + return Err(anyhow::anyhow!("failed to start bubble merge: {}", err)); + } else { + tracing::warn!( + "TopicActor: failed to start bubble merge: {}, but continuing because BubbleMerge.fail_topic_creation_on_merge_startup_failure is false", + err + ); + } + } + } + } Ok(()) } pub async fn start_message_overlap_merge(&mut self) -> Result<()> { - tracing::debug!("TopicActor: initializing message overlap merge"); - let message_overlap_merge = MessageOverlapMerge::new( - self.record_publisher.clone(), - self.bootstrap.gossip_sender().await?, - self.bootstrap.gossip_receiver().await?, - )?; - self.message_overlap_merge = Some(message_overlap_merge); - tracing::debug!("TopicActor: message overlap merge started"); + if let MessageOverlapMergeConfig::Enabled(config) = self + .record_publisher + .config() + .merge_config() + .message_overlap_merge() + { + tracing::debug!("TopicActor: initializing message overlap merge"); + let message_overlap_merge = async { + MessageOverlapMerge::new( + self.record_publisher.clone(), + self.bootstrap.gossip_sender().await?, + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + self.record_publisher.config().max_join_peer_count(), + config.base_interval(), + config.max_jitter(), + ) + }; + + match message_overlap_merge.await { + Ok(message_overlap_merge) => { + self.message_overlap_merge = Some(message_overlap_merge); + tracing::debug!("TopicActor: message overlap merge started"); + } + Err(err) => { + if config.fail_topic_creation_on_merge_startup_failure() { + return Err(anyhow::anyhow!( + "failed to start message overlap merge: {}", + err + )); + } else { + tracing::warn!( + "TopicActor: failed to start message overlap merge: {}, but continuing because MessageOverlapMerge.fail_topic_creation_on_merge_startup_failure is false", + err + ); + } + } + } + } Ok(()) } } + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn test_receiver_returns_none_after_shutdown() { + let secret_key = iroh::SecretKey::generate(&mut rand::rng()); + let signing_key = mainline::SigningKey::from_bytes(&secret_key.to_bytes()); + let endpoint = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await + .expect("failed to bind endpoint"); + let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); + + let topic_id = crate::TopicId::new("shutdown-receiver-test".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + let record_publisher = crate::RecordPublisher::new( + topic_id.clone(), + signing_key.clone(), + None, + initial_secret, + crate::config::Config::default(), + ); + + let topic = crate::Topic::new(record_publisher, gossip.clone(), true) + .await + .expect("failed to create topic"); + + let cancel_token = topic.cancel_token(); + let (_sender, receiver) = topic.split().await.expect("failed to split topic"); + + // Clone the receiver before shutdown, this survivor must not hang + let mut survivor = receiver.clone(); + + cancel_token.cancel(); + + // next() on a receiver that was alive before shutdown must return ChannelError, + // not hang. If the broadcast channel didn't close, this would block forever + let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.next()) + .await + .expect("next() hung after shutdown - broadcast channel didn't close"); + assert!(result.is_err(), "expected Err from next() after shutdown"); + + // joined() must also return Err after shutdown + let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.joined()) + .await + .expect("joined() hung after shutdown - broadcast channel didn't close"); + assert!(result.is_err(), "expected Err from joined() after shutdown"); + + // A clone made after shutdown must also return Err immediately + // (WeakSender::upgrade fails -> gets an already closed channel) + let mut late_clone = survivor.clone(); + + let result = tokio::time::timeout(std::time::Duration::from_secs(5), late_clone.next()) + .await + .expect("next() hung on post shutdown clone, WeakSender upgrade should fail"); + assert!( + result.is_err(), + "expected Err from next() on post shutdown clone" + ); + + let result = tokio::time::timeout(std::time::Duration::from_secs(5), late_clone.joined()) + .await + .expect("joined() hung on post shutdown clone, WeakSender upgrade should fail"); + assert!( + result.is_err(), + "expected Err from joined() on post shutdown clone" + ); + } + + #[tokio::test] + async fn test_topic_full_shutdown_on_drop() { + let secret_key = iroh::SecretKey::generate(&mut rand::rng()); + let signing_key = mainline::SigningKey::from_bytes(&secret_key.to_bytes()); + let endpoint = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await + .expect("failed to bind endpoint"); + let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); + + let topic_id = crate::TopicId::new("my-iroh-gossip-topic".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + let record_publisher = crate::RecordPublisher::new( + topic_id.clone(), + signing_key.clone(), + None, + initial_secret, + crate::config::Config::default(), + ); + + let topic = crate::Topic::new(record_publisher, gossip.clone(), true) + .await + .expect("failed to create Topic"); + + let cancel_token = topic.cancel_token(); + + let (sender, receiver) = topic.split().await.expect("failed to split topic"); + + assert!(!cancel_token.is_cancelled()); + + drop(sender); + drop(receiver); + drop(topic); + + tokio::time::timeout(std::time::Duration::from_secs(5), cancel_token.cancelled()) + .await + .expect("cancel token timed out"); + + assert!(cancel_token.is_cancelled()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 1001740..bd76e6d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ #![doc = include_str!("../README.md")] +mod config; mod crypto; mod dht; @@ -8,20 +9,24 @@ mod gossip; #[cfg(feature = "iroh-gossip")] pub use gossip::{ AutoDiscoveryGossip, Bootstrap, BubbleMerge, GossipReceiver, GossipRecordContent, GossipSender, - MessageOverlapMerge, Publisher, Topic, TopicId, + MessageOverlapMerge, Publisher, Topic, }; +pub use config::{ + BootstrapConfig, BootstrapConfigBuilder, BubbleMergeConfig, BubbleMergeConfigBuilder, Config, + ConfigBuilder, DhtConfig, DhtConfigBuilder, MergeConfig, MergeConfigBuilder, + MessageOverlapMergeConfig, MessageOverlapMergeConfigBuilder, PublisherConfig, + PublisherConfigBuilder, TimeoutConfig, TimeoutConfigBuilder, +}; pub use crypto::{ - DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RecordTopic, RotationHandle, - SecretRotation, encryption_keypair, salt, signing_keypair, + DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RotationHandle, + SecretRotation, TopicId, encryption_keypair, salt, signing_keypair, }; pub use dht::Dht; -/// Maximum number of bootstrap records allowed per topic per time slot (minute). -/// -/// When publishing to the DHT, records are not published if this threshold -/// has already been reached for the current minute slot. -pub const MAX_BOOTSTRAP_RECORDS: usize = 100; +/// These are part of the on-wire format: DO NOT CHANGE without increasing protocol version. +pub const MAX_RECORD_PEERS: usize = 5; +pub const MAX_MESSAGE_HASHES: usize = 5; /// Get the current Unix minute timestamp, optionally offset. /// @@ -35,6 +40,9 @@ pub const MAX_BOOTSTRAP_RECORDS: usize = 100; /// let now = unix_minute(0); /// let prev_minute = unix_minute(-1); /// ``` +#[doc(hidden)] pub fn unix_minute(minute_offset: i64) -> u64 { - ((chrono::Utc::now().timestamp() as f64 / 60.0f64).floor() as i64 + minute_offset) as u64 + ((chrono::Utc::now().timestamp() / 60).saturating_add(minute_offset)) + .try_into() + .expect("timestamp overflow") } diff --git a/tests/test_gossip.rs b/tests/test_gossip.rs deleted file mode 100644 index 064ac19..0000000 --- a/tests/test_gossip.rs +++ /dev/null @@ -1,16 +0,0 @@ -use distributed_topic_tracker::TopicId; - -#[test] -fn test_topic_id_creation() { - let topic_id = TopicId::new("test-topic".to_string()); - assert_eq!(topic_id.raw(), "test-topic"); - assert_eq!(topic_id.hash().len(), 32); - - // Same input should produce same hash - let topic_id2 = TopicId::new("test-topic".to_string()); - assert_eq!(topic_id.hash(), topic_id2.hash()); - - // Different input should produce different hash - let topic_id3 = TopicId::new("different-topic".to_string()); - assert_ne!(topic_id.hash(), topic_id3.hash()); -} diff --git a/tests/tests.rs b/tests/tests.rs index 5d5cd17..67b75cb 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,44 +1,40 @@ -use std::str::FromStr; +use std::{str::FromStr, sync::Arc}; use distributed_topic_tracker::{ - DefaultSecretRotation, EncryptedRecord, GossipRecordContent, Record, RecordTopic, - RotationHandle, encryption_keypair, salt, signing_keypair, unix_minute, + AutoDiscoveryGossip, BubbleMergeConfig, Config, DefaultSecretRotation, EncryptedRecord, + GossipReceiver, GossipRecordContent, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, MergeConfig, + MessageOverlapMergeConfig, PublisherConfig, Record, RecordPublisher, RotationHandle, TopicId, + encryption_keypair, salt, signing_keypair, unix_minute, }; use mainline::SigningKey; +use tokio::sync::Barrier; #[test] fn test_record_serialization_roundtrip() { let signing_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = [2u8; 32]; - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign( - topic, - unix_minute, - node_id, - record_content.clone(), - &signing_key, - ) - .expect("Failed to sign record"); + let record = Record::sign(topic, unix_minute, record_content.clone(), &signing_key) + .expect("failed to sign record"); // Test serialization roundtrip let bytes = record.to_bytes(); - let deserialized = Record::from_bytes(bytes).expect("Failed to deserialize record"); + let deserialized = Record::from_bytes(bytes).expect("failed to deserialize record"); let deserialized_content: GossipRecordContent = - deserialized.content().expect("Failed to get content"); + deserialized.content().expect("failed to get content"); assert_eq!(record.topic(), deserialized.topic()); assert_eq!(record.unix_minute(), deserialized.unix_minute()); - assert_eq!(record.node_id(), deserialized.node_id()); + assert_eq!(record.pub_key(), deserialized.pub_key()); assert_eq!( record_content.active_peers, deserialized_content.active_peers @@ -55,16 +51,16 @@ fn test_record_verification() { let signing_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = signing_key.verifying_key().to_bytes(); - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign(topic, unix_minute, node_id, record_content, &signing_key).unwrap(); + let record = Record::sign(topic, unix_minute, record_content, &signing_key) + .expect("failed to sign record"); // Valid verification should pass assert!(record.verify(&topic, unix_minute).is_ok()); @@ -83,34 +79,29 @@ fn test_encrypted_record_roundtrip() { let encryption_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = signing_key.verifying_key().to_bytes(); - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign( - topic, - unix_minute, - node_id, - record_content.clone(), - &signing_key, - ) - .expect("Failed to sign record"); + let record = Record::sign(topic, unix_minute, record_content.clone(), &signing_key) + .expect("failed to sign record"); // Test encryption/decryption roundtrip let encrypted = record.encrypt(&encryption_key); - let decrypted = encrypted.decrypt(&encryption_key).unwrap(); + let decrypted = encrypted + .decrypt(&encryption_key) + .expect("failed to decrypt record"); let deserialized_content: GossipRecordContent = - decrypted.content().expect("Failed to get content"); + decrypted.content().expect("failed to get content"); assert_eq!(record.topic(), decrypted.topic()); assert_eq!(record.unix_minute(), decrypted.unix_minute()); - assert_eq!(record.node_id(), decrypted.node_id()); + assert_eq!(record.pub_key(), decrypted.pub_key()); assert_eq!( record_content.active_peers, deserialized_content.active_peers @@ -128,29 +119,28 @@ fn test_encrypted_record_serialization() { let encryption_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = signing_key.verifying_key().to_bytes(); - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign(topic, unix_minute, node_id, record_content, &signing_key) - .expect("Failed to sign record"); + let record = Record::sign(topic, unix_minute, record_content, &signing_key) + .expect("failed to sign record"); let encrypted = record.encrypt(&encryption_key); // Test serialization roundtrip - let bytes = encrypted.to_bytes(); + let bytes = encrypted.to_bytes().expect("record max size must be < EncryptedRecord::MAX_SIZE"); let deserialized = - EncryptedRecord::from_bytes(bytes).expect("Failed to deserialize encrypted record"); + EncryptedRecord::from_bytes(bytes).expect("failed to deserialize encrypted record"); // Should be able to decrypt the deserialized version let decrypted = deserialized .decrypt(&encryption_key) - .expect("Failed to decrypt record"); + .expect("failed to decrypt record"); assert_eq!(record.topic(), decrypted.topic()); assert_eq!(record.unix_minute(), decrypted.unix_minute()); } @@ -194,58 +184,212 @@ fn test_unix_minute_function() { #[test] fn test_topic_signing_keypair_deterministic() { - let topic_id = RecordTopic::from_str("test-topic").unwrap(); - let record_topic = topic_id; + let topic_id = TopicId::from_str("test-topic").expect("failed to create TopicId from_str"); let unix_minute = 12345u64; - let key1 = signing_keypair(record_topic, unix_minute); - let key2 = signing_keypair(record_topic, unix_minute); + let key1 = signing_keypair(&topic_id, unix_minute); + let key2 = signing_keypair(&topic_id, unix_minute); // Same inputs should produce same keypair assert_eq!(key1.to_bytes(), key2.to_bytes()); // Different unix_minute should produce different keypair - let key3 = signing_keypair(record_topic, unix_minute + 1); + let key3 = signing_keypair(&topic_id, unix_minute + 1); assert_ne!(key1.to_bytes(), key3.to_bytes()); } #[test] fn test_topic_encryption_keypair_deterministic() { - let topic_id = RecordTopic::from_str("test-topic").unwrap(); - let record_topic = topic_id; + let topic_id = TopicId::from_str("test-topic").expect("failed to create TopicId from_str"); let unix_minute = 12345u64; let initial_secret_hash = [1u8; 32]; let rotation = RotationHandle::new(DefaultSecretRotation); - let key1 = encryption_keypair(record_topic, &rotation, initial_secret_hash, unix_minute); - let key2 = encryption_keypair(record_topic, &rotation, initial_secret_hash, unix_minute); + let key1 = encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute); + let key2 = encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute); // Same inputs should produce same keypair assert_eq!(key1.to_bytes(), key2.to_bytes()); // Different unix_minute should produce different keypair - let key3 = crate::encryption_keypair( - record_topic, - &rotation, - initial_secret_hash, - unix_minute + 1, - ); + let key3 = encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute + 1); assert_ne!(key1.to_bytes(), key3.to_bytes()); } #[test] fn test_topic_salt_deterministic() { - let topic_id = RecordTopic::from_str("test-topic").unwrap(); - let record_topic = topic_id; + let topic_id = TopicId::from_str("test-topic").expect("failed to create TopicId from_str"); let unix_minute = 12345u64; - let salt1 = salt(record_topic, unix_minute); - let salt2 = salt(record_topic, unix_minute); + let salt1 = salt(&topic_id, unix_minute); + let salt2 = salt(&topic_id, unix_minute); // Same inputs should produce same salt assert_eq!(salt1, salt2); // Different unix_minute should produce different salt - let salt3 = salt(record_topic, unix_minute + 1); + let salt3 = salt(&topic_id, unix_minute + 1); assert_ne!(salt1, salt3); } + +#[test] +fn test_topic_id_creation() { + let topic_id = TopicId::new("test-topic".to_string()); + assert_eq!(topic_id.hash().len(), 32); + + // Same input should produce same hash + let topic_id2 = TopicId::new("test-topic".to_string()); + assert_eq!(topic_id.hash(), topic_id2.hash()); + + // Different input should produce different hash + let topic_id3 = TopicId::new("different-topic".to_string()); + assert_ne!(topic_id.hash(), topic_id3.hash()); +} + +#[cfg(feature = "iroh-gossip")] +#[tokio::test] +async fn test_multiple_receivers_all_get_events() { + const N: usize = 3; + const MSG_COUNT: usize = 3; + + let config = Config::builder() + .publisher_config(PublisherConfig::Disabled) + .merge_config(MergeConfig::builder() + .bubble_merge(BubbleMergeConfig::Disabled) + .message_overlap_merge(MessageOverlapMergeConfig::Disabled) + .build() + ) + .build(); + + let topic_id = TopicId::new("test-multi-receiver".to_string()); + + // Peer A + let secret_a = iroh::SecretKey::generate(&mut rand::rng()); + let signing_a = mainline::SigningKey::from_bytes(&secret_a.to_bytes()); + let endpoint_a = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_a) + .bind() + .await + .expect("failed to bind endpoint A"); + let gossip_a = iroh_gossip::net::Gossip::builder().spawn(endpoint_a.clone()); + let _router_a = iroh::protocol::Router::builder(endpoint_a.clone()) + .accept(iroh_gossip::ALPN, gossip_a.clone()) + .spawn(); + + let rp_a = RecordPublisher::new( + topic_id.clone(), + signing_a, + None, + b"secret".to_vec(), + config.clone(), + ); + + let topic_a = gossip_a + .subscribe_and_join_with_auto_discovery_no_wait(rp_a) + .await + .expect("failed to subscribe and join topic A"); + let (sender_a, mut receiver_a) = topic_a.split().await.expect("failed to split topic A"); + + // Peer B + let secret_b = iroh::SecretKey::generate(&mut rand::rng()); + let signing_b = mainline::SigningKey::from_bytes(&secret_b.to_bytes()); + let endpoint_b = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_b) + .bind() + .await + .expect("failed to bind endpoint B"); + let gossip_b = iroh_gossip::net::Gossip::builder().spawn(endpoint_b.clone()); + let _router_b = iroh::protocol::Router::builder(endpoint_b.clone()) + .accept(iroh_gossip::ALPN, gossip_b.clone()) + .spawn(); + + let rp_b = RecordPublisher::new( + topic_id.clone(), + signing_b, + None, + b"secret".to_vec(), + config, + ); + + let topic_b = gossip_b + .subscribe_and_join_with_auto_discovery(rp_b) + .await + .expect("failed to subscribe and join topic B"); + let (sender_b, mut receiver_b) = topic_b.split().await.expect("failed to split topic B"); + + // Join peers + sender_a + .join_peers(vec![endpoint_b.id()], None) + .await + .expect("failed to join peers from sender A"); + sender_b + .join_peers(vec![endpoint_a.id()], None) + .await + .expect("failed to join peers from sender B"); + + receiver_a + .joined() + .await + .expect("failed to wait for receiver A to join"); + receiver_b + .joined() + .await + .expect("failed to wait for receiver B to join"); + + let receivers: Vec = (0..N) + .map(|_| receiver_b.clone()) + .chain((0..N).map(|_| receiver_a.clone())) + .collect(); + let barrier = Arc::new(Barrier::new(receivers.len() + 1)); + + let handles = receivers + .into_iter() + .enumerate() + .map(|(i, mut rx)| { + let barrier = barrier.clone(); + tokio::spawn(async move { + barrier.wait().await; + + let mut received = Vec::new(); + while received.len() < MSG_COUNT { + match tokio::time::timeout(std::time::Duration::from_secs(30), rx.next()).await + { + Ok(Ok(iroh_gossip::api::Event::Received(msg))) => { + received.push(msg.content.to_vec()); + } + Ok(Ok(_)) => continue, + other => panic!("receiver {i}: unexpected result: {other:?}"), + } + } + received + }) + }) + .collect::>(); + + barrier.wait().await; + + for i in 0..MSG_COUNT { + sender_a + .broadcast(format!("msg-a-{i}").into_bytes()) + .await + .expect("failed to broadcast from sender A"); + sender_b + .broadcast(format!("msg-b-{i}").into_bytes()) + .await + .expect("failed to broadcast from sender B"); + } + + for (i, handle) in handles.into_iter().enumerate() { + let received = tokio::time::timeout(std::time::Duration::from_secs(60), handle) + .await + .expect(&format!("receiver {i} timed out")) + .expect(&format!("receiver {i} panicked")); + + assert_eq!( + received.len(), + MSG_COUNT, + "receiver {i} got {} messages, expected {MSG_COUNT}", + received.len(), + ); + } +}