From eabc354a57dd629aacb3dc7ca4d90bd6d5e47b34 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 12 Apr 2026 15:57:41 +0200 Subject: [PATCH 01/30] chore: dep update actor-helper --- Cargo.lock | 404 ++++++++++++---------------- Cargo.toml | 2 +- src/dht.rs | 29 +- src/gossip/merge/bubble.rs | 27 +- src/gossip/merge/message_overlap.rs | 25 +- src/gossip/receiver.rs | 35 ++- src/gossip/sender.rs | 33 +-- src/gossip/topic/bootstrap.rs | 24 +- src/gossip/topic/publisher.rs | 28 +- src/gossip/topic/topic.rs | 41 +-- 10 files changed, 249 insertions(+), 399 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f024c20..c12b16f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "actor-helper" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db99f3032635124f4ad5639cfdb8fc571c84fd9c6f351e05dab7c6558ca2b157" +checksum = "e27b633e8510b190e2a968f872136b8a53b38d855089f56030ce0ead131b3377" dependencies = [ "anyhow", "flume 0.12.0", @@ -33,7 +33,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -178,16 +178,16 @@ checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.57" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "shlex", @@ -268,7 +268,7 @@ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -368,6 +368,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -461,7 +470,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "fiat-crypto 0.2.9", "rustc_version", @@ -476,7 +485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f9200d1d13637f15a6acb71e758f64624048d85b31a5fdbfd8eca1e2687d0b7" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.11.0-rc.10", "fiat-crypto 0.3.0", @@ -845,9 +854,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "ff" @@ -1208,6 +1217,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heapless" version = "0.7.17" @@ -1373,18 +1388,18 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -1396,7 +1411,6 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -1404,15 +1418,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "c2b52f86d1d4bc0d6b4e6826d960b1b333217e07d36b882dca570a5e1c48895b" dependencies = [ "http", "hyper", "hyper-util", "rustls", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -1436,7 +1449,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "tokio", "tower-service", "tracing", @@ -1468,12 +1481,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -1481,9 +1495,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -1494,9 +1508,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1508,15 +1522,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -1528,15 +1542,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -1588,12 +1602,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -1609,14 +1623,15 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ - "socket2 0.5.10", + "socket2", "widestring", - "windows-sys 0.48.0", - "winreg", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", ] [[package]] @@ -1627,9 +1642,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -1841,16 +1856,18 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1869,9 +1886,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "libm" @@ -1881,9 +1898,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "litrs" @@ -1979,9 +1996,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -1990,9 +2007,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -2154,7 +2171,7 @@ dependencies = [ "objc2-system-configuration", "pin-project-lite", "serde", - "socket2 0.6.3", + "socket2", "time", "tokio", "tokio-util", @@ -2178,7 +2195,7 @@ dependencies = [ "pin-project-lite", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror", "tokio", "tokio-stream", @@ -2221,7 +2238,7 @@ checksum = "bb9be4fedd6b98f3ba82ccd3506f4d0219fb723c3f97c67e12fe1494aa020e44" dependencies = [ "cfg_aliases", "libc", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.61.2", ] @@ -2252,9 +2269,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-traits" @@ -2377,9 +2394,9 @@ dependencies = [ [[package]] name = "papaya" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f" +checksum = "997ee03cd38c01469a7046643714f0ad28880bcb9e6679ff0666e24817ca19b7" dependencies = [ "equivalent", "seize", @@ -2471,17 +2488,11 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkarr" -version = "5.0.3" +version = "5.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f950360d31be432c0c9467fba5024a94f55128e7f32bc9d32db140369f24c77" +checksum = "d7bfb9143bbba379f246211eb68074d78db9cc048e4c5701f3b0e6cb1ec67ca2" dependencies = [ "base32", "bytes", @@ -2525,7 +2536,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" dependencies = [ - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2537,7 +2548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2578,9 +2589,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2659,7 +2670,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror", "tokio", "tracing", @@ -2696,7 +2707,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -2724,9 +2735,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -2849,9 +2860,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -2889,9 +2900,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" dependencies = [ "ring", "rustls-pki-types", @@ -2953,9 +2964,9 @@ checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "send_wrapper" @@ -3051,7 +3062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.10.7", ] @@ -3062,7 +3073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1e3878ab0f98e35b2df35fe53201d088299b41a6bb63e3e34dada2ac4abd924" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest 0.11.0-rc.10", ] @@ -3105,9 +3116,9 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple-dns" -version = "0.9.3" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee851d0e5e7af3721faea1843e8015e820a234f81fda3dea9247e15bac9a86a" +checksum = "df350943049174c4ae8ced56c604e28270258faec12a6a48637a7655287c9ce0" dependencies = [ "bitflags", ] @@ -3130,16 +3141,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -3184,9 +3185,9 @@ checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" [[package]] name = "spki" -version = "0.8.0-rc.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8baeff88f34ed0691978ec34440140e1572b68c7dd4a495fd14a3dc1944daa80" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", "der 0.8.0", @@ -3333,9 +3334,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -3358,25 +3359,25 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" dependencies = [ "bytes", "libc", "mio", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -3443,18 +3444,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.0.1+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.5+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca1a40644a28bce036923f6a431df0b34236949d111cc07cb6dca830c9ef2e1" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", "toml_datetime", @@ -3464,9 +3465,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] @@ -3598,9 +3599,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-xid" @@ -3645,9 +3646,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -3749,9 +3750,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -3762,23 +3763,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3786,9 +3783,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -3799,9 +3796,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -3855,9 +3852,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -3993,6 +3990,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -4011,15 +4019,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -4056,21 +4055,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -4113,12 +4097,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4131,12 +4109,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4149,12 +4121,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4179,12 +4145,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4197,12 +4157,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4215,12 +4169,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4233,12 +4181,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4253,23 +4195,13 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -4360,9 +4292,9 @@ dependencies = [ [[package]] name = "wmi" -version = "0.18.3" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "003e65f4934cf9449b9ce913ad822cd054a5af669d24f93db101fdb02856bb23" +checksum = "7c81b85c57a57500e56669586496bf2abd5cf082b9d32995251185d105208b64" dependencies = [ "chrono", "futures", @@ -4375,9 +4307,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "ws_stream_wasm" @@ -4412,9 +4344,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -4423,9 +4355,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -4441,18 +4373,18 @@ checksum = "2164e798d9e3d84ee2c91139ace54638059a3b23e361f5c11781c2c6459bde0f" [[package]] name = "zerocopy" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.42" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -4461,18 +4393,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -4502,9 +4434,9 @@ dependencies = [ [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -4513,9 +4445,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -4524,9 +4456,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index ed523b1..08572ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ chrono = { version = "0.4", default-features = false, features = ["clock"] } mainline = { version = "6", default-features = false, features = ["async"]} rand = { version = "0.9", default-features = false, features = ["std", "std_rng"] } -actor-helper = { version = "0.2", features = ["tokio", "anyhow"] } +actor-helper = { version = "0.3", features = ["tokio", "anyhow"] } postcard = "1" serde = { version = "1", default-features = false, features = ["std"] } diff --git a/src/dht.rs b/src/dht.rs index dc3f421..f533a71 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -5,7 +5,7 @@ use std::time::Duration; -use actor_helper::{Action, Actor, Handle, Receiver, act}; +use actor_helper::{Handle, act}; use anyhow::{Context, Result, bail}; use ed25519_dalek::VerifyingKey; use futures_lite::StreamExt; @@ -22,9 +22,8 @@ pub struct Dht { api: Handle, } -#[derive(Debug)] +#[derive(Debug, Default)] struct DhtActor { - rx: Receiver>, dht: Option, } @@ -33,14 +32,9 @@ impl Dht { /// /// Spawns a background actor for handling DHT operations. pub fn new() -> Self { - let (api, rx) = Handle::channel(); - - tokio::spawn(async move { - let mut actor = DhtActor { rx, dht: None }; - let _ = actor.run().await; - }); - - Self { api } + Self { + api: Handle::spawn(DhtActor::default()).0, + } } /// Retrieve mutable records from the DHT. @@ -92,19 +86,6 @@ impl Default for Dht { } } -impl Actor for DhtActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; - } - else => break Ok(()), - } - } - } -} - impl DhtActor { pub async fn get( &mut self, diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 7c28945..2157e98 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -2,7 +2,7 @@ //! //! If local peer count < 4, extract suggested peers from DHT records and join them. -use actor_helper::{Action, Actor, Handle, Receiver}; +use actor_helper::Handle; use iroh::EndpointId; use std::{collections::HashSet, time::Duration}; @@ -20,8 +20,6 @@ pub struct BubbleMerge { #[derive(Debug)] struct BubbleMergeActor { - rx: Receiver>, - record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, gossip_sender: GossipSender, @@ -37,32 +35,33 @@ impl BubbleMerge { gossip_sender: GossipSender, gossip_receiver: GossipReceiver, ) -> Result { - let (api, rx) = Handle::channel(); - let mut ticker = tokio::time::interval(Duration::from_secs(10)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - tokio::spawn(async move { - let mut actor = BubbleMergeActor { - rx, + let api = Handle::spawn_with( + BubbleMergeActor { record_publisher, gossip_receiver, gossip_sender, ticker, - }; - let _ = actor.run().await; - }); + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { _api: api }) } } -impl Actor for BubbleMergeActor { - async fn run(&mut self) -> Result<()> { +impl BubbleMergeActor { + async fn run( + &mut self, + rx: actor_helper::Receiver>, + ) -> Result<()> { tracing::debug!("BubbleMerge: starting bubble merge actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { + Ok(action) = rx.recv_async() => { action(self).await; } _ = self.ticker.tick() => { diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index ea8c2bd..ba555c7 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -1,6 +1,6 @@ //! Split-brain detection via message hash overlap in DHT records. -use actor_helper::{Action, Actor, Handle, Receiver}; +use actor_helper::{Action, Handle, Receiver}; use iroh::EndpointId; use std::{collections::HashSet, time::Duration}; @@ -17,8 +17,6 @@ pub struct MessageOverlapMerge { #[derive(Debug)] struct MessageOverlapMergeActor { - rx: Receiver>, - record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, gossip_sender: GossipSender, @@ -32,32 +30,29 @@ impl MessageOverlapMerge { gossip_sender: GossipSender, gossip_receiver: GossipReceiver, ) -> Result { - let (api, rx) = Handle::channel(); - let mut ticker = tokio::time::interval(Duration::from_secs(10)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - tokio::spawn(async move { - let mut actor = MessageOverlapMergeActor { - rx, + let api = Handle::spawn_with( + MessageOverlapMergeActor { record_publisher, gossip_receiver, gossip_sender, ticker, - }; - let _ = actor.run().await; - }); - + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { _api: api }) } } -impl Actor for MessageOverlapMergeActor { - async fn run(&mut self) -> Result<()> { +impl MessageOverlapMergeActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("MessageOverlapMerge: starting message overlap merge actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { + Ok(action) = rx.recv_async() => { action(self).await; } _ = self.ticker.tick() => { diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 1f11adb..99142fe 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -2,7 +2,7 @@ use std::collections::{HashSet, VecDeque}; -use actor_helper::{Action, Actor, Handle, Receiver, act, act_ok}; +use actor_helper::{Action, Handle, Receiver, act, act_ok}; use anyhow::Result; use futures_lite::StreamExt; use iroh::EndpointId; @@ -20,7 +20,6 @@ pub struct GossipReceiver { #[derive(Debug)] pub struct GossipReceiverActor { - rx: Receiver>, gossip_receiver: iroh_gossip::api::GossipReceiver, last_message_hashes: Vec<[u8; 32]>, msg_queue: VecDeque>>, @@ -38,21 +37,17 @@ impl GossipReceiver { gossip_receiver: iroh_gossip::api::GossipReceiver, gossip: iroh_gossip::net::Gossip, ) -> Self { - let (api, rx) = Handle::channel(); - tokio::spawn({ - let gossip = gossip.clone(); - async move { - let mut actor = GossipReceiverActor { - rx, - gossip_receiver, - last_message_hashes: Vec::new(), - msg_queue: VecDeque::new(), - waiters: VecDeque::new(), - _gossip: gossip.clone(), - }; - let _ = actor.run().await; - } - }); + let api = Handle::spawn_with( + GossipReceiverActor { + gossip_receiver, + last_message_hashes: Vec::new(), + msg_queue: VecDeque::new(), + waiters: VecDeque::new(), + _gossip: gossip.clone(), + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Self { api, @@ -103,12 +98,12 @@ impl GossipReceiver { } } -impl Actor for GossipReceiverActor { - async fn run(&mut self) -> Result<()> { +impl GossipReceiverActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("GossipReceiver: starting gossip receiver actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { + Ok(action) = rx.recv_async() => { action(self).await; } raw_event = self.gossip_receiver.next() => { diff --git a/src/gossip/sender.rs b/src/gossip/sender.rs index f4db8ab..a0a8495 100644 --- a/src/gossip/sender.rs +++ b/src/gossip/sender.rs @@ -1,6 +1,6 @@ //! Actor-based wrapper for iroh-gossip broadcast operations. -use actor_helper::{Action, Actor, Handle, Receiver, act}; +use actor_helper::{Handle, act}; use anyhow::Result; use iroh::EndpointId; use rand::seq::SliceRandom; @@ -17,7 +17,6 @@ pub struct GossipSender { #[derive(Debug)] pub struct GossipSenderActor { - rx: Receiver>, gossip_sender: iroh_gossip::api::GossipSender, _gossip: iroh_gossip::net::Gossip, } @@ -28,18 +27,11 @@ impl GossipSender { gossip_sender: iroh_gossip::api::GossipSender, gossip: iroh_gossip::net::Gossip, ) -> Self { - let (api, rx) = Handle::channel(); - tokio::spawn({ - let gossip = gossip.clone(); - async move { - let mut actor = GossipSenderActor { - rx, - gossip_sender, - _gossip: gossip.clone(), - }; - let _ = actor.run().await; - } - }); + let api = Handle::spawn(GossipSenderActor { + gossip_sender, + _gossip: gossip.clone(), + }) + .0; Self { api, @@ -96,16 +88,3 @@ impl GossipSender { .await } } - -impl Actor for GossipSenderActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - action(self).await; - } - else => break Ok(()), - } - } - } -} diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 3ff6dcf..8952a11 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, time::Duration}; -use actor_helper::{Action, Actor, Handle, Receiver, act, act_ok}; +use actor_helper::{Action, Handle, Receiver, act, act_ok}; use anyhow::Result; use iroh::EndpointId; use tokio::time::sleep; @@ -24,8 +24,6 @@ pub struct Bootstrap { #[derive(Debug)] struct BootstrapActor { - rx: Receiver>, - record_publisher: crate::crypto::RecordPublisher, gossip_sender: GossipSender, @@ -50,17 +48,15 @@ impl Bootstrap { GossipReceiver::new(gossip_receiver, gossip.clone()), ); - let (api, rx) = Handle::channel(); - - tokio::spawn(async move { - let mut actor = BootstrapActor { - rx, + let api = Handle::spawn_with( + BootstrapActor { record_publisher, gossip_sender, gossip_receiver, - }; - let _ = actor.run().await; - }); + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { api }) } @@ -87,11 +83,11 @@ impl Bootstrap { } } -impl Actor for BootstrapActor { - async fn run(&mut self) -> Result<()> { +impl BootstrapActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { + Ok(action) = rx.recv_async() => { action(self).await; } else => break Ok(()), diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 18256e3..227cedc 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -1,6 +1,6 @@ //! Background publisher that updates DHT records with active peer info. -use actor_helper::{Action, Actor, Handle, Receiver}; +use actor_helper::{Action, Handle, Receiver}; use std::time::Duration; use crate::{GossipReceiver, RecordPublisher}; @@ -18,8 +18,6 @@ pub struct Publisher { #[derive(Debug)] struct PublisherActor { - rx: Receiver>, - record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, ticker: tokio::time::Interval, @@ -30,30 +28,28 @@ impl Publisher { /// /// Spawns a background task that periodically publishes records. pub fn new(record_publisher: RecordPublisher, gossip_receiver: GossipReceiver) -> Result { - let (api, rx) = Handle::channel(); - - tokio::spawn(async move { - let mut ticker = tokio::time::interval(Duration::from_secs(10)); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - let mut actor = PublisherActor { - rx, + let mut ticker = tokio::time::interval(Duration::from_secs(10)); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let api = Handle::spawn_with( + PublisherActor { record_publisher, gossip_receiver, ticker, - }; - let _ = actor.run().await; - }); + }, + |mut actor, rx| async move { actor.run(rx).await }, + ) + .0; Ok(Self { _api: api }) } } -impl Actor for PublisherActor { - async fn run(&mut self) -> Result<()> { +impl PublisherActor { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("Publisher: starting publisher actor"); loop { tokio::select! { - Ok(action) = self.rx.recv_async() => { + Ok(action) = rx.recv_async() => { action(self).await; } _ = self.ticker.tick() => { diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 7ea0f88..952964a 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -8,7 +8,7 @@ use crate::{ topic::{bootstrap::Bootstrap, publisher::Publisher}, }, }; -use actor_helper::{Action, Actor, Handle, Receiver, act, act_ok}; +use actor_helper::{Handle, act, act_ok}; use anyhow::Result; use sha2::Digest; @@ -72,7 +72,6 @@ pub struct Topic { #[derive(Debug)] struct TopicActor { - rx: Receiver>, bootstrap: Bootstrap, publisher: Option, bubble_merge: Option, @@ -98,26 +97,17 @@ impl Topic { async_bootstrap ); - let (api, rx) = Handle::channel(); - let bootstrap = Bootstrap::new(record_publisher.clone(), gossip.clone()).await?; tracing::debug!("Topic: bootstrap instance created"); - tokio::spawn({ - let bootstrap = bootstrap.clone(); - async move { - tracing::debug!("Topic: starting topic actor"); - let mut actor = TopicActor { - rx, - bootstrap: bootstrap.clone(), - record_publisher, - publisher: None, - bubble_merge: None, - message_overlap_merge: None, - }; - let _ = actor.run().await; - } - }); + let api = Handle::spawn(TopicActor { + bootstrap: bootstrap.clone(), + record_publisher, + publisher: None, + bubble_merge: None, + message_overlap_merge: None, + }) + .0; let bootstrap_done = bootstrap.bootstrap().await?; if !async_bootstrap { @@ -170,19 +160,6 @@ impl Topic { } } -impl Actor for TopicActor { - async fn run(&mut self) -> Result<()> { - loop { - tokio::select! { - Ok(action) = self.rx.recv_async() => { - let _ = action(self).await; - } - else => break Ok(()), - } - } - } -} - impl TopicActor { pub async fn start_publishing(&mut self) -> Result<()> { tracing::debug!("TopicActor: initializing publisher"); From a6a6a796570c542386c5467beb2f2bcfa889faa3 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 12 Apr 2026 16:05:19 +0200 Subject: [PATCH 02/30] fix: cleanup --- src/gossip/merge/bubble.rs | 4 ++-- src/gossip/topic/bootstrap.rs | 28 ++++++---------------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 2157e98..0f3f338 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -2,7 +2,7 @@ //! //! If local peer count < 4, extract suggested peers from DHT records and join them. -use actor_helper::Handle; +use actor_helper::{Action, Handle, Receiver}; use iroh::EndpointId; use std::{collections::HashSet, time::Duration}; @@ -56,7 +56,7 @@ impl BubbleMerge { impl BubbleMergeActor { async fn run( &mut self, - rx: actor_helper::Receiver>, + rx: Receiver>, ) -> Result<()> { tracing::debug!("BubbleMerge: starting bubble merge actor"); loop { diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 8952a11..fe17f75 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, time::Duration}; -use actor_helper::{Action, Handle, Receiver, act, act_ok}; +use actor_helper::{Handle, act, act_ok}; use anyhow::Result; use iroh::EndpointId; use tokio::time::sleep; @@ -48,14 +48,11 @@ impl Bootstrap { GossipReceiver::new(gossip_receiver, gossip.clone()), ); - let api = Handle::spawn_with( - BootstrapActor { - record_publisher, - gossip_sender, - gossip_receiver, - }, - |mut actor, rx| async move { actor.run(rx).await }, - ) + let api = Handle::spawn(BootstrapActor { + record_publisher, + gossip_sender, + gossip_receiver, + }) .0; Ok(Self { api }) @@ -83,19 +80,6 @@ impl Bootstrap { } } -impl BootstrapActor { - async fn run(&mut self, rx: Receiver>) -> Result<()> { - loop { - tokio::select! { - Ok(action) = rx.recv_async() => { - action(self).await; - } - else => break Ok(()), - } - } - } -} - impl BootstrapActor { pub async fn start_bootstrap(&mut self) -> Result> { let (sender, receiver) = tokio::sync::oneshot::channel(); From b58fdfef80e6ed7d771c0f4a98f594320151defa Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 12 Apr 2026 18:58:35 +0200 Subject: [PATCH 03/30] fix: cicd actions/checkout update to v6 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a95d9c5..3f514bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Build Docker image run: docker build -t distributed-topic-tracker . From 391abccf64be376b14bfe1396c349897c9f77d9e Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Mon, 13 Apr 2026 23:15:03 +0200 Subject: [PATCH 04/30] fix: on drop of topic, sender and receiver pointer stops all background tasks when no references remain --- Cargo.lock | 13 ++--- Cargo.toml | 1 + src/gossip/merge/bubble.rs | 6 +++ src/gossip/merge/message_overlap.rs | 6 +++ src/gossip/receiver.rs | 45 +++++++++++++---- src/gossip/sender.rs | 5 -- src/gossip/topic/bootstrap.rs | 32 +++++++++--- src/gossip/topic/publisher.rs | 7 ++- src/gossip/topic/topic.rs | 78 ++++++++++++++++++++++++++++- 9 files changed, 162 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c12b16f..8e69d3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -701,6 +701,7 @@ dependencies = [ "serde", "sha2 0.10.9", "tokio", + "tokio-util", "tracing", "tracing-subscriber", ] @@ -1886,9 +1887,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.184" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libm" @@ -2735,9 +2736,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec095654a25171c2124e9e3393a930bddbffdc939556c914957a4c3e0a87166" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", "rand_core 0.9.5", @@ -2875,9 +2876,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "log", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 08572ae..90b8c35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ ed25519-dalek = { version = "3.0.0-pre.1", default-features = false, features = ed25519-dalek-hpke = { version = "0.0.5" } tokio = { version = "1", default-features = false, features = ["macros", "time", "sync", "rt-multi-thread"] } +tokio-util = { version = "0.7", default-features = false } iroh = { version = "0.97", optional = true, default-features = false } iroh-gossip = { version = "0.97", optional = true, default-features = false, features = ["net"]} diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 0f3f338..bec3b99 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -24,6 +24,7 @@ struct BubbleMergeActor { gossip_receiver: GossipReceiver, gossip_sender: GossipSender, ticker: tokio::time::Interval, + cancel_token: tokio_util::sync::CancellationToken, } impl BubbleMerge { @@ -34,6 +35,7 @@ impl BubbleMerge { record_publisher: RecordPublisher, gossip_sender: GossipSender, gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, ) -> Result { let mut ticker = tokio::time::interval(Duration::from_secs(10)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -44,6 +46,7 @@ impl BubbleMerge { gossip_receiver, gossip_sender, ticker, + cancel_token, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -71,6 +74,9 @@ impl BubbleMergeActor { tracing::debug!("BubbleMerge: next check in {}s", next_interval); self.ticker.reset_after(Duration::from_secs(next_interval)); } + _ = self.cancel_token.cancelled() => { + break Ok(()); + } else => break Ok(()), } } diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index ba555c7..e617fb4 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -21,6 +21,7 @@ struct MessageOverlapMergeActor { gossip_receiver: GossipReceiver, gossip_sender: GossipSender, ticker: tokio::time::Interval, + cancel_token: tokio_util::sync::CancellationToken, } impl MessageOverlapMerge { @@ -29,6 +30,7 @@ impl MessageOverlapMerge { record_publisher: RecordPublisher, gossip_sender: GossipSender, gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, ) -> Result { let mut ticker = tokio::time::interval(Duration::from_secs(10)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -39,6 +41,7 @@ impl MessageOverlapMerge { gossip_receiver, gossip_sender, ticker, + cancel_token }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -62,6 +65,9 @@ impl MessageOverlapMergeActor { tracing::debug!("MessageOverlapMerge: next check in {}s", next_interval); self.ticker.reset_after(Duration::from_secs(next_interval)); } + _ = self.cancel_token.cancelled() => { + break Ok(()); + } else => break Ok(()), } } diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 99142fe..5cfff15 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -15,35 +15,34 @@ use sha2::Digest; #[derive(Debug, Clone)] pub struct GossipReceiver { api: Handle, - _gossip: iroh_gossip::net::Gossip, } #[derive(Debug)] pub struct GossipReceiverActor { gossip_receiver: iroh_gossip::api::GossipReceiver, - last_message_hashes: Vec<[u8; 32]>, + last_message_hashes: VecDeque<[u8; 32]>, msg_queue: VecDeque>>, waiters: VecDeque< tokio::sync::oneshot::Sender< Option>, >, >, - _gossip: iroh_gossip::net::Gossip, + cancel_token: tokio_util::sync::CancellationToken, } impl GossipReceiver { /// Create a new gossip receiver from an iroh topic receiver. pub fn new( gossip_receiver: iroh_gossip::api::GossipReceiver, - gossip: iroh_gossip::net::Gossip, + cancel_token: tokio_util::sync::CancellationToken, ) -> Self { let api = Handle::spawn_with( GossipReceiverActor { gossip_receiver, - last_message_hashes: Vec::new(), + last_message_hashes: VecDeque::with_capacity(5), msg_queue: VecDeque::new(), waiters: VecDeque::new(), - _gossip: gossip.clone(), + cancel_token, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -51,7 +50,6 @@ impl GossipReceiver { Self { api, - _gossip: gossip.clone(), } } @@ -90,7 +88,7 @@ impl GossipReceiver { /// Get SHA512 hashes (first 32 bytes) of recently received messages. /// /// Used for detecting message overlap during network partition recovery. - pub async fn last_message_hashes(&self) -> Vec<[u8; 32]> { + pub async fn last_message_hashes(&self) -> VecDeque<[u8; 32]> { self.api .call(act_ok!(actor => async move { actor.last_message_hashes.clone() })) .await @@ -107,6 +105,26 @@ impl GossipReceiverActor { action(self).await; } raw_event = self.gossip_receiver.next() => { + match raw_event { + None => { + // if none => receiver closed + while let Some(waiter) = self.waiters.pop_back() { + let _ = waiter.send(None); + } + break Ok(()); + } + Some(Err(_)) => { + // if Err => stream error, treat as closed + if let Some(waiter) = self.waiters.pop_back() { + let _ = waiter.send(raw_event); + } + while let Some(waiter) = self.waiters.pop_back() { + let _ = waiter.send(None); + } + break Ok(()); + } + Some(Ok(_)) => {} + }; self.msg_queue.push_front(raw_event); if let Some(waiter) = self.waiters.pop_back() { @@ -124,7 +142,10 @@ impl GossipReceiverActor { let mut hash = sha2::Sha512::new(); hash.update(msg.content.clone()); if let Ok(lmh) = hash.finalize()[..32].try_into() { - self.last_message_hashes.push(lmh); + if self.last_message_hashes.len() == 5 { + self.last_message_hashes.pop_front(); + } + self.last_message_hashes.push_back(lmh); } } iroh_gossip::api::Event::NeighborUp(node_id) => { @@ -139,6 +160,12 @@ impl GossipReceiverActor { } } } + _ = self.cancel_token.cancelled() => { + while let Some(waiter) = self.waiters.pop_back() { + let _ = waiter.send(None); + } + break Ok(()); + } else => break Ok(()), } } diff --git a/src/gossip/sender.rs b/src/gossip/sender.rs index a0a8495..6e28db6 100644 --- a/src/gossip/sender.rs +++ b/src/gossip/sender.rs @@ -12,30 +12,25 @@ use rand::seq::SliceRandom; #[derive(Debug, Clone)] pub struct GossipSender { api: Handle, - _gossip: iroh_gossip::net::Gossip, } #[derive(Debug)] pub struct GossipSenderActor { gossip_sender: iroh_gossip::api::GossipSender, - _gossip: iroh_gossip::net::Gossip, } impl GossipSender { /// Create a new gossip sender from an iroh topic sender. pub fn new( gossip_sender: iroh_gossip::api::GossipSender, - gossip: iroh_gossip::net::Gossip, ) -> Self { let api = Handle::spawn(GossipSenderActor { gossip_sender, - _gossip: gossip.clone(), }) .0; Self { api, - _gossip: gossip, } } diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index fe17f75..2c34440 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -25,9 +25,9 @@ pub struct Bootstrap { #[derive(Debug)] struct BootstrapActor { record_publisher: crate::crypto::RecordPublisher, - gossip_sender: GossipSender, gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, } impl Bootstrap { @@ -35,6 +35,7 @@ impl Bootstrap { pub async fn new( record_publisher: crate::crypto::RecordPublisher, gossip: iroh_gossip::net::Gossip, + cancel_token: tokio_util::sync::CancellationToken, ) -> Result { let gossip_topic: iroh_gossip::api::GossipTopic = gossip .subscribe( @@ -44,14 +45,15 @@ impl Bootstrap { .await?; let (gossip_sender, gossip_receiver) = gossip_topic.split(); let (gossip_sender, gossip_receiver) = ( - GossipSender::new(gossip_sender, gossip.clone()), - GossipReceiver::new(gossip_receiver, gossip.clone()), + GossipSender::new(gossip_sender), + GossipReceiver::new(gossip_receiver, cancel_token.clone()), ); let api = Handle::spawn(BootstrapActor { record_publisher, gossip_sender, gossip_receiver, + cancel_token, }) .0; @@ -88,9 +90,10 @@ impl BootstrapActor { let (gossip_sender, gossip_receiver) = (self.gossip_sender.clone(), self.gossip_receiver.clone()); let record_publisher = self.record_publisher.clone(); + let cancel_token = self.cancel_token.clone(); async move { tracing::debug!("Bootstrap: starting bootstrap process"); - loop { + while !cancel_token.is_cancelled() { // Check if we are connected to at least one node if gossip_receiver.is_joined().await { tracing::debug!("Bootstrap: already joined, exiting bootstrap loop"); @@ -140,7 +143,10 @@ impl BootstrapActor { }); } } - sleep(Duration::from_millis(100)).await; + tokio::select! { + _ = sleep(Duration::from_millis(100)) => {} + _ = cancel_token.cancelled() => break, + } continue; } @@ -182,7 +188,11 @@ impl BootstrapActor { match gossip_sender.join_peers(vec![*node_id], None).await { Ok(_) => { tracing::debug!("Bootstrap: attempted to join peer {}", node_id); - sleep(Duration::from_millis(100)).await; + + tokio::select! { + _ = sleep(Duration::from_millis(100)) => {} + _ = cancel_token.cancelled() => break, + } if gossip_receiver.is_joined().await { tracing::debug!( "Bootstrap: successfully joined via peer {}", @@ -208,7 +218,10 @@ impl BootstrapActor { tracing::debug!( "Bootstrap: not joined yet, waiting 500ms before final check" ); - sleep(Duration::from_millis(500)).await; + tokio::select! { + _ = sleep(Duration::from_millis(500)) => {} + _ = cancel_token.cancelled() => break, + } } // If we are connected: return @@ -240,7 +253,10 @@ impl BootstrapActor { }); } } - sleep(Duration::from_millis(100)).await; + tokio::select! { + _ = sleep(Duration::from_millis(100)) => {} + _ = cancel_token.cancelled() => break, + } continue; } } diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 227cedc..29b2da3 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -21,13 +21,14 @@ struct PublisherActor { record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, ticker: tokio::time::Interval, + cancel_token: tokio_util::sync::CancellationToken, } impl Publisher { /// Create a new background publisher. /// /// Spawns a background task that periodically publishes records. - pub fn new(record_publisher: RecordPublisher, gossip_receiver: GossipReceiver) -> Result { + pub fn new(record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, cancel_token: tokio_util::sync::CancellationToken) -> Result { let mut ticker = tokio::time::interval(Duration::from_secs(10)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -35,6 +36,7 @@ impl Publisher { record_publisher, gossip_receiver, ticker, + cancel_token, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -59,6 +61,9 @@ impl PublisherActor { tracing::debug!("Publisher: next publish in {}s", next_interval); self.ticker.reset_after(Duration::from_secs(next_interval)); } + _ = self.cancel_token.cancelled() => { + break Ok(()); + } else => break Ok(()), } } diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 952964a..b89e29f 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -11,6 +11,7 @@ use crate::{ use actor_helper::{Handle, act, act_ok}; use anyhow::Result; use sha2::Digest; +use tokio_util::sync::CancellationToken; /// Topic identifier derived from a string via SHA512 hashing. /// @@ -68,6 +69,7 @@ impl TopicId { #[derive(Debug, Clone)] pub struct Topic { api: Handle, + cancel_token: CancellationToken, } #[derive(Debug)] @@ -77,6 +79,13 @@ struct TopicActor { bubble_merge: Option, message_overlap_merge: Option, record_publisher: crate::crypto::RecordPublisher, + cancel_token: CancellationToken, +} + +impl Drop for TopicActor { + fn drop(&mut self) { + self.cancel_token.cancel(); + } } impl Topic { @@ -97,7 +106,8 @@ impl Topic { async_bootstrap ); - let bootstrap = Bootstrap::new(record_publisher.clone(), gossip.clone()).await?; + let cancel_token = CancellationToken::new(); + let bootstrap = Bootstrap::new(record_publisher.clone(), gossip.clone(), cancel_token.clone()).await?; tracing::debug!("Topic: bootstrap instance created"); let api = Handle::spawn(TopicActor { @@ -106,6 +116,7 @@ impl Topic { publisher: None, bubble_merge: None, message_overlap_merge: None, + cancel_token: cancel_token.clone(), }) .0; @@ -130,7 +141,7 @@ impl Topic { .await; tracing::debug!("Topic: fully initialized"); - Ok(Self { api }) + Ok(Self { api, cancel_token }) } /// Split into sender and receiver for message exchange. @@ -158,6 +169,11 @@ impl Topic { .call(act_ok!(actor => async move { actor.record_publisher.clone() })) .await } + + #[allow(dead_code)] + pub(crate) fn cancel_token(&self) -> CancellationToken { + self.cancel_token.clone() + } } impl TopicActor { @@ -166,6 +182,7 @@ impl TopicActor { let publisher = Publisher::new( self.record_publisher.clone(), self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), )?; self.publisher = Some(publisher); tracing::debug!("TopicActor: publisher started"); @@ -178,6 +195,7 @@ impl TopicActor { self.record_publisher.clone(), self.bootstrap.gossip_sender().await?, self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), )?; self.bubble_merge = Some(bubble_merge); tracing::debug!("TopicActor: bubble merge started"); @@ -190,9 +208,65 @@ impl TopicActor { self.record_publisher.clone(), self.bootstrap.gossip_sender().await?, self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), )?; self.message_overlap_merge = Some(message_overlap_merge); tracing::debug!("TopicActor: message overlap merge started"); Ok(()) } } + +mod tests { + #[tokio::test] + async fn test_topic_full_shutdown_on_drop() { + let secret_key = iroh::SecretKey::generate(&mut rand::rng()); + let signing_key = mainline::SigningKey::from_bytes(&secret_key.to_bytes()); + let endpoint = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await.unwrap(); + let gossip = iroh_gossip::net::Gossip::builder() + .spawn(endpoint.clone()); + + let topic_id = crate::TopicId::new("my-iroh-gossip-topic".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + let record_publisher = crate::RecordPublisher::new( + topic_id.clone(), + signing_key.verifying_key(), + signing_key.clone(), + None, + initial_secret, + ); + + let topic = crate::Topic::new(record_publisher, gossip.clone(), true).await.unwrap(); + + let cancel_token = topic.cancel_token(); + let receiver_probe = topic.gossip_receiver().await.unwrap(); + + let (sender, receiver) = topic.split().await.unwrap(); + + assert!(!cancel_token.is_cancelled()); + + drop(sender); + drop(receiver); + drop(topic); + + tokio::time::timeout( + std::time::Duration::from_secs(5), + cancel_token.cancelled(), + ) + .await + .expect("TopicActor did not shut down, cancel token was never cancelled"); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(5), + receiver_probe.next(), + ) + .await + .expect("GossipReceiverActor did not shut down, next() hung"); + assert!(result.is_none(), "expected None from dead receiver, got an event"); + + drop(receiver_probe); + } +} \ No newline at end of file From a0c73fa42f4dc33c7db960a1c21615b307a361ac Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Tue, 14 Apr 2026 00:44:01 +0200 Subject: [PATCH 05/30] chore: explicit handeling of action error in actor run loops + fixed publisher record_content failing try_into if not exactly 5 + topic test only compiled for tests + fixed receiver pop before hashing --- src/gossip/merge/bubble.rs | 12 +++--- src/gossip/merge/message_overlap.rs | 9 +++-- src/gossip/receiver.rs | 60 ++++++++++++++--------------- src/gossip/topic/publisher.rs | 27 ++++++++++--- src/gossip/topic/topic.rs | 1 + 5 files changed, 65 insertions(+), 44 deletions(-) diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index bec3b99..89e5a1b 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -57,15 +57,15 @@ impl BubbleMerge { } impl BubbleMergeActor { - async fn run( - &mut self, - rx: Receiver>, - ) -> Result<()> { + async fn run(&mut self, rx: Receiver>) -> Result<()> { tracing::debug!("BubbleMerge: starting bubble merge actor"); loop { tokio::select! { - Ok(action) = rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } _ = self.ticker.tick() => { tracing::debug!("BubbleMerge: tick fired, checking for bubbles"); diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index e617fb4..23404bb 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -41,7 +41,7 @@ impl MessageOverlapMerge { gossip_receiver, gossip_sender, ticker, - cancel_token + cancel_token, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -55,8 +55,11 @@ impl MessageOverlapMergeActor { tracing::debug!("MessageOverlapMerge: starting message overlap merge actor"); loop { tokio::select! { - Ok(action) = rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } _ = self.ticker.tick() => { tracing::debug!("MessageOverlapMerge: tick fired, checking for split-brain"); diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 5cfff15..c5f13dd 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -48,9 +48,7 @@ impl GossipReceiver { ) .0; - Self { - api, - } + Self { api } } /// Get the set of currently connected neighbor node IDs. @@ -101,8 +99,11 @@ impl GossipReceiverActor { tracing::debug!("GossipReceiver: starting gossip receiver actor"); loop { tokio::select! { - Ok(action) = rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } raw_event = self.gossip_receiver.next() => { match raw_event { @@ -123,7 +124,30 @@ impl GossipReceiverActor { } break Ok(()); } - Some(Ok(_)) => {} + Some(Ok(ref event)) => { + match event { + iroh_gossip::api::Event::Received(msg) => { + tracing::debug!("GossipReceiver: received message from {:?}", msg.delivered_from); + let mut hash = sha2::Sha512::new(); + hash.update(msg.content.clone()); + if let Ok(lmh) = hash.finalize()[..32].try_into() { + if self.last_message_hashes.len() == 5 { + self.last_message_hashes.pop_front(); + } + self.last_message_hashes.push_back(lmh); + } + } + iroh_gossip::api::Event::NeighborUp(node_id) => { + tracing::debug!("GossipReceiver: neighbor UP: {}", node_id); + } + iroh_gossip::api::Event::NeighborDown(node_id) => { + tracing::debug!("GossipReceiver: neighbor DOWN: {}", node_id); + } + iroh_gossip::api::Event::Lagged => { + tracing::debug!("GossipReceiver: event stream lagged"); + } + } + } }; self.msg_queue.push_front(raw_event); @@ -135,30 +159,6 @@ impl GossipReceiverActor { // this should never happen } } - if let Some(Some(Ok(event))) = self.msg_queue.front() { - match event { - iroh_gossip::api::Event::Received(msg) => { - tracing::debug!("GossipReceiver: received message from {:?}", msg.delivered_from); - let mut hash = sha2::Sha512::new(); - hash.update(msg.content.clone()); - if let Ok(lmh) = hash.finalize()[..32].try_into() { - if self.last_message_hashes.len() == 5 { - self.last_message_hashes.pop_front(); - } - self.last_message_hashes.push_back(lmh); - } - } - iroh_gossip::api::Event::NeighborUp(node_id) => { - tracing::debug!("GossipReceiver: neighbor UP: {}", node_id); - } - iroh_gossip::api::Event::NeighborDown(node_id) => { - tracing::debug!("GossipReceiver: neighbor DOWN: {}", node_id); - } - iroh_gossip::api::Event::Lagged => { - tracing::debug!("GossipReceiver: event stream lagged"); - } - } - } } _ = self.cancel_token.cancelled() => { while let Some(waiter) = self.waiters.pop_back() { diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 29b2da3..8c7c6ec 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -28,7 +28,11 @@ impl Publisher { /// Create a new background publisher. /// /// Spawns a background task that periodically publishes records. - pub fn new(record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, cancel_token: tokio_util::sync::CancellationToken) -> Result { + pub fn new( + record_publisher: RecordPublisher, + gossip_receiver: GossipReceiver, + cancel_token: tokio_util::sync::CancellationToken, + ) -> Result { let mut ticker = tokio::time::interval(Duration::from_secs(10)); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -51,8 +55,11 @@ impl PublisherActor { tracing::debug!("Publisher: starting publisher actor"); loop { tokio::select! { - Ok(action) = rx.recv_async() => { - action(self).await; + result = rx.recv_async() => { + match result { + Ok(action) => action(self).await, + Err(_) => break Ok(()), + } } _ = self.ticker.tick() => { tracing::debug!("Publisher: tick fired, attempting to publish"); @@ -80,6 +87,7 @@ impl PublisherActor { .await .iter() .filter_map(|pub_key| TryInto::<[u8; 32]>::try_into(pub_key.as_slice()).ok()) + .take(5) .collect::>(); let last_message_hashes = self @@ -88,6 +96,7 @@ impl PublisherActor { .await .iter() .filter_map(|hash| TryInto::<[u8; 32]>::try_into(hash.as_slice()).ok()) + .take(5) .collect::>(); tracing::debug!( @@ -98,8 +107,16 @@ impl PublisherActor { ); let record_content = crate::gossip::GossipRecordContent { - active_peers: active_peers.as_slice().try_into()?, - last_message_hashes: last_message_hashes.as_slice().try_into()?, + active_peers: { + let mut peers = [Default::default(); 5]; + peers[..active_peers.len()].copy_from_slice(&active_peers); + peers + }, + last_message_hashes: { + let mut hashes = [Default::default(); 5]; + hashes[..last_message_hashes.len()].copy_from_slice(&last_message_hashes); + hashes + }, }; tracing::debug!("Publisher: created record content: {:?}", record_content); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index b89e29f..e1171a7 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -216,6 +216,7 @@ impl TopicActor { } } +#[cfg(test)] mod tests { #[tokio::test] async fn test_topic_full_shutdown_on_drop() { From 93b909e11216df535ae19a7cb8150b6e3a66a8fe Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Tue, 14 Apr 2026 02:52:02 +0200 Subject: [PATCH 06/30] chore: added timeouts to sender calls that can halt due to full iroh-gossip mailbox (only 64) + fixed circular reference in topic split code path (if topic => new => split is one call without keeping topic around specifically) + cleanup + cargo test phase added to cicd --- .github/workflows/test.yml | 8 +++++++- src/gossip/receiver.rs | 19 +++++++++++++----- src/gossip/sender.rs | 38 +++++++++++++++++++++++------------ src/gossip/topic/bootstrap.rs | 4 +--- src/gossip/topic/publisher.rs | 4 ++-- src/gossip/topic/topic.rs | 29 ++++++++++---------------- 6 files changed, 60 insertions(+), 42 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f514bd..6024974 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,10 +13,16 @@ env: CARGO_TERM_COLOR: always jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Run tests + run: cargo test --verbose + e2e-tests: name: End-to-End Tests runs-on: ubuntu-latest - steps: - uses: actions/checkout@v6 diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index c5f13dd..20eeca6 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -1,6 +1,9 @@ //! Actor-based wrapper for iroh-gossip message receiving. -use std::collections::{HashSet, VecDeque}; +use std::{ + collections::{HashSet, VecDeque}, + sync::Arc, +}; use actor_helper::{Action, Handle, Receiver, act, act_ok}; use anyhow::Result; @@ -8,6 +11,8 @@ use futures_lite::StreamExt; use iroh::EndpointId; use sha2::Digest; +use crate::Topic; + /// Gossip receiver that collects incoming messages and neighbor info. /// /// Tracks SHA512 message hashes (first 32 bytes) for overlap detection and provides @@ -15,6 +20,7 @@ use sha2::Digest; #[derive(Debug, Clone)] pub struct GossipReceiver { api: Handle, + pub(crate) _topic_keep_alive: Option>, } #[derive(Debug)] @@ -48,7 +54,10 @@ impl GossipReceiver { ) .0; - Self { api } + Self { + api, + _topic_keep_alive: None, + } } /// Get the set of currently connected neighbor node IDs. @@ -58,7 +67,7 @@ impl GossipReceiver { actor.gossip_receiver.neighbors().collect::>() })) .await - .expect("actor stopped") + .unwrap_or_default() } /// Check if the local node has joined the topic. @@ -66,7 +75,7 @@ impl GossipReceiver { self.api .call(act_ok!(actor => async move { actor.gossip_receiver.is_joined() })) .await - .expect("actor stopped") + .unwrap_or(false) } /// Receive the next gossip event. @@ -90,7 +99,7 @@ impl GossipReceiver { self.api .call(act_ok!(actor => async move { actor.last_message_hashes.clone() })) .await - .expect("void") + .unwrap_or_default() } } diff --git a/src/gossip/sender.rs b/src/gossip/sender.rs index 6e28db6..3d9ba39 100644 --- a/src/gossip/sender.rs +++ b/src/gossip/sender.rs @@ -1,10 +1,14 @@ //! Actor-based wrapper for iroh-gossip broadcast operations. +use std::{sync::Arc, time::Duration}; + use actor_helper::{Handle, act}; use anyhow::Result; use iroh::EndpointId; use rand::seq::SliceRandom; +use crate::Topic; + /// Gossip sender that broadcasts messages to peers. /// /// Provides methods for broadcasting to all peers or just direct neighbors, @@ -12,6 +16,7 @@ use rand::seq::SliceRandom; #[derive(Debug, Clone)] pub struct GossipSender { api: Handle, + pub(crate) _topic_keep_alive: Option>, } #[derive(Debug)] @@ -21,16 +26,12 @@ pub struct GossipSenderActor { impl GossipSender { /// Create a new gossip sender from an iroh topic sender. - pub fn new( - gossip_sender: iroh_gossip::api::GossipSender, - ) -> Self { - let api = Handle::spawn(GossipSenderActor { - gossip_sender, - }) - .0; + pub fn new(gossip_sender: iroh_gossip::api::GossipSender) -> Self { + let api = Handle::spawn(GossipSenderActor { gossip_sender }).0; Self { api, + _topic_keep_alive: None, } } @@ -39,8 +40,12 @@ impl GossipSender { tracing::debug!("GossipSender: broadcasting message ({} bytes)", data.len()); self.api .call(act!(actor => async move { - actor.gossip_sender - .broadcast(data.into()).await.map_err(|e| anyhow::anyhow!(e)) + tokio::time::timeout( + Duration::from_secs(5), + actor.gossip_sender.broadcast(data.into()) + ).await + .map_err(|_| anyhow::anyhow!("broadcast timed out"))? + .map_err(|e| anyhow::anyhow!(e)) })) .await } @@ -53,7 +58,12 @@ impl GossipSender { ); self.api .call(act!(actor => async move { - actor.gossip_sender.broadcast_neighbors(data.into()).await.map_err(|e| anyhow::anyhow!(e)) + tokio::time::timeout( + Duration::from_secs(5), + actor.gossip_sender.broadcast_neighbors(data.into()) + ).await + .map_err(|_| anyhow::anyhow!("broadcast_neighbors timed out"))? + .map_err(|e| anyhow::anyhow!(e)) })) .await } @@ -75,9 +85,11 @@ impl GossipSender { self.api .call(act!(actor => async move { - actor.gossip_sender - .join_peers(peers) - .await + tokio::time::timeout( + Duration::from_secs(5), + actor.gossip_sender.join_peers(peers) + ).await + .map_err(|_| anyhow::anyhow!("join_peers timed out"))? .map_err(|e| anyhow::anyhow!(e)) })) .await diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 2c34440..2c749af 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -8,9 +8,7 @@ use iroh::EndpointId; use tokio::time::sleep; use crate::{ - GossipSender, - crypto::Record, - gossip::{GossipRecordContent, receiver::GossipReceiver}, + GossipSender, crypto::Record, gossip::{GossipRecordContent, receiver::GossipReceiver} }; /// Manages the peer discovery and joining process. diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 8c7c6ec..17a4def 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -86,7 +86,7 @@ impl PublisherActor { .neighbors() .await .iter() - .filter_map(|pub_key| TryInto::<[u8; 32]>::try_into(pub_key.as_slice()).ok()) + .filter_map(|pub_key| pub_key.as_bytes().as_ref().try_into().ok()) .take(5) .collect::>(); @@ -95,7 +95,7 @@ impl PublisherActor { .last_message_hashes() .await .iter() - .filter_map(|hash| TryInto::<[u8; 32]>::try_into(hash.as_slice()).ok()) + .filter_map(|hash| hash.as_ref().try_into().ok()) .take(5) .collect::>(); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index e1171a7..baf2b1f 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -1,5 +1,7 @@ //! Main topic handle combining bootstrap, publishing, and merging. +use std::sync::Arc; + use crate::{ GossipSender, crypto::RecordTopic, @@ -146,7 +148,12 @@ impl Topic { /// Split into sender and receiver for message exchange. pub async fn split(&self) -> Result<(GossipSender, crate::gossip::receiver::GossipReceiver)> { - Ok((self.gossip_sender().await?, self.gossip_receiver().await?)) + let topic_ref = Arc::new(self.clone()); + let mut sender = self.gossip_sender().await?; + let mut receiver = self.gossip_receiver().await?; + sender._topic_keep_alive = Some(topic_ref.clone()); + receiver._topic_keep_alive = Some(topic_ref); + Ok((sender, receiver)) } /// Get the gossip sender for this topic. @@ -243,7 +250,6 @@ mod tests { let topic = crate::Topic::new(record_publisher, gossip.clone(), true).await.unwrap(); let cancel_token = topic.cancel_token(); - let receiver_probe = topic.gossip_receiver().await.unwrap(); let (sender, receiver) = topic.split().await.unwrap(); @@ -253,21 +259,8 @@ mod tests { drop(receiver); drop(topic); - tokio::time::timeout( - std::time::Duration::from_secs(5), - cancel_token.cancelled(), - ) - .await - .expect("TopicActor did not shut down, cancel token was never cancelled"); - - let result = tokio::time::timeout( - std::time::Duration::from_secs(5), - receiver_probe.next(), - ) - .await - .expect("GossipReceiverActor did not shut down, next() hung"); - assert!(result.is_none(), "expected None from dead receiver, got an event"); - - drop(receiver_probe); + tokio::task::yield_now().await; + + assert!(cancel_token.is_cancelled()); } } \ No newline at end of file From 1de3c6e9ca217ba4651154dda7407e48e5d2122e Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Tue, 14 Apr 2026 03:00:52 +0200 Subject: [PATCH 07/30] fix: redundent test removed from cicd and removed edited from on pull_request from cicd --- .github/workflows/test.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6024974..8de2099 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,23 +6,17 @@ on: # always on pullrequest pull_request: - types: [opened, synchronize, reopened, edited] + types: [opened, synchronize, reopened] env: CARGO_TERM_COLOR: always jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Run tests - run: cargo test --verbose - e2e-tests: name: End-to-End Tests runs-on: ubuntu-latest + steps: - uses: actions/checkout@v6 From 8afe88f5ea63702f442b5424dda129a3bd723254 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Tue, 14 Apr 2026 03:05:25 +0200 Subject: [PATCH 08/30] chore: test_topic_full_shutdown_on_drop test using cancel_token.cancelled instead of yield --- src/gossip/topic/bootstrap.rs | 2 +- src/gossip/topic/topic.rs | 27 ++++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 2c749af..c901c58 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -258,7 +258,7 @@ impl BootstrapActor { continue; } } - tracing::debug!("Bootstrap: completed successfully"); + tracing::debug!("Bootstrap: exited"); let _ = sender.send(()); } }); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index baf2b1f..ac58351 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -109,7 +109,12 @@ impl Topic { ); let cancel_token = CancellationToken::new(); - let bootstrap = Bootstrap::new(record_publisher.clone(), gossip.clone(), cancel_token.clone()).await?; + let bootstrap = Bootstrap::new( + record_publisher.clone(), + gossip.clone(), + cancel_token.clone(), + ) + .await?; tracing::debug!("Topic: bootstrap instance created"); let api = Handle::spawn(TopicActor { @@ -232,13 +237,13 @@ mod tests { let endpoint = iroh::Endpoint::builder(iroh::endpoint::presets::N0) .secret_key(secret_key.clone()) .bind() - .await.unwrap(); - let gossip = iroh_gossip::net::Gossip::builder() - .spawn(endpoint.clone()); + .await + .unwrap(); + let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); let topic_id = crate::TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - + let record_publisher = crate::RecordPublisher::new( topic_id.clone(), signing_key.verifying_key(), @@ -246,8 +251,10 @@ mod tests { None, initial_secret, ); - - let topic = crate::Topic::new(record_publisher, gossip.clone(), true).await.unwrap(); + + let topic = crate::Topic::new(record_publisher, gossip.clone(), true) + .await + .unwrap(); let cancel_token = topic.cancel_token(); @@ -259,8 +266,10 @@ mod tests { drop(receiver); drop(topic); - tokio::task::yield_now().await; + tokio::time::timeout(std::time::Duration::from_secs(5), cancel_token.cancelled()) + .await + .unwrap(); assert!(cancel_token.is_cancelled()); } -} \ No newline at end of file +} From 10303f40d50bcd4680ac89b0e8d726b9e379e197 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Tue, 14 Apr 2026 13:55:50 +0200 Subject: [PATCH 09/30] chore: receiver functions converted to Result<> --- src/gossip/merge/bubble.rs | 2 +- src/gossip/merge/message_overlap.rs | 2 +- src/gossip/receiver.rs | 11 ++++----- src/gossip/topic/bootstrap.rs | 36 +++++++++++++++++++++-------- src/gossip/topic/publisher.rs | 4 ++-- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 89e5a1b..ed1def6 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -90,7 +90,7 @@ impl BubbleMergeActor { let mut records = self.record_publisher.get_records(unix_minute - 1).await; records.extend(self.record_publisher.get_records(unix_minute).await); - let neighbors = self.gossip_receiver.neighbors().await; + let neighbors = self.gossip_receiver.neighbors().await?; tracing::debug!( "BubbleMerge: checking with {} neighbors and {} records", neighbors.len(), diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 23404bb..06916ae 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -83,7 +83,7 @@ impl MessageOverlapMergeActor { let mut records = self.record_publisher.get_records(unix_minute - 1).await; records.extend(self.record_publisher.get_records(unix_minute).await); - let local_hashes = self.gossip_receiver.last_message_hashes().await; + let local_hashes = self.gossip_receiver.last_message_hashes().await?; tracing::debug!( "MessageOverlapMerge: checking {} records with {} local message hashes", records.len(), diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 20eeca6..a0bea34 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -61,21 +61,19 @@ impl GossipReceiver { } /// Get the set of currently connected neighbor node IDs. - pub async fn neighbors(&self) -> HashSet { + pub async fn neighbors(&self) -> Result> { self.api .call(act_ok!(actor => async move { actor.gossip_receiver.neighbors().collect::>() })) .await - .unwrap_or_default() } /// Check if the local node has joined the topic. - pub async fn is_joined(&self) -> bool { + pub async fn is_joined(&self) -> Result { self.api .call(act_ok!(actor => async move { actor.gossip_receiver.is_joined() })) .await - .unwrap_or(false) } /// Receive the next gossip event. @@ -95,11 +93,10 @@ impl GossipReceiver { /// Get SHA512 hashes (first 32 bytes) of recently received messages. /// /// Used for detecting message overlap during network partition recovery. - pub async fn last_message_hashes(&self) -> VecDeque<[u8; 32]> { + pub async fn last_message_hashes(&self) -> Result> { self.api .call(act_ok!(actor => async move { actor.last_message_hashes.clone() })) .await - .unwrap_or_default() } } @@ -138,7 +135,7 @@ impl GossipReceiverActor { iroh_gossip::api::Event::Received(msg) => { tracing::debug!("GossipReceiver: received message from {:?}", msg.delivered_from); let mut hash = sha2::Sha512::new(); - hash.update(msg.content.clone()); + hash.update(&msg.content); if let Ok(lmh) = hash.finalize()[..32].try_into() { if self.last_message_hashes.len() == 5 { self.last_message_hashes.pop_front(); diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index c901c58..c483fcd 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -93,9 +93,13 @@ impl BootstrapActor { tracing::debug!("Bootstrap: starting bootstrap process"); while !cancel_token.is_cancelled() { // Check if we are connected to at least one node - if gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined && is_joined { tracing::debug!("Bootstrap: already joined, exiting bootstrap loop"); break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); + break; } // On the first try we check the prev unix minute, after that the current one @@ -175,9 +179,13 @@ impl BootstrapActor { // Maybe in the meantime someone connected to us via one of our published records // we don't want to disrup the gossip rotations any more then we have to // so we check again before joining new peers - if gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined && is_joined { tracing::debug!("Bootstrap: joined while processing records, exiting"); break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); + break; } // Instead of throwing everything into join_peers() at once we go node_id by node_id @@ -191,11 +199,12 @@ impl BootstrapActor { _ = sleep(Duration::from_millis(100)) => {} _ = cancel_token.cancelled() => break, } - if gossip_receiver.is_joined().await { - tracing::debug!( - "Bootstrap: successfully joined via peer {}", - node_id - ); + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined && is_joined { + tracing::debug!("Bootstrap: successfully joined via peer {}", node_id); + break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); break; } } @@ -212,7 +221,8 @@ impl BootstrapActor { // If we are still not connected to anyone: // give it the default iroh-gossip connection timeout before the final is_joined() check - if !gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined && is_joined { tracing::debug!( "Bootstrap: not joined yet, waiting 500ms before final check" ); @@ -220,12 +230,20 @@ impl BootstrapActor { _ = sleep(Duration::from_millis(500)) => {} _ = cancel_token.cancelled() => break, } + break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); + break; } // If we are connected: return - if gossip_receiver.is_joined().await { + let is_joined = gossip_receiver.is_joined().await; + if let Ok(is_joined) = is_joined && is_joined { tracing::debug!("Bootstrap: successfully joined after final wait"); break; + } else if let Err(e) = is_joined { + tracing::debug!("Bootstrap: error checking join status: {:?}", e); + break; } else { tracing::debug!("Bootstrap: still not joined after attempting all peers"); // If we are not connected: check if we should publish a record this minute diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 17a4def..9b05398 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -84,7 +84,7 @@ impl PublisherActor { let active_peers = self .gossip_receiver .neighbors() - .await + .await? .iter() .filter_map(|pub_key| pub_key.as_bytes().as_ref().try_into().ok()) .take(5) @@ -93,7 +93,7 @@ impl PublisherActor { let last_message_hashes = self .gossip_receiver .last_message_hashes() - .await + .await? .iter() .filter_map(|hash| hash.as_ref().try_into().ok()) .take(5) From bcb94c660af33c9c3f8898b2c0c5114b18d913e2 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 04:06:50 +0200 Subject: [PATCH 10/30] add: config everything + fixes all around --- Cargo.toml | 11 +- README.md | 3 +- examples/chat.rs | 10 +- examples/chat_no_wait.rs | 8 +- examples/e2e_test.rs | 8 +- examples/full_config.rs | 106 +++++++ examples/secret_rotation.rs | 8 +- examples/simple.rs | 4 +- examples/without_mergers.rs | 55 ++++ src/config.rs | 462 ++++++++++++++++++++++++++++ src/crypto/keys.rs | 18 +- src/crypto/mod.rs | 2 +- src/crypto/record.rs | 154 ++++++---- src/dht.rs | 53 ++-- src/gossip/merge/bubble.rs | 28 +- src/gossip/merge/message_overlap.rs | 34 +- src/gossip/merge/mod.rs | 4 +- src/gossip/mod.rs | 6 +- src/gossip/receiver.rs | 145 +++++---- src/gossip/sender.rs | 17 +- src/gossip/topic/bootstrap.rs | 121 ++++++-- src/gossip/topic/publisher.rs | 25 +- src/gossip/topic/topic.rs | 167 ++++++---- src/lib.rs | 17 +- tests/test_gossip.rs | 16 - tests/tests.rs | 241 +++++++++++---- 26 files changed, 1336 insertions(+), 387 deletions(-) create mode 100644 examples/full_config.rs create mode 100644 examples/without_mergers.rs create mode 100644 src/config.rs delete mode 100644 tests/test_gossip.rs diff --git a/Cargo.toml b/Cargo.toml index 90b8c35..dc2e8c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,9 +42,6 @@ serde = { version = "1", default-features = false, features = ["std"] } tracing = { version = "0.1", default-features = false, features = ["std"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "env-filter", "ansi"] } -[lib] -crate-type = ["cdylib", "rlib"] - [[test]] name = "test_gossip" required-features = ["iroh-gossip"] @@ -72,3 +69,11 @@ required-features = ["iroh-gossip"] [[example]] name = "simple" required-features = ["iroh-gossip"] + +[[example]] +name = "full_config" +required-features = ["iroh-gossip"] + +[[example]] +name = "without_mergers" +required-features = ["iroh-gossip"] \ No newline at end of file diff --git a/README.md b/README.md index c91985c..5142d44 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,7 @@ distributed-topic-tracker = "0.2" ``` Basic iroh-gossip integration: - -```rust,no_run +```rust,no_run,ignore use anyhow::Result; use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; diff --git a/examples/chat.rs b/examples/chat.rs index 5023adb..38e5c3e 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -5,7 +5,9 @@ use iroh_gossip::{api::Event, net::Gossip}; use ed25519_dalek::SigningKey; // Imports from distributed-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +use distributed_topic_tracker::{ + AutoDiscoveryGossip, Config, RecordPublisher, TopicId, +}; #[tokio::main] async fn main() -> Result<()> { @@ -44,14 +46,14 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); // Split into sink (sending) and stream (receiving) - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await? .split() @@ -61,7 +63,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Some(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", diff --git a/examples/chat_no_wait.rs b/examples/chat_no_wait.rs index 11b2022..3537b5d 100644 --- a/examples/chat_no_wait.rs +++ b/examples/chat_no_wait.rs @@ -3,7 +3,7 @@ use iroh::{Endpoint, SecretKey}; use iroh_gossip::{api::Event, net::Gossip}; // Imports from distrubuted-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; use mainline::SigningKey; #[tokio::main] @@ -31,12 +31,12 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery_no_wait(record_publisher) .await? .split() @@ -46,7 +46,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Some(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", diff --git a/examples/e2e_test.rs b/examples/e2e_test.rs index d356e2e..415a237 100644 --- a/examples/e2e_test.rs +++ b/examples/e2e_test.rs @@ -5,7 +5,7 @@ use iroh::{ use iroh_gossip::net::Gossip; // Imports from distrubuted-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; #[tokio::main] async fn main() -> Result<()> { @@ -32,19 +32,19 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await? .split() .await?; tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Some(event) = gossip_receiver.next().await { println!("event: {event:?}"); } }); diff --git a/examples/full_config.rs b/examples/full_config.rs new file mode 100644 index 0000000..4ad2b54 --- /dev/null +++ b/examples/full_config.rs @@ -0,0 +1,106 @@ +use std::time::Duration; + +use anyhow::Result; +use ed25519_dalek::SigningKey; +use iroh::{Endpoint, SecretKey}; +use iroh_gossip::net::Gossip; + +// Imports from distrubuted-topic-tracker +use distributed_topic_tracker::{ + AutoDiscoveryGossip, BootstrapConfig, BubbleMergeConfig, Config, DefaultSecretRotation, + DhtConfig, MergeConfig, MessageOverlapMergeConfig, PublisherConfig, RecordPublisher, + RotationHandle, TimeoutConfig, TopicId, +}; + +// Default configuration, all in one spot as an overview +// same as `Config::default()` +fn config_builder() -> Config { + Config::builder() + .dht_config( + DhtConfig::builder() + .retries(3) + .base_retry_interval(Duration::from_secs(5)) + .max_retry_jitter(Duration::from_secs(10)) + .get_timeout(Duration::from_secs(10)) + .put_timeout(Duration::from_secs(10)) + .build(), + ) + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(5) + .publish_record_on_startup(true) + .discovery_poll_interval(Duration::from_millis(2000)) + .no_peers_retry_interval(Duration::from_millis(1500)) + .per_peer_join_settle_time(Duration::from_millis(100)) + .join_confirmation_wait_time(Duration::from_millis(500)) + .build(), + ) + .max_join_peer_count(4) + .publisher_config(PublisherConfig::Enabled { + initial_delay: Duration::from_secs(10), + base_interval: Duration::from_secs(10), + max_jitter: Duration::from_secs(50), + }) + .merge_config(MergeConfig::new( + BubbleMergeConfig::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + min_neighbors: 4, + }, + MessageOverlapMergeConfig::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + }, + )) + .timeouts( + TimeoutConfig::builder() + .join_peer_timeout(Duration::from_secs(5)) + .broadcast_neighbors_timeout(Duration::from_secs(5)) + .broadcast_timeout(Duration::from_secs(5)) + .build(), + ) + .build() +} + +#[tokio::main] +async fn main() -> Result<()> { + // Generate a new random secret key + let secret_key = SecretKey::generate(&mut rand::rng()); + let signing_key = SigningKey::from_bytes(&secret_key.to_bytes()); + + // Set up endpoint with discovery enabled + let endpoint = Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await?; + + // Initialize gossip with auto-discovery + let gossip = Gossip::builder().spawn(endpoint.clone()); + + // Set up protocol router + let _router = iroh::protocol::Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .spawn(); + + let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + // Split into sink (sending) and stream (receiving) + let record_publisher = + RecordPublisher::builder(topic_id.clone(), signing_key.clone(), initial_secret) + .config(config_builder()) + .secret_rotation(RotationHandle::new(DefaultSecretRotation)) + .build(); + + let topic = gossip + .subscribe_and_join_with_auto_discovery(record_publisher) + .await?; + + println!("[joined topic]"); + + // Do something with the gossip topic + // (bonus: GossipSender and GossipReceiver are safely clonable) + let (_gossip_sender, _gossip_receiver) = topic.split().await?; + + Ok(()) +} diff --git a/examples/secret_rotation.rs b/examples/secret_rotation.rs index 25f7068..3003fd4 100644 --- a/examples/secret_rotation.rs +++ b/examples/secret_rotation.rs @@ -6,7 +6,7 @@ use sha2::Digest; // Imports from distrubuted-topic-tracker use distributed_topic_tracker::{ - AutoDiscoveryGossip, RecordPublisher, RotationHandle, SecretRotation, TopicId, + AutoDiscoveryGossip, Config, RecordPublisher, RotationHandle, SecretRotation, TopicId }; struct MySecretRotation; @@ -53,12 +53,12 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), Some(RotationHandle::new(MySecretRotation)), initial_secret, + Config::default(), ); - let (gossip_sender, gossip_receiver) = gossip + let (gossip_sender, mut gossip_receiver) = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await? .split() @@ -68,7 +68,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(Ok(event)) = gossip_receiver.next().await { + while let Some(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", diff --git a/examples/simple.rs b/examples/simple.rs index 79b4354..db451af 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -4,7 +4,7 @@ use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; // Imports from distrubuted-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, RecordPublisher, TopicId}; +use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; #[tokio::main] async fn main() -> Result<()> { @@ -33,10 +33,10 @@ async fn main() -> Result<()> { let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); let topic = gossip diff --git a/examples/without_mergers.rs b/examples/without_mergers.rs new file mode 100644 index 0000000..98fb9d2 --- /dev/null +++ b/examples/without_mergers.rs @@ -0,0 +1,55 @@ +use anyhow::Result; +use ed25519_dalek::SigningKey; +use iroh::{Endpoint, SecretKey}; +use iroh_gossip::net::Gossip; + +// Imports from distrubuted-topic-tracker +use distributed_topic_tracker::{AutoDiscoveryGossip, BubbleMergeConfig, Config, MergeConfig, MessageOverlapMergeConfig, RecordPublisher, TopicId}; + +#[tokio::main] +async fn main() -> Result<()> { + // Generate a new random secret key + let secret_key = SecretKey::generate(&mut rand::rng()); + let signing_key = SigningKey::from_bytes(&secret_key.to_bytes()); + + // Set up endpoint with discovery enabled + let endpoint = Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await?; + + // Initialize gossip with auto-discovery + let gossip = Gossip::builder().spawn(endpoint.clone()); + + // Set up protocol router + let _router = iroh::protocol::Router::builder(endpoint.clone()) + .accept(iroh_gossip::ALPN, gossip.clone()) + .spawn(); + + let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + // Split into sink (sending) and stream (receiving) + + let record_publisher = RecordPublisher::new( + topic_id.clone(), + signing_key.clone(), + None, + initial_secret, + + // [!] Disable merge workers (BubbleMerge and MessageOverlapMerge) + Config::builder().merge_config(MergeConfig::new(BubbleMergeConfig::Disabled, MessageOverlapMergeConfig::Disabled)).build(), + ); + + let topic = gossip + .subscribe_and_join_with_auto_discovery(record_publisher) + .await?; + + println!("[joined topic]"); + + // Do something with the gossip topic + // (bonus: GossipSender and GossipReceiver are safely clonable) + let (_gossip_sender, _gossip_receiver) = topic.split().await?; + + Ok(()) +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..81cdc2d --- /dev/null +++ b/src/config.rs @@ -0,0 +1,462 @@ +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct TimeoutConfig { + join_peer_timeout: Duration, + broadcast_timeout: Duration, + broadcast_neighbors_timeout: Duration, +} + +impl TimeoutConfig { + pub fn builder() -> TimeoutConfigBuilder { + TimeoutConfigBuilder { + timeouts: TimeoutConfig::default(), + } + } + + /// How long to wait when joining a peer. Default: 5s. + pub fn join_peer_timeout(&self) -> Duration { + self.join_peer_timeout + } + + /// How long to wait when broadcasting messages. Default: 5s. + pub fn broadcast_timeout(&self) -> Duration { + self.broadcast_timeout + } + + /// How long to wait when broadcasting to neighbors. Default: 5s. + pub fn broadcast_neighbors_timeout(&self) -> Duration { + self.broadcast_neighbors_timeout + } +} + +impl Default for TimeoutConfig { + fn default() -> Self { + Self { + join_peer_timeout: Duration::from_secs(5), + broadcast_timeout: Duration::from_secs(5), + broadcast_neighbors_timeout: Duration::from_secs(5), + } + } +} + +#[derive(Debug)] +pub struct TimeoutConfigBuilder { + timeouts: TimeoutConfig, +} + +impl TimeoutConfigBuilder { + /// How long to wait when joining a peer. Default: 5s. + pub fn join_peer_timeout(mut self, timeout: Duration) -> Self { + self.timeouts.join_peer_timeout = timeout; + self + } + + /// How long to wait when broadcasting messages. Default: 5s. + pub fn broadcast_timeout(mut self, timeout: Duration) -> Self { + self.timeouts.broadcast_timeout = timeout; + self + } + + /// How long to wait when broadcasting to neighbors. Default: 5s. + pub fn broadcast_neighbors_timeout(mut self, timeout: Duration) -> Self { + self.timeouts.broadcast_neighbors_timeout = timeout; + self + } + + pub fn build(self) -> TimeoutConfig { + self.timeouts + } +} + +#[derive(Debug, Clone)] +pub struct DhtConfig { + retries: usize, + base_retry_interval: Duration, + max_retry_jitter: Duration, + put_timeout: Duration, + get_timeout: Duration, +} + +#[derive(Debug, Clone)] +pub struct DhtConfigBuilder { + config: DhtConfig, +} + +impl DhtConfigBuilder { + /// Number of DHT operation retry attempts (first attempt + retries). Default: 3. + pub fn retries(mut self, retries: usize) -> Self { + self.config.retries = retries; + self + } + + /// Base delay between retries. Default: 5s. + pub fn base_retry_interval(mut self, interval: Duration) -> Self { + self.config.base_retry_interval = interval; + self + } + + /// Max random jitter added to retry interval. Default: 10s. + pub fn max_retry_jitter(mut self, jitter: Duration) -> Self { + self.config.max_retry_jitter = jitter; + self + } + + /// Timeout for DHT put operations. Default: 10s. + pub fn put_timeout(mut self, timeout: Duration) -> Self { + self.config.put_timeout = timeout; + self + } + + /// Timeout for DHT get operations. Default: 10s. + pub fn get_timeout(mut self, timeout: Duration) -> Self { + self.config.get_timeout = timeout; + self + } + + pub fn build(self) -> DhtConfig { + self.config + } +} + +impl DhtConfig { + pub fn builder() -> DhtConfigBuilder { + DhtConfigBuilder { + config: DhtConfig::default(), + } + } + + /// Number of DHT operation retry attempts (first attempt + retries). Default: 3. + pub fn retries(&self) -> usize { + self.retries + } + + /// Base delay between retries. Default: 5s. + pub fn base_retry_interval(&self) -> Duration { + self.base_retry_interval + } + + /// Max random jitter added to retry interval. Default: 10s. + pub fn max_retry_jitter(&self) -> Duration { + self.max_retry_jitter + } + + /// Timeout for DHT put operations. Default: 10s. + pub fn put_timeout(&self) -> Duration { + self.put_timeout + } + + /// Timeout for DHT get operations. Default: 10s. + pub fn get_timeout(&self) -> Duration { + self.get_timeout + } +} + +impl Default for DhtConfig { + fn default() -> Self { + Self { + retries: 3, + base_retry_interval: Duration::from_secs(5), + max_retry_jitter: Duration::from_secs(10), + put_timeout: Duration::from_secs(10), + get_timeout: Duration::from_secs(10), + } + } +} + +#[derive(Debug, Clone)] +pub enum BubbleMergeConfig { + Enabled { + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, + }, + Disabled, +} + +#[derive(Debug, Clone)] +pub enum MessageOverlapMergeConfig { + Enabled { + base_interval: Duration, + max_jitter: Duration, + }, + Disabled, +} + +#[derive(Debug, Clone)] +pub enum PublisherConfig { + Enabled { + initial_delay: Duration, + base_interval: Duration, + max_jitter: Duration, + }, + Disabled, +} + +/// total_interval = base_interval + rand::random::() % max_jitter; +#[derive(Debug, Clone)] +pub struct MergeConfig { + bubble_merge: BubbleMergeConfig, + message_overlap_merge: MessageOverlapMergeConfig, +} + +impl Default for PublisherConfig { + fn default() -> Self { + Self::Enabled { + initial_delay: Duration::from_secs(10), + base_interval: Duration::from_secs(10), + max_jitter: Duration::from_secs(50), + } + } +} + +impl MergeConfig { + /// Defaults: bubble_merge=Enabled(60s base, 120s jitter, 4 min neighbors), message_overlap_merge=Enabled(60s base, 120s jitter). + pub fn new( + bubble_merge: BubbleMergeConfig, + message_overlap_merge: MessageOverlapMergeConfig, + ) -> Self { + Self { + bubble_merge, + message_overlap_merge, + } + } + + /// Bubble merge strategy config. Default: enabled, 60s base, 120s jitter, 4 min neighbors. + pub fn bubble_merge(&self) -> &BubbleMergeConfig { + &self.bubble_merge + } + + /// Message overlap merge strategy config. Default: enabled, 60s base, 120s jitter. + pub fn message_overlap_merge(&self) -> &MessageOverlapMergeConfig { + &self.message_overlap_merge + } +} + +#[derive(Debug, Clone)] +pub struct BootstrapConfig { + max_bootstrap_records: usize, + no_peers_retry_interval: Duration, + per_peer_join_settle_time: Duration, + join_confirmation_wait_time: Duration, + discovery_poll_interval: Duration, + publish_record_on_startup: bool, +} + +impl Default for BootstrapConfig { + fn default() -> Self { + Self { + max_bootstrap_records: 5, + no_peers_retry_interval: Duration::from_millis(1500), + per_peer_join_settle_time: Duration::from_millis(100), + join_confirmation_wait_time: Duration::from_millis(500), + discovery_poll_interval: Duration::from_millis(2000), + publish_record_on_startup: true, + } + } +} + +#[derive(Debug)] +pub struct BootstrapConfigBuilder { + config: BootstrapConfig, +} + +impl BootstrapConfigBuilder { + /// Max bootstrap records per topic per minute slot (min 1). Default: 5. + pub fn max_bootstrap_records(mut self, max_records: usize) -> Self { + self.config.max_bootstrap_records = max_records; + self + } + + /// How long to wait when no peers are found before retrying. Default: 1500ms. + pub fn no_peers_retry_interval(mut self, interval_ms: Duration) -> Self { + self.config.no_peers_retry_interval = interval_ms; + self + } + + /// How long to wait after joining a peer before attempting to join another. Default: 100ms. + pub fn per_peer_join_settle_time(mut self, interval_ms: Duration) -> Self { + self.config.per_peer_join_settle_time = interval_ms; + self + } + + /// How long to wait after joining a peer before checking if joined successfully. Default: 500ms. + pub fn join_confirmation_wait_time(mut self, interval_ms: Duration) -> Self { + self.config.join_confirmation_wait_time = interval_ms; + self + } + + /// How long to wait between DHT discovery attempts. Default: 2000ms. + pub fn discovery_poll_interval(mut self, interval_ms: Duration) -> Self { + self.config.discovery_poll_interval = interval_ms; + self + } + + /// Whether to publish a bootstrap record unconditionally on startup before dht get. Default: true. + pub fn publish_record_on_startup(mut self, publish: bool) -> Self { + self.config.publish_record_on_startup = publish; + self + } + + pub fn build(self) -> BootstrapConfig { + self.config + } +} + +impl BootstrapConfig { + pub fn builder() -> BootstrapConfigBuilder { + BootstrapConfigBuilder { + config: BootstrapConfig::default(), + } + } + + /// Max bootstrap records per topic per minute slot (min 1). Default: 5. + pub fn max_bootstrap_records(&self) -> usize { + self.max_bootstrap_records + } + + /// How long to wait when no peers are found before retrying. Default: 1500ms. + pub fn no_peers_retry_interval(&self) -> Duration { + self.no_peers_retry_interval + } + + /// How long to wait after joining a peer before attempting to join another. Default: 100ms. + pub fn per_peer_join_settle_time(&self) -> Duration { + self.per_peer_join_settle_time + } + + /// How long to wait after joining a peer before checking if joined successfully. Default: 500ms. + pub fn join_confirmation_wait_time(&self) -> Duration { + self.join_confirmation_wait_time + } + + /// How long to wait between DHT discovery attempts. Default: 2000ms. + pub fn discovery_poll_interval(&self) -> Duration { + self.discovery_poll_interval + } + + /// Whether to publish a bootstrap record unconditionally on startup before dht get. Default: true. + pub fn publish_record_on_startup(&self) -> bool { + self.publish_record_on_startup + } +} + +#[derive(Debug, Clone)] +pub struct Config { + bootstrap_config: BootstrapConfig, + publisher_config: PublisherConfig, + dht_config: DhtConfig, + + merge_config: MergeConfig, + + max_join_peer_count: usize, + timeouts: TimeoutConfig, +} + +impl Config { + pub fn builder() -> ConfigBuilder { + ConfigBuilder { + config: Config::default(), + } + } + + /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. + pub fn publisher_config(&self) -> &PublisherConfig { + &self.publisher_config + } + + /// DHT operation settings. Default: DhtConfig::default(). + pub fn dht_config(&self) -> &DhtConfig { + &self.dht_config + } + + /// Bootstrap strategy settings. Default: max 5 bootstrap records per topic per minute, 1500ms no peers retry interval, 100ms per peer join settle time, 500ms join confirmation wait time, 2000ms discovery poll interval. + pub fn bootstrap_config(&self) -> &BootstrapConfig { + &self.bootstrap_config + } + + /// Max peers to join simultaneously (min 1). Default: 4. + pub fn max_join_peer_count(&self) -> usize { + self.max_join_peer_count + } + + /// Timeout settings. Default: TimeoutConfig::default(). + pub fn timeouts(&self) -> &TimeoutConfig { + &self.timeouts + } + + /// Merge strategy settings. Default: bubble and overlap merges enabled. + pub fn merge_config(&self) -> &MergeConfig { + &self.merge_config + } +} + +impl Default for Config { + fn default() -> Self { + Self { + merge_config: MergeConfig { + bubble_merge: BubbleMergeConfig::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + min_neighbors: 4, + }, + message_overlap_merge: MessageOverlapMergeConfig::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + }, + }, + bootstrap_config: BootstrapConfig::default(), + publisher_config: PublisherConfig::default(), + dht_config: DhtConfig::default(), + max_join_peer_count: 4, + timeouts: TimeoutConfig::default(), + } + } +} + +#[derive(Debug)] +pub struct ConfigBuilder { + config: Config, +} + +impl ConfigBuilder { + /// Merge strategy settings. Default: bubble and overlap merges enabled. + pub fn merge_config(mut self, merge_config: MergeConfig) -> Self { + self.config.merge_config = merge_config; + self + } + + /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. + pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { + self.config.publisher_config = publisher_config; + self + } + + /// DHT operation settings. Default: DhtConfig::default(). + pub fn dht_config(mut self, dht_config: DhtConfig) -> Self { + self.config.dht_config = dht_config; + self + } + + /// Bootstrap strategy settings. Default: max 5 bootstrap records per topic per minute, 1500ms no peers retry interval, 100ms per peer join settle time, 500ms join confirmation wait time, 2000ms discovery poll interval. + pub fn bootstrap_config(mut self, bootstrap_config: BootstrapConfig) -> Self { + self.config.bootstrap_config = bootstrap_config; + self + } + + /// Max peers to join simultaneously (min 1). Default: 4. + pub fn max_join_peer_count(mut self, max_peers: usize) -> Self { + self.config.max_join_peer_count = max_peers; + self + } + + /// Timeout settings. Default: TimeoutConfig::default(). + pub fn timeouts(mut self, timeouts: TimeoutConfig) -> Self { + self.config.timeouts = timeouts; + self + } + + pub fn build(self) -> Config { + self.config + } +} diff --git a/src/crypto/keys.rs b/src/crypto/keys.rs index 4327bf5..adb1659 100644 --- a/src/crypto/keys.rs +++ b/src/crypto/keys.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use sha2::Digest; -use crate::crypto::RecordTopic; +use crate::TopicId; /// Trait for deriving time-rotated encryption keys. /// @@ -96,13 +96,13 @@ impl RotationHandle { /// # Example /// /// ```ignore -/// let topic = RecordTopic::from_str("my-topic")?; +/// let topic = TopicId::from_str("my-topic")?; /// let unix_minute = crate::unix_minute(0); /// let signing_key = signing_keypair(topic, unix_minute); /// ``` -pub fn signing_keypair(record_topic: RecordTopic, unix_minute: u64) -> ed25519_dalek::SigningKey { +pub fn signing_keypair(topic_id: &TopicId, unix_minute: u64) -> ed25519_dalek::SigningKey { let mut sign_keypair_hash = sha2::Sha512::new(); - sign_keypair_hash.update(record_topic.hash()); + sign_keypair_hash.update(topic_id.hash()); sign_keypair_hash.update(unix_minute.to_le_bytes()); let sign_keypair_seed: [u8; 32] = sign_keypair_hash.finalize()[..32] .try_into() @@ -117,12 +117,12 @@ pub fn signing_keypair(record_topic: RecordTopic, unix_minute: u64) -> ed25519_d /// # Example /// /// ```ignore -/// let topic = RecordTopic::from_str("my-topic")?; +/// let topic = TopicId::from_str("my-topic")?; /// let rotation = RotationHandle::default(); /// let enc_key = encryption_keypair(topic, &rotation, initial_hash, 0); /// ``` pub fn encryption_keypair( - record_topic: RecordTopic, + topic_id: &TopicId, secret_rotation_function: &RotationHandle, initial_secret_hash: [u8; 32], unix_minute: u64, @@ -130,7 +130,7 @@ pub fn encryption_keypair( let enc_keypair_seed = secret_rotation_function .0 - .derive(record_topic.hash(), unix_minute, initial_secret_hash); + .derive(topic_id.hash(), unix_minute, initial_secret_hash); ed25519_dalek::SigningKey::from_bytes(&enc_keypair_seed) } @@ -138,9 +138,9 @@ pub fn encryption_keypair( /// /// Salt = SHA512(topic_hash || unix_minute.to_le_bytes())[..32] /// Ensures records are stored in different DHT slots per minute. -pub fn salt(record_topic: RecordTopic, unix_minute: u64) -> [u8; 32] { +pub fn salt(topic_id: &TopicId, unix_minute: u64) -> [u8; 32] { let mut slot_hash = sha2::Sha512::new(); - slot_hash.update(record_topic.hash()); + slot_hash.update(topic_id.hash()); slot_hash.update(unix_minute.to_le_bytes()); slot_hash.finalize()[..32] .try_into() diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index 599a247..62c415d 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -10,4 +10,4 @@ pub use keys::{ DefaultSecretRotation, RotationHandle, SecretRotation, encryption_keypair, salt, signing_keypair, }; -pub use record::{EncryptedRecord, Record, RecordPublisher, RecordTopic}; +pub use record::{EncryptedRecord, Record, RecordPublisher}; diff --git a/src/crypto/record.rs b/src/crypto/record.rs index 46272aa..fca18c4 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, str::FromStr, time::Duration}; +use std::collections::HashSet; use anyhow::{Result, bail}; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; @@ -7,42 +7,7 @@ use ed25519_dalek_hpke::{Ed25519hpkeDecryption, Ed25519hpkeEncryption}; use serde::{Deserialize, Serialize}; use sha2::Digest; -/// Topic identifier derived from a string via SHA512 hashing. -/// -/// Used as a stable identifier for peer discovery records. -/// -/// # Example -/// -/// ```ignore -/// let topic = RecordTopic::from_str("chat-app-v1")?; -/// ``` -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RecordTopic([u8; 32]); - -impl FromStr for RecordTopic { - type Err = anyhow::Error; - - fn from_str(s: &str) -> std::result::Result { - let mut hasher = sha2::Sha512::new(); - hasher.update(s.as_bytes()); - let hash: [u8; 32] = hasher.finalize()[..32] - .try_into() - .map_err(|_| anyhow::anyhow!("hashing failed"))?; - Ok(RecordTopic(hash)) - } -} - -impl RecordTopic { - /// Create from a pre-computed 32-byte hash. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { - Self(*bytes) - } - - /// Get the raw 32-byte hash. - pub fn hash(&self) -> [u8; 32] { - self.0 - } -} +use crate::{Config, TopicId}; /// DHT record encrypted with HPKE. /// @@ -98,29 +63,75 @@ impl RecordContent { pub struct RecordPublisher { dht: crate::dht::Dht, - record_topic: RecordTopic, + config: crate::config::Config, + + topic_id: TopicId, pub_key: VerifyingKey, signing_key: SigningKey, secret_rotation: Option, initial_secret_hash: [u8; 32], } +#[derive(Debug)] +pub struct RecordPublisherBuilder { + topic_id: TopicId, + signing_key: SigningKey, + secret_rotation: Option, + initial_secret: Vec, + config: crate::config::Config, +} + +impl RecordPublisherBuilder { + pub fn secret_rotation(mut self, secret_rotation: crate::crypto::keys::RotationHandle) -> Self { + self.secret_rotation = Some(secret_rotation); + self + } + + pub fn config(mut self, config: crate::config::Config) -> Self { + self.config = config; + self + } + + pub fn build(self) -> RecordPublisher { + RecordPublisher::new( + self.topic_id, + self.signing_key, + self.secret_rotation, + self.initial_secret, + self.config, + ) + } +} + impl RecordPublisher { + pub fn builder( + topic_id: impl Into, + signing_key: SigningKey, + initial_secret: impl Into>, + ) -> RecordPublisherBuilder { + RecordPublisherBuilder { + topic_id: topic_id.into(), + signing_key, + secret_rotation: None, + initial_secret: initial_secret.into(), + config: crate::config::Config::default(), + } + } + /// Create a new record publisher. /// /// # Arguments /// - /// * `record_topic` - Topic identifier - /// * `pub_key` - Ed25519 public key (verifying key) + /// * `topic_id` - Topic identifier /// * `signing_key` - Ed25519 secret key (signing key) /// * `secret_rotation` - Optional custom key rotation strategy /// * `initial_secret` - Initial secret for key derivation pub fn new( - record_topic: impl Into, - pub_key: VerifyingKey, + topic_id: impl Into, signing_key: SigningKey, secret_rotation: Option, initial_secret: Vec, + config: crate::config::Config, ) -> Self { let mut initial_secret_hash = sha2::Sha512::new(); initial_secret_hash.update(initial_secret); @@ -129,9 +140,10 @@ impl RecordPublisher { .expect("hashing failed"); Self { - dht: crate::dht::Dht::new(), - record_topic: record_topic.into(), - pub_key, + dht: crate::dht::Dht::new(config.dht_config()), + config, + topic_id: topic_id.into(), + pub_key: signing_key.verifying_key(), signing_key, secret_rotation, initial_secret_hash, @@ -150,9 +162,8 @@ impl RecordPublisher { record_content: impl Serialize + Deserialize<'a>, ) -> Result { Record::sign( - self.record_topic.hash(), + self.topic_id.hash(), unix_minute, - self.pub_key.to_bytes(), record_content, &self.signing_key, ) @@ -164,13 +175,13 @@ impl RecordPublisher { } /// Get the record topic. - pub fn record_topic(&self) -> RecordTopic { - self.record_topic + pub fn topic_id(&self) -> &TopicId { + &self.topic_id } /// Get the signing key. - pub fn signing_key(&self) -> ed25519_dalek::SigningKey { - self.signing_key.clone() + pub fn signing_key(&self) -> &ed25519_dalek::SigningKey { + &self.signing_key } /// Get the secret rotation handle if set. @@ -182,6 +193,10 @@ impl RecordPublisher { pub fn initial_secret_hash(&self) -> [u8; 32] { self.initial_secret_hash } + + pub fn config(&self) -> &Config { + &self.config + } } impl RecordPublisher { @@ -203,19 +218,19 @@ impl RecordPublisher { record.unix_minute() ); - if records.len() >= crate::MAX_BOOTSTRAP_RECORDS { + if records.len() >= self.config.bootstrap_config().max_bootstrap_records().max(1) { tracing::debug!( "RecordPublisher: max records reached ({}), skipping publish", - crate::MAX_BOOTSTRAP_RECORDS + self.config.bootstrap_config().max_bootstrap_records() ); return Ok(()); } // Publish own records - let sign_key = crate::crypto::keys::signing_keypair(self.record_topic, record.unix_minute); - let salt = crate::crypto::keys::salt(self.record_topic, record.unix_minute); + let sign_key = crate::crypto::keys::signing_keypair(self.topic_id(), record.unix_minute); + let salt = crate::crypto::keys::salt(self.topic_id(), record.unix_minute); let encryption_key = crate::crypto::keys::encryption_keypair( - self.record_topic, + self.topic_id(), &self.secret_rotation.clone().unwrap_or_default(), self.initial_secret_hash, record.unix_minute, @@ -233,8 +248,6 @@ impl RecordPublisher { sign_key.verifying_key(), Some(salt.to_vec()), encrypted_record.to_bytes().to_vec(), - Some(3), - Duration::from_secs(10), ) .await?; @@ -251,14 +264,14 @@ impl RecordPublisher { unix_minute ); - let topic_sign = crate::crypto::keys::signing_keypair(self.record_topic, unix_minute); + let topic_sign = crate::crypto::keys::signing_keypair(self.topic_id(), unix_minute); let encryption_key = crate::crypto::keys::encryption_keypair( - self.record_topic, + self.topic_id(), &self.secret_rotation.clone().unwrap_or_default(), self.initial_secret_hash, unix_minute, ); - let salt = crate::crypto::keys::salt(self.record_topic, unix_minute); + let salt = crate::crypto::keys::salt(self.topic_id(), unix_minute); // Get records, decrypt and verify let records_iter = self @@ -267,7 +280,6 @@ impl RecordPublisher { topic_sign.verifying_key(), Some(salt.to_vec()), None, - Duration::from_secs(10), ) .await .unwrap_or_default(); @@ -282,7 +294,7 @@ impl RecordPublisher { .filter_map( |record| match EncryptedRecord::from_bytes(record.value().to_vec()) { Ok(encrypted_record) => match encrypted_record.decrypt(&encryption_key) { - Ok(record) => match record.verify(&self.record_topic.hash(), unix_minute) { + Ok(record) => match record.verify(&self.topic_id.hash(), unix_minute) { Ok(_) => match record.node_id().eq(self.pub_key.as_bytes()) { true => { tracing::debug!("RecordPublisher: filtered out self"); @@ -333,8 +345,15 @@ impl EncryptedRecord { /// Deserialize from bytes. pub fn from_bytes(buf: Vec) -> Result { + if buf.len() < 4 { + bail!("buffer too short for EncryptedRecord deserialization") + } let (encrypted_record_len, buf) = buf.split_at(4); let encrypted_record_len = u32::from_le_bytes(encrypted_record_len.try_into()?); + + if buf.len() != encrypted_record_len as usize + 88 /* encrypted_decryption_key.len() == 88 */ { + bail!("buffer length does not match expected encrypted record length") + } let (encrypted_record, encrypted_decryption_key) = buf.split_at(encrypted_record_len as usize); @@ -350,7 +369,6 @@ impl Record { pub fn sign<'a>( topic: [u8; 32], unix_minute: u64, - node_id: [u8; 32], record_content: impl Serialize + Deserialize<'a>, signing_key: &ed25519_dalek::SigningKey, ) -> anyhow::Result { @@ -358,14 +376,13 @@ impl Record { let mut signature_data = Vec::new(); signature_data.extend_from_slice(&topic); signature_data.extend_from_slice(&unix_minute.to_le_bytes()); - signature_data.extend_from_slice(&node_id); + signature_data.extend_from_slice(&signing_key.verifying_key().to_bytes()); signature_data.extend(&record_content.clone().0); - let signing_key = signing_key.clone(); let signature = signing_key.sign(&signature_data); Ok(Self { topic, unix_minute, - pub_key: node_id, + pub_key: signing_key.verifying_key().to_bytes(), content: record_content, signature: signature.to_bytes(), }) @@ -373,6 +390,9 @@ impl Record { /// Deserialize from bytes. pub fn from_bytes(buf: Vec) -> Result { + if buf.len() < 32 + 8 + 32 + 64 + 1 { + bail!("buffer too short for Record deserialization") + } let (topic, buf) = buf.split_at(32); let (unix_minute, buf) = buf.split_at(8); let (node_id, buf) = buf.split_at(32); diff --git a/src/dht.rs b/src/dht.rs index f533a71..ae0ccf7 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -11,7 +11,7 @@ use ed25519_dalek::VerifyingKey; use futures_lite::StreamExt; use mainline::{MutableItem, SigningKey}; -const RETRY_DEFAULT: usize = 3; +use crate::config::DhtConfig; /// DHT client wrapper with actor-based concurrency. /// @@ -25,15 +25,20 @@ pub struct Dht { #[derive(Debug, Default)] struct DhtActor { dht: Option, + config: crate::config::DhtConfig, } impl Dht { /// Create a new DHT client. /// /// Spawns a background actor for handling DHT operations. - pub fn new() -> Self { + pub fn new(dht_config: &DhtConfig) -> Self { Self { - api: Handle::spawn(DhtActor::default()).0, + api: Handle::spawn(DhtActor { + dht: None, + config: dht_config.clone(), + }) + .0, } } @@ -50,10 +55,9 @@ impl Dht { pub_key: VerifyingKey, salt: Option>, more_recent_than: Option, - timeout: Duration, ) -> Result> { self.api - .call(act!(actor => actor.get(pub_key, salt, more_recent_than, timeout))) + .call(act!(actor => actor.get(pub_key, salt, more_recent_than))) .await } @@ -73,16 +77,10 @@ impl Dht { pub_key: VerifyingKey, salt: Option>, data: Vec, - retry_count: Option, - timeout: Duration, ) -> Result<()> { - self.api.call(act!(actor => actor.put_mutable(signing_key, pub_key, salt, data, retry_count, timeout))).await - } -} - -impl Default for Dht { - fn default() -> Self { - Self::new() + self.api + .call(act!(actor => actor.put_mutable(signing_key, pub_key, salt, data))) + .await } } @@ -92,7 +90,6 @@ impl DhtActor { pub_key: VerifyingKey, salt: Option>, more_recent_than: Option, - timeout: Duration, ) -> Result> { if self.dht.is_none() { self.reset().await?; @@ -100,7 +97,7 @@ impl DhtActor { let dht = self.dht.as_mut().context("DHT not initialized")?; Ok(tokio::time::timeout( - timeout, + self.config.get_timeout(), dht.get_mutable(pub_key.as_bytes(), salt.as_deref(), more_recent_than) .collect::>(), ) @@ -113,18 +110,16 @@ impl DhtActor { pub_key: VerifyingKey, salt: Option>, data: Vec, - retry_count: Option, - timeout: Duration, ) -> Result<()> { if self.dht.is_none() { self.reset().await?; } - for i in 0..retry_count.unwrap_or(RETRY_DEFAULT) { + for i in 0..1+self.config.retries() { let dht = self.dht.as_mut().context("DHT not initialized")?; let most_recent_result = tokio::time::timeout( - timeout, + self.config.put_timeout(), dht.get_mutable_most_recent(pub_key.as_bytes(), salt.as_deref()), ) .await?; @@ -141,7 +136,7 @@ impl DhtActor { }; let put_result = match tokio::time::timeout( - Duration::from_secs(10), + self.config.put_timeout(), dht.put_mutable(item.clone(), Some(item.seq())), ) .await @@ -152,13 +147,25 @@ impl DhtActor { if put_result.is_some() { break; - } else if i == retry_count.unwrap_or(RETRY_DEFAULT) - 1 { + } else if i == self.config.retries() { bail!("failed to publish record") } self.reset().await?; - tokio::time::sleep(Duration::from_millis(rand::random::() % 2000)).await; + let retry_interval = self.config.base_retry_interval() + + Duration::from_millis(if self.config.max_retry_jitter().as_millis() > 0 { + rand::random::() % self.config.max_retry_jitter().as_millis() as u64 + } else { + 0 + }); + tracing::debug!( + "DHTActor: put_mutable attempt {}/{} failed, retrying in {:?}", + i + 1, + 1 + self.config.retries(), + retry_interval + ); + tokio::time::sleep(retry_interval).await; } Ok(()) } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index ed1def6..a4f048b 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -25,19 +25,28 @@ struct BubbleMergeActor { gossip_sender: GossipSender, ticker: tokio::time::Interval, cancel_token: tokio_util::sync::CancellationToken, + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, } impl BubbleMerge { /// Create a new bubble merge detector. /// /// Spawns a background task that periodically checks cluster size. + #[allow(clippy::too_many_arguments)] pub fn new( record_publisher: RecordPublisher, gossip_sender: GossipSender, gossip_receiver: GossipReceiver, cancel_token: tokio_util::sync::CancellationToken, + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, ) -> Result { - let mut ticker = tokio::time::interval(Duration::from_secs(10)); + let mut ticker = tokio::time::interval(base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -47,6 +56,10 @@ impl BubbleMerge { gossip_sender, ticker, cancel_token, + max_join_peers, + base_interval, + max_jitter, + min_neighbors, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -70,7 +83,7 @@ impl BubbleMergeActor { _ = self.ticker.tick() => { tracing::debug!("BubbleMerge: tick fired, checking for bubbles"); let _ = self.merge().await; - let next_interval = rand::random::() % 50; + let next_interval = self.base_interval.as_secs() + if self.max_jitter.as_secs() > 0 { rand::random::() % self.max_jitter.as_secs() } else {0}; tracing::debug!("BubbleMerge: next check in {}s", next_interval); self.ticker.reset_after(Duration::from_secs(next_interval)); } @@ -97,10 +110,11 @@ impl BubbleMergeActor { records.len() ); - if neighbors.len() < 4 && !records.is_empty() { + if neighbors.len() < self.min_neighbors && !records.is_empty() { tracing::debug!( - "BubbleMerge: detected small bubble ({} neighbors < 4), attempting merge", - neighbors.len() + "BubbleMerge: detected small bubble ({} neighbors < {}), attempting merge", + neighbors.len(), + self.min_neighbors ); let node_ids = records .iter() @@ -114,7 +128,7 @@ impl BubbleMergeActor { .filter_map(|&active_peer| { if active_peer == [0; 32] || neighbors.contains(&active_peer) - || active_peer.eq(record.node_id().to_vec().as_slice()) + || active_peer == record.node_id() || active_peer.eq(self.record_publisher.pub_key().as_bytes()) { None @@ -145,7 +159,7 @@ impl BubbleMergeActor { self.gossip_sender .join_peers( node_ids.iter().cloned().collect::>(), - Some(super::MAX_JOIN_PEERS_COUNT), + Some(self.max_join_peers), ) .await?; tracing::debug!("BubbleMerge: join_peers request sent"); diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 06916ae..66dea54 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -22,6 +22,10 @@ struct MessageOverlapMergeActor { gossip_sender: GossipSender, ticker: tokio::time::Interval, cancel_token: tokio_util::sync::CancellationToken, + + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, } impl MessageOverlapMerge { @@ -31,8 +35,11 @@ impl MessageOverlapMerge { gossip_sender: GossipSender, gossip_receiver: GossipReceiver, cancel_token: tokio_util::sync::CancellationToken, + max_join_peers: usize, + base_interval: Duration, + max_jitter: Duration, ) -> Result { - let mut ticker = tokio::time::interval(Duration::from_secs(10)); + let mut ticker = tokio::time::interval(base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -42,6 +49,9 @@ impl MessageOverlapMerge { gossip_sender, ticker, cancel_token, + max_join_peers, + base_interval, + max_jitter, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -64,7 +74,7 @@ impl MessageOverlapMergeActor { _ = self.ticker.tick() => { tracing::debug!("MessageOverlapMerge: tick fired, checking for split-brain"); let _ = self.merge().await; - let next_interval = rand::random::() % 50; + let next_interval = self.base_interval.as_secs() + if self.max_jitter.as_secs() > 0 { rand::random::() % self.max_jitter.as_secs() } else {0}; tracing::debug!("MessageOverlapMerge: next check in {}s", next_interval); self.ticker.reset_after(Duration::from_secs(next_interval)); } @@ -112,25 +122,35 @@ impl MessageOverlapMergeActor { ); if !peers_to_join.is_empty() { + let active_neighbors = self.gossip_receiver.neighbors().await?; let node_ids = peers_to_join .iter() .flat_map(|&record| { let mut peers = vec![]; - if let Ok(node_id) = EndpointId::from_bytes(&record.node_id()) { - peers.push(node_id); + if let Ok(endpoint_id) = EndpointId::from_bytes(&record.node_id()) + && endpoint_id + != EndpointId::from_verifying_key(self.record_publisher.pub_key()) + { + peers.push(endpoint_id); } if let Ok(content) = record.content::() { for active_peer in content.active_peers { if active_peer == [0; 32] { continue; } - if let Ok(node_id) = EndpointId::from_bytes(&active_peer) { - peers.push(node_id); + if let Ok(endpoint_id) = EndpointId::from_bytes(&active_peer) + && endpoint_id + != EndpointId::from_verifying_key( + self.record_publisher.pub_key(), + ) + { + peers.push(endpoint_id); } } } peers }) + .filter(|node_id| !active_neighbors.contains(node_id)) .collect::>(); tracing::debug!( @@ -141,7 +161,7 @@ impl MessageOverlapMergeActor { self.gossip_sender .join_peers( node_ids.iter().cloned().collect::>(), - Some(super::MAX_JOIN_PEERS_COUNT), + Some(self.max_join_peers), ) .await?; diff --git a/src/gossip/merge/mod.rs b/src/gossip/merge/mod.rs index b791400..87052c6 100644 --- a/src/gossip/merge/mod.rs +++ b/src/gossip/merge/mod.rs @@ -9,6 +9,4 @@ mod bubble; mod message_overlap; pub use bubble::BubbleMerge; -pub use message_overlap::MessageOverlapMerge; - -pub const MAX_JOIN_PEERS_COUNT: usize = 30; +pub use message_overlap::MessageOverlapMerge; \ No newline at end of file diff --git a/src/gossip/mod.rs b/src/gossip/mod.rs index fb79200..d3740ba 100644 --- a/src/gossip/mod.rs +++ b/src/gossip/mod.rs @@ -15,7 +15,7 @@ pub use sender::GossipSender; use serde::{Deserialize, Serialize}; pub use topic::{Bootstrap, Publisher, Topic, TopicId}; -use crate::RecordPublisher; +use crate::{MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher}; /// Record content for peer discovery. /// @@ -24,9 +24,9 @@ use crate::RecordPublisher; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GossipRecordContent { /// Fixed array of 5 peer node IDs (as 32-byte arrays, empty slots are zero-filled) - pub active_peers: [[u8; 32]; 5], + pub active_peers: [[u8; 32]; MAX_RECORD_PEERS], /// Fixed array of 5 recent message hashes for overlap detection (empty slots are zero-filled) - pub last_message_hashes: [[u8; 32]; 5], + pub last_message_hashes: [[u8; 32]; MAX_MESSAGE_HASHES], } /// Extension trait for iroh Gossip enabling auto-discovery. diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index a0bea34..04c4e57 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -5,35 +5,53 @@ use std::{ sync::Arc, }; -use actor_helper::{Action, Handle, Receiver, act, act_ok}; +use actor_helper::{Action, Handle, Receiver, act_ok}; use anyhow::Result; use futures_lite::StreamExt; use iroh::EndpointId; use sha2::Digest; -use crate::Topic; +use crate::{MAX_MESSAGE_HASHES, Topic}; /// Gossip receiver that collects incoming messages and neighbor info. /// /// Tracks SHA512 message hashes (first 32 bytes) for overlap detection and provides /// neighbor list for topology analysis. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct GossipReceiver { api: Handle, pub(crate) _topic_keep_alive: Option>, + _next_channel_sender: tokio::sync::broadcast::Sender< + Option>, + next_channel_receiver: tokio::sync::broadcast::Receiver< + Option, + >, + _join_channel_sender: tokio::sync::broadcast::Sender>, + join_channel_receiver: tokio::sync::broadcast::Receiver>, +} + +impl Clone for GossipReceiver { + fn clone(&self) -> Self { + Self { + api: self.api.clone(), + _topic_keep_alive: self._topic_keep_alive.clone(), + _next_channel_sender: self._next_channel_sender.clone(), + next_channel_receiver: self._next_channel_sender.subscribe(), + _join_channel_sender: self._join_channel_sender.clone(), + join_channel_receiver: self._join_channel_sender.subscribe(), + } + } } #[derive(Debug)] pub struct GossipReceiverActor { gossip_receiver: iroh_gossip::api::GossipReceiver, last_message_hashes: VecDeque<[u8; 32]>, - msg_queue: VecDeque>>, - waiters: VecDeque< - tokio::sync::oneshot::Sender< - Option>, - >, - >, cancel_token: tokio_util::sync::CancellationToken, + next_channel_sender: tokio::sync::broadcast::Sender< + Option, + >, + join_channel_sender: tokio::sync::broadcast::Sender>, } impl GossipReceiver { @@ -42,13 +60,15 @@ impl GossipReceiver { gossip_receiver: iroh_gossip::api::GossipReceiver, cancel_token: tokio_util::sync::CancellationToken, ) -> Self { + let (next_tx, next_rx) = tokio::sync::broadcast::channel(64); + let (join_tx, join_rx) = tokio::sync::broadcast::channel(64); let api = Handle::spawn_with( GossipReceiverActor { gossip_receiver, - last_message_hashes: VecDeque::with_capacity(5), - msg_queue: VecDeque::new(), - waiters: VecDeque::new(), + last_message_hashes: VecDeque::with_capacity(MAX_MESSAGE_HASHES), cancel_token, + next_channel_sender: next_tx.clone(), + join_channel_sender: join_tx.clone(), }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -57,6 +77,10 @@ impl GossipReceiver { Self { api, _topic_keep_alive: None, + next_channel_receiver: next_rx, + _next_channel_sender: next_tx, + join_channel_receiver: join_rx, + _join_channel_sender: join_tx, } } @@ -80,14 +104,34 @@ impl GossipReceiver { /// /// Returns `None` if the receiver is closed. pub async fn next( - &self, - ) -> Option> { - let (tx, rx) = tokio::sync::oneshot::channel(); - self.api - .call(act!(actor => actor.register_next(tx))) - .await - .ok()?; - rx.await.ok()? + &mut self, + ) -> Option { + match self.next_channel_receiver.recv().await { + Ok(event) => event, + Err(err) => match err { + tokio::sync::broadcast::error::RecvError::Closed => None, + tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { + tracing::warn!("GossipReceiver: event stream lagged, {skipped} events may have been missed"); + Box::pin(self.next()).await + } + }, + } + } + + pub async fn joined(&mut self) -> Option<()> { + if self.is_joined().await.unwrap_or(false) { + return Some(()); + } + match self.join_channel_receiver.recv().await { + Ok(event) => event, + Err(err) => match err { + tokio::sync::broadcast::error::RecvError::Closed => None, + tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { + tracing::warn!("GossipReceiver: join event stream lagged, {skipped} events may have been missed"); + Box::pin(self.joined()).await + } + }, + } } /// Get SHA512 hashes (first 32 bytes) of recently received messages. @@ -112,22 +156,17 @@ impl GossipReceiverActor { } } raw_event = self.gossip_receiver.next() => { - match raw_event { + let event = match raw_event { None => { - // if none => receiver closed - while let Some(waiter) = self.waiters.pop_back() { - let _ = waiter.send(None); - } + tracing::debug!("GossipReceiver: gossip receiver closed"); + self.join_channel_sender.send(None).ok(); + self.next_channel_sender.send(None).ok(); break Ok(()); } - Some(Err(_)) => { - // if Err => stream error, treat as closed - if let Some(waiter) = self.waiters.pop_back() { - let _ = waiter.send(raw_event); - } - while let Some(waiter) = self.waiters.pop_back() { - let _ = waiter.send(None); - } + Some(Err(err)) => { + tracing::warn!("GossipReceiver: error receiving gossip event: {err}"); + self.next_channel_sender.send(None).ok(); + self.join_channel_sender.send(None).ok(); break Ok(()); } Some(Ok(ref event)) => { @@ -137,14 +176,16 @@ impl GossipReceiverActor { let mut hash = sha2::Sha512::new(); hash.update(&msg.content); if let Ok(lmh) = hash.finalize()[..32].try_into() { - if self.last_message_hashes.len() == 5 { + if self.last_message_hashes.len() == MAX_MESSAGE_HASHES { self.last_message_hashes.pop_front(); } self.last_message_hashes.push_back(lmh); } + self.join_channel_sender.send(Some(())).ok(); } iroh_gossip::api::Event::NeighborUp(node_id) => { tracing::debug!("GossipReceiver: neighbor UP: {}", node_id); + self.join_channel_sender.send(Some(())).ok(); } iroh_gossip::api::Event::NeighborDown(node_id) => { tracing::debug!("GossipReceiver: neighbor DOWN: {}", node_id); @@ -152,44 +193,20 @@ impl GossipReceiverActor { iroh_gossip::api::Event::Lagged => { tracing::debug!("GossipReceiver: event stream lagged"); } - } + }; + event.clone() } }; - self.msg_queue.push_front(raw_event); - if let Some(waiter) = self.waiters.pop_back() { - if let Some(event) = self.msg_queue.pop_back() { - let _ = waiter.send(event); - } else { - let _ = waiter.send(None); - // this should never happen - } - } + self.next_channel_sender.send(Some(event.clone())).ok(); } _ = self.cancel_token.cancelled() => { - while let Some(waiter) = self.waiters.pop_back() { - let _ = waiter.send(None); - } + self.join_channel_sender.send(None).ok(); + self.next_channel_sender.send(None).ok(); break Ok(()); } else => break Ok(()), } } } -} - -impl GossipReceiverActor { - pub async fn register_next( - &mut self, - waiter: tokio::sync::oneshot::Sender< - Option>, - >, - ) -> Result<()> { - if let Some(event) = self.msg_queue.pop_back() { - let _ = waiter.send(event); - } else { - self.waiters.push_front(waiter); - } - Ok(()) - } -} +} \ No newline at end of file diff --git a/src/gossip/sender.rs b/src/gossip/sender.rs index 3d9ba39..44d008e 100644 --- a/src/gossip/sender.rs +++ b/src/gossip/sender.rs @@ -1,13 +1,13 @@ //! Actor-based wrapper for iroh-gossip broadcast operations. -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use actor_helper::{Handle, act}; use anyhow::Result; use iroh::EndpointId; use rand::seq::SliceRandom; -use crate::Topic; +use crate::{TimeoutConfig, Topic}; /// Gossip sender that broadcasts messages to peers. /// @@ -17,6 +17,7 @@ use crate::Topic; pub struct GossipSender { api: Handle, pub(crate) _topic_keep_alive: Option>, + pub(crate) timeout_config: TimeoutConfig, } #[derive(Debug)] @@ -26,22 +27,24 @@ pub struct GossipSenderActor { impl GossipSender { /// Create a new gossip sender from an iroh topic sender. - pub fn new(gossip_sender: iroh_gossip::api::GossipSender) -> Self { + pub fn new(gossip_sender: iroh_gossip::api::GossipSender, timeout_config: TimeoutConfig) -> Self { let api = Handle::spawn(GossipSenderActor { gossip_sender }).0; Self { api, _topic_keep_alive: None, + timeout_config, } } /// Broadcast a message to all peers in the topic. pub async fn broadcast(&self, data: Vec) -> Result<()> { tracing::debug!("GossipSender: broadcasting message ({} bytes)", data.len()); + let broadcast_timeout = self.timeout_config.broadcast_timeout(); self.api .call(act!(actor => async move { tokio::time::timeout( - Duration::from_secs(5), + broadcast_timeout, actor.gossip_sender.broadcast(data.into()) ).await .map_err(|_| anyhow::anyhow!("broadcast timed out"))? @@ -56,10 +59,11 @@ impl GossipSender { "GossipSender: broadcasting to neighbors ({} bytes)", data.len() ); + let broadcast_neighbors_timeout = self.timeout_config.broadcast_neighbors_timeout(); self.api .call(act!(actor => async move { tokio::time::timeout( - Duration::from_secs(5), + broadcast_neighbors_timeout, actor.gossip_sender.broadcast_neighbors(data.into()) ).await .map_err(|_| anyhow::anyhow!("broadcast_neighbors timed out"))? @@ -83,10 +87,11 @@ impl GossipSender { tracing::debug!("GossipSender: joining {} peers", peers.len()); + let join_peers_timeout = self.timeout_config.join_peer_timeout(); self.api .call(act!(actor => async move { tokio::time::timeout( - Duration::from_secs(5), + join_peers_timeout, actor.gossip_sender.join_peers(peers) ).await .map_err(|_| anyhow::anyhow!("join_peers timed out"))? diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index c483fcd..a77a530 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -1,6 +1,6 @@ //! Bootstrap process for discovering and joining peers via DHT. -use std::{collections::HashSet, time::Duration}; +use std::collections::HashSet; use actor_helper::{Handle, act, act_ok}; use anyhow::Result; @@ -8,7 +8,10 @@ use iroh::EndpointId; use tokio::time::sleep; use crate::{ - GossipSender, crypto::Record, gossip::{GossipRecordContent, receiver::GossipReceiver} + GossipSender, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, + config::BootstrapConfig, + crypto::Record, + gossip::{GossipRecordContent, receiver::GossipReceiver}, }; /// Manages the peer discovery and joining process. @@ -26,6 +29,7 @@ struct BootstrapActor { gossip_sender: GossipSender, gossip_receiver: GossipReceiver, cancel_token: tokio_util::sync::CancellationToken, + config: BootstrapConfig, } impl Bootstrap { @@ -34,16 +38,18 @@ impl Bootstrap { record_publisher: crate::crypto::RecordPublisher, gossip: iroh_gossip::net::Gossip, cancel_token: tokio_util::sync::CancellationToken, + timeout_config: crate::config::TimeoutConfig, + bootstrap_config: BootstrapConfig, ) -> Result { let gossip_topic: iroh_gossip::api::GossipTopic = gossip .subscribe( - iroh_gossip::proto::TopicId::from(record_publisher.record_topic().hash()), + iroh_gossip::proto::TopicId::from(record_publisher.topic_id().hash()), vec![], ) .await?; let (gossip_sender, gossip_receiver) = gossip_topic.split(); let (gossip_sender, gossip_receiver) = ( - GossipSender::new(gossip_sender), + GossipSender::new(gossip_sender, timeout_config), GossipReceiver::new(gossip_receiver, cancel_token.clone()), ); @@ -52,6 +58,7 @@ impl Bootstrap { gossip_sender, gossip_receiver, cancel_token, + config: bootstrap_config, }) .0; @@ -61,7 +68,7 @@ impl Bootstrap { /// Start the bootstrap process. /// /// Returns a receiver that signals completion when the node has joined the topic (has at least one neighbor). - pub async fn bootstrap(&self) -> Result> { + pub async fn bootstrap(&self) -> Result>> { self.api.call(act!(actor=> actor.start_bootstrap())).await } @@ -81,21 +88,48 @@ impl Bootstrap { } impl BootstrapActor { - pub async fn start_bootstrap(&mut self) -> Result> { + pub async fn start_bootstrap(&mut self) -> Result>> { let (sender, receiver) = tokio::sync::oneshot::channel(); tokio::spawn({ let mut last_published_unix_minute = 0; - let (gossip_sender, gossip_receiver) = + let (gossip_sender, mut gossip_receiver) = (self.gossip_sender.clone(), self.gossip_receiver.clone()); let record_publisher = self.record_publisher.clone(); let cancel_token = self.cancel_token.clone(); + let bootstrap_config = self.config.clone(); + let mut is_joined_ret = false; + + if self.config.publish_record_on_startup() { + let unix_minute = crate::unix_minute(0); + tracing::debug!("Bootstrap: initial startup record publish {}", unix_minute); + last_published_unix_minute = unix_minute; + let record_creator = record_publisher.clone(); + let record_content = GossipRecordContent { + active_peers: [[0; 32]; MAX_RECORD_PEERS], + last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], + }; + if let Ok(record) = Record::sign( + record_publisher.topic_id().hash(), + unix_minute, + record_content, + &record_publisher.signing_key(), + ) { + tokio::spawn(async move { + let _ = record_creator.publish_record(record).await; + }); + } + } + async move { tracing::debug!("Bootstrap: starting bootstrap process"); while !cancel_token.is_cancelled() { // Check if we are connected to at least one node let is_joined = gossip_receiver.is_joined().await; - if let Ok(is_joined) = is_joined && is_joined { + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!("Bootstrap: already joined, exiting bootstrap loop"); + is_joined_ret = true; break; } else if let Err(e) = is_joined { tracing::debug!("Bootstrap: error checking join status: {:?}", e); @@ -130,13 +164,12 @@ impl BootstrapActor { last_published_unix_minute = unix_minute; let record_creator = record_publisher.clone(); let record_content = GossipRecordContent { - active_peers: [[0; 32]; 5], - last_message_hashes: [[0; 32]; 5], + active_peers: [[0; 32]; MAX_RECORD_PEERS], + last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], }; if let Ok(record) = Record::sign( - record_publisher.record_topic().hash(), + record_publisher.topic_id().hash(), unix_minute, - record_publisher.pub_key().to_bytes(), record_content, &record_publisher.signing_key(), ) { @@ -146,7 +179,8 @@ impl BootstrapActor { } } tokio::select! { - _ = sleep(Duration::from_millis(100)) => {} + _ = sleep(bootstrap_config.no_peers_retry_interval()) => {} + _ = gossip_receiver.joined() => continue, _ = cancel_token.cancelled() => break, } continue; @@ -180,8 +214,11 @@ impl BootstrapActor { // we don't want to disrup the gossip rotations any more then we have to // so we check again before joining new peers let is_joined = gossip_receiver.is_joined().await; - if let Ok(is_joined) = is_joined && is_joined { + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!("Bootstrap: joined while processing records, exiting"); + is_joined_ret = true; break; } else if let Err(e) = is_joined { tracing::debug!("Bootstrap: error checking join status: {:?}", e); @@ -194,17 +231,27 @@ impl BootstrapActor { match gossip_sender.join_peers(vec![*node_id], None).await { Ok(_) => { tracing::debug!("Bootstrap: attempted to join peer {}", node_id); - + tokio::select! { - _ = sleep(Duration::from_millis(100)) => {} + _ = sleep(bootstrap_config.per_peer_join_settle_time()) => {} + _ = gossip_receiver.joined() => {}, _ = cancel_token.cancelled() => break, } let is_joined = gossip_receiver.is_joined().await; - if let Ok(is_joined) = is_joined && is_joined { - tracing::debug!("Bootstrap: successfully joined via peer {}", node_id); + if let Ok(is_joined) = is_joined + && is_joined + { + tracing::debug!( + "Bootstrap: successfully joined via peer {}", + node_id + ); + is_joined_ret = true; break; } else if let Err(e) = is_joined { - tracing::debug!("Bootstrap: error checking join status: {:?}", e); + tracing::debug!( + "Bootstrap: error checking join status: {:?}", + e + ); break; } } @@ -222,15 +269,18 @@ impl BootstrapActor { // If we are still not connected to anyone: // give it the default iroh-gossip connection timeout before the final is_joined() check let is_joined = gossip_receiver.is_joined().await; - if let Ok(is_joined) = is_joined && is_joined { + if let Ok(is_joined) = is_joined + && !is_joined + { tracing::debug!( - "Bootstrap: not joined yet, waiting 500ms before final check" + "Bootstrap: not joined yet, waiting {:?} before final check", + bootstrap_config.join_confirmation_wait_time() ); tokio::select! { - _ = sleep(Duration::from_millis(500)) => {} + _ = sleep(bootstrap_config.join_confirmation_wait_time()) => {} + _ = gossip_receiver.joined() => {}, _ = cancel_token.cancelled() => break, } - break; } else if let Err(e) = is_joined { tracing::debug!("Bootstrap: error checking join status: {:?}", e); break; @@ -238,8 +288,11 @@ impl BootstrapActor { // If we are connected: return let is_joined = gossip_receiver.is_joined().await; - if let Ok(is_joined) = is_joined && is_joined { + if let Ok(is_joined) = is_joined + && is_joined + { tracing::debug!("Bootstrap: successfully joined after final wait"); + is_joined_ret = true; break; } else if let Err(e) = is_joined { tracing::debug!("Bootstrap: error checking join status: {:?}", e); @@ -255,12 +308,11 @@ impl BootstrapActor { last_published_unix_minute = unix_minute; let record_creator = record_publisher.clone(); if let Ok(record) = Record::sign( - record_publisher.record_topic().hash(), + record_publisher.topic_id().hash(), unix_minute, - record_publisher.pub_key().to_bytes(), GossipRecordContent { - active_peers: [[0; 32]; 5], - last_message_hashes: [[0; 32]; 5], + active_peers: [[0; 32]; MAX_RECORD_PEERS], + last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], }, &record_publisher.signing_key(), ) { @@ -270,14 +322,21 @@ impl BootstrapActor { } } tokio::select! { - _ = sleep(Duration::from_millis(100)) => {} + _ = sleep(bootstrap_config.discovery_poll_interval()) => continue, + _ = gossip_receiver.joined() => continue, _ = cancel_token.cancelled() => break, } - continue; } } tracing::debug!("Bootstrap: exited"); - let _ = sender.send(()); + + if is_joined_ret { + let _ = sender.send(Ok(())); + } else { + let _ = sender.send(Err(anyhow::anyhow!( + "Bootstrap process failed or was cancelled" + ))); + } } }); diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 9b05398..2e2d5aa 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -3,13 +3,13 @@ use actor_helper::{Action, Handle, Receiver}; use std::time::Duration; -use crate::{GossipReceiver, RecordPublisher}; +use crate::{GossipReceiver, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher}; use anyhow::Result; /// Periodically publishes node state to DHT for peer discovery. /// -/// Publishes a record after an initial 10s delay, then repeatedly with -/// randomized 0-49s intervals, containing this node's active neighbor list +/// Publishes a record after an initial delay initial_delay, then repeatedly with +/// randomized base_interval + rand(0 to max_jitter) intervals, containing this node's active neighbor list /// and message hashes for bubble detection and merging. #[derive(Debug, Clone)] pub struct Publisher { @@ -22,6 +22,8 @@ struct PublisherActor { gossip_receiver: GossipReceiver, ticker: tokio::time::Interval, cancel_token: tokio_util::sync::CancellationToken, + base_interval: Duration, + max_jitter: Duration, } impl Publisher { @@ -32,8 +34,11 @@ impl Publisher { record_publisher: RecordPublisher, gossip_receiver: GossipReceiver, cancel_token: tokio_util::sync::CancellationToken, + initial_delay: Duration, + base_interval: Duration, + max_jitter: Duration, ) -> Result { - let mut ticker = tokio::time::interval(Duration::from_secs(10)); + let mut ticker = tokio::time::interval(initial_delay); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( PublisherActor { @@ -41,6 +46,8 @@ impl Publisher { gossip_receiver, ticker, cancel_token, + base_interval, + max_jitter, }, |mut actor, rx| async move { actor.run(rx).await }, ) @@ -64,7 +71,7 @@ impl PublisherActor { _ = self.ticker.tick() => { tracing::debug!("Publisher: tick fired, attempting to publish"); let _ = self.publish().await; - let next_interval = rand::random::() % 50; + let next_interval = self.base_interval.as_secs() + if self.max_jitter.as_secs() > 0 { rand::random::() % self.max_jitter.as_secs() } else {0}; tracing::debug!("Publisher: next publish in {}s", next_interval); self.ticker.reset_after(Duration::from_secs(next_interval)); } @@ -87,7 +94,7 @@ impl PublisherActor { .await? .iter() .filter_map(|pub_key| pub_key.as_bytes().as_ref().try_into().ok()) - .take(5) + .take(MAX_RECORD_PEERS) .collect::>(); let last_message_hashes = self @@ -96,7 +103,7 @@ impl PublisherActor { .await? .iter() .filter_map(|hash| hash.as_ref().try_into().ok()) - .take(5) + .take(MAX_MESSAGE_HASHES) .collect::>(); tracing::debug!( @@ -108,12 +115,12 @@ impl PublisherActor { let record_content = crate::gossip::GossipRecordContent { active_peers: { - let mut peers = [Default::default(); 5]; + let mut peers = [Default::default(); MAX_RECORD_PEERS]; peers[..active_peers.len()].copy_from_slice(&active_peers); peers }, last_message_hashes: { - let mut hashes = [Default::default(); 5]; + let mut hashes = [Default::default(); MAX_MESSAGE_HASHES]; hashes[..last_message_hashes.len()].copy_from_slice(&last_message_hashes); hashes }, diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index ac58351..4bd5ff3 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -1,10 +1,10 @@ //! Main topic handle combining bootstrap, publishing, and merging. -use std::sync::Arc; +use std::{str::FromStr, sync::Arc}; use crate::{ - GossipSender, - crypto::RecordTopic, + BubbleMergeConfig, GossipSender, MessageOverlapMergeConfig, + config::PublisherConfig, gossip::{ merge::{BubbleMerge, MessageOverlapMerge}, topic::{bootstrap::Bootstrap, publisher::Publisher}, @@ -25,14 +25,13 @@ use tokio_util::sync::CancellationToken; /// let topic_id = TopicId::new("chat-room-1".to_string()); /// ``` #[derive(Debug, Clone)] -pub struct TopicId { - _raw: String, - hash: [u8; 32], -} +pub struct TopicId([u8; 32]); + +impl FromStr for TopicId { + type Err = anyhow::Error; -impl From for RecordTopic { - fn from(val: TopicId) -> Self { - RecordTopic::from_bytes(&val.hash) + fn from_str(s: &str) -> std::result::Result { + Ok(Self::new(s.to_string())) } } @@ -44,23 +43,21 @@ impl TopicId { let mut raw_hash = sha2::Sha512::new(); raw_hash.update(raw.as_bytes()); - Self { - _raw: raw, - hash: raw_hash.finalize()[..32] + Self( + raw_hash.finalize()[..32] .try_into() .expect("hashing 'raw' failed"), - } + ) } - /// Get the hash bytes. - pub fn hash(&self) -> [u8; 32] { - self.hash + /// Create from a pre-computed 32-byte hash. + pub fn from_bytes(bytes: &[u8; 32]) -> Self { + Self(*bytes) } - /// Get the original string. - #[allow(dead_code)] - pub fn raw(&self) -> &str { - &self._raw + /// Get the hash bytes. + pub fn hash(&self) -> [u8; 32] { + self.0 } } @@ -113,13 +110,15 @@ impl Topic { record_publisher.clone(), gossip.clone(), cancel_token.clone(), + record_publisher.config().timeouts().clone(), + record_publisher.config().bootstrap_config().clone(), ) .await?; tracing::debug!("Topic: bootstrap instance created"); let api = Handle::spawn(TopicActor { bootstrap: bootstrap.clone(), - record_publisher, + record_publisher: record_publisher.clone(), publisher: None, bubble_merge: None, message_overlap_merge: None, @@ -130,22 +129,40 @@ impl Topic { let bootstrap_done = bootstrap.bootstrap().await?; if !async_bootstrap { tracing::debug!("Topic: waiting for bootstrap to complete"); - bootstrap_done.await?; + bootstrap_done.await??; tracing::debug!("Topic: bootstrap completed"); } else { tracing::debug!("Topic: bootstrap started asynchronously"); } - tracing::debug!("Topic: starting publisher"); - let _ = api.call(act!(actor => actor.start_publishing())).await; + if matches!( + record_publisher.config().publisher_config(), + PublisherConfig::Enabled { .. } + ) { + tracing::debug!("Topic: starting publisher"); + let _ = api.call(act!(actor => actor.start_publishing())).await; + } - tracing::debug!("Topic: starting bubble merge"); - let _ = api.call(act!(actor => actor.start_bubble_merge())).await; + if matches!( + record_publisher.config().merge_config().bubble_merge(), + BubbleMergeConfig::Enabled { .. } + ) { + tracing::debug!("Topic: starting bubble merge"); + let _ = api.call(act!(actor => actor.start_bubble_merge())).await; + } - tracing::debug!("Topic: starting message overlap merge"); - let _ = api - .call(act!(actor => actor.start_message_overlap_merge())) - .await; + if matches!( + record_publisher + .config() + .merge_config() + .message_overlap_merge(), + MessageOverlapMergeConfig::Enabled { .. } + ) { + tracing::debug!("Topic: starting message overlap merge"); + let _ = api + .call(act!(actor => actor.start_message_overlap_merge())) + .await; + } tracing::debug!("Topic: fully initialized"); Ok(Self { api, cancel_token }) @@ -190,40 +207,74 @@ impl Topic { impl TopicActor { pub async fn start_publishing(&mut self) -> Result<()> { - tracing::debug!("TopicActor: initializing publisher"); - let publisher = Publisher::new( - self.record_publisher.clone(), - self.bootstrap.gossip_receiver().await?, - self.cancel_token.clone(), - )?; - self.publisher = Some(publisher); - tracing::debug!("TopicActor: publisher started"); + if let PublisherConfig::Enabled { + initial_delay, + base_interval, + max_jitter, + } = self.record_publisher.config().publisher_config() + { + tracing::debug!("TopicActor: initializing publisher"); + let publisher = Publisher::new( + self.record_publisher.clone(), + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + *initial_delay, + *base_interval, + *max_jitter, + )?; + self.publisher = Some(publisher); + tracing::debug!("TopicActor: publisher started"); + } Ok(()) } pub async fn start_bubble_merge(&mut self) -> Result<()> { - tracing::debug!("TopicActor: initializing bubble merge"); - let bubble_merge = BubbleMerge::new( - self.record_publisher.clone(), - self.bootstrap.gossip_sender().await?, - self.bootstrap.gossip_receiver().await?, - self.cancel_token.clone(), - )?; - self.bubble_merge = Some(bubble_merge); - tracing::debug!("TopicActor: bubble merge started"); + if let BubbleMergeConfig::Enabled { + base_interval, + max_jitter, + min_neighbors, + } = self.record_publisher.config().merge_config().bubble_merge() + { + tracing::debug!("TopicActor: initializing bubble merge"); + let bubble_merge = BubbleMerge::new( + self.record_publisher.clone(), + self.bootstrap.gossip_sender().await?, + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + self.record_publisher.config().max_join_peer_count().max(1), + *base_interval, + *max_jitter, + *min_neighbors, + )?; + self.bubble_merge = Some(bubble_merge); + tracing::debug!("TopicActor: bubble merge started"); + } Ok(()) } pub async fn start_message_overlap_merge(&mut self) -> Result<()> { - tracing::debug!("TopicActor: initializing message overlap merge"); - let message_overlap_merge = MessageOverlapMerge::new( - self.record_publisher.clone(), - self.bootstrap.gossip_sender().await?, - self.bootstrap.gossip_receiver().await?, - self.cancel_token.clone(), - )?; - self.message_overlap_merge = Some(message_overlap_merge); - tracing::debug!("TopicActor: message overlap merge started"); + if let MessageOverlapMergeConfig::Enabled { + base_interval, + max_jitter, + } = self + .record_publisher + .config() + .merge_config() + .message_overlap_merge() + { + tracing::debug!("TopicActor: initializing message overlap merge"); + let message_overlap_merge = MessageOverlapMerge::new( + self.record_publisher.clone(), + self.bootstrap.gossip_sender().await?, + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + self.record_publisher.config().max_join_peer_count().max(1), + *base_interval, + *max_jitter, + )?; + self.message_overlap_merge = Some(message_overlap_merge); + tracing::debug!("TopicActor: message overlap merge started"); + } Ok(()) } } @@ -246,10 +297,10 @@ mod tests { let record_publisher = crate::RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + crate::config::Config::default(), ); let topic = crate::Topic::new(record_publisher, gossip.clone(), true) diff --git a/src/lib.rs b/src/lib.rs index 1001740..555748f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ #![doc = include_str!("../README.md")] +mod config; mod crypto; mod dht; @@ -11,17 +12,18 @@ pub use gossip::{ MessageOverlapMerge, Publisher, Topic, TopicId, }; +pub use config::{ + BubbleMergeConfig, Config, ConfigBuilder, DhtConfig, MergeConfig, MessageOverlapMergeConfig, + TimeoutConfig, TimeoutConfigBuilder, PublisherConfig, BootstrapConfig, +}; pub use crypto::{ - DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RecordTopic, RotationHandle, + DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RotationHandle, SecretRotation, encryption_keypair, salt, signing_keypair, }; pub use dht::Dht; -/// Maximum number of bootstrap records allowed per topic per time slot (minute). -/// -/// When publishing to the DHT, records are not published if this threshold -/// has already been reached for the current minute slot. -pub const MAX_BOOTSTRAP_RECORDS: usize = 100; +pub const MAX_RECORD_PEERS: usize = 5; +pub const MAX_MESSAGE_HASHES: usize = 5; /// Get the current Unix minute timestamp, optionally offset. /// @@ -35,6 +37,7 @@ pub const MAX_BOOTSTRAP_RECORDS: usize = 100; /// let now = unix_minute(0); /// let prev_minute = unix_minute(-1); /// ``` +#[doc(hidden)] pub fn unix_minute(minute_offset: i64) -> u64 { - ((chrono::Utc::now().timestamp() as f64 / 60.0f64).floor() as i64 + minute_offset) as u64 + ((chrono::Utc::now().timestamp() / 60) + minute_offset) as u64 } diff --git a/tests/test_gossip.rs b/tests/test_gossip.rs deleted file mode 100644 index 064ac19..0000000 --- a/tests/test_gossip.rs +++ /dev/null @@ -1,16 +0,0 @@ -use distributed_topic_tracker::TopicId; - -#[test] -fn test_topic_id_creation() { - let topic_id = TopicId::new("test-topic".to_string()); - assert_eq!(topic_id.raw(), "test-topic"); - assert_eq!(topic_id.hash().len(), 32); - - // Same input should produce same hash - let topic_id2 = TopicId::new("test-topic".to_string()); - assert_eq!(topic_id.hash(), topic_id2.hash()); - - // Different input should produce different hash - let topic_id3 = TopicId::new("different-topic".to_string()); - assert_ne!(topic_id.hash(), topic_id3.hash()); -} diff --git a/tests/tests.rs b/tests/tests.rs index 5d5cd17..3e8ee0e 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,33 +1,29 @@ -use std::str::FromStr; +use std::{str::FromStr, sync::Arc}; use distributed_topic_tracker::{ - DefaultSecretRotation, EncryptedRecord, GossipRecordContent, Record, RecordTopic, - RotationHandle, encryption_keypair, salt, signing_keypair, unix_minute, + AutoDiscoveryGossip, BubbleMergeConfig, Config, DefaultSecretRotation, EncryptedRecord, + GossipReceiver, GossipRecordContent, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, MergeConfig, + MessageOverlapMergeConfig, PublisherConfig, Record, RecordPublisher, RotationHandle, TopicId, + encryption_keypair, salt, signing_keypair, unix_minute, }; use mainline::SigningKey; +use tokio::sync::Barrier; #[test] fn test_record_serialization_roundtrip() { let signing_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = [2u8; 32]; - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign( - topic, - unix_minute, - node_id, - record_content.clone(), - &signing_key, - ) - .expect("Failed to sign record"); + let record = Record::sign(topic, unix_minute, record_content.clone(), &signing_key) + .expect("Failed to sign record"); // Test serialization roundtrip let bytes = record.to_bytes(); @@ -55,16 +51,15 @@ fn test_record_verification() { let signing_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = signing_key.verifying_key().to_bytes(); - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign(topic, unix_minute, node_id, record_content, &signing_key).unwrap(); + let record = Record::sign(topic, unix_minute, record_content, &signing_key).unwrap(); // Valid verification should pass assert!(record.verify(&topic, unix_minute).is_ok()); @@ -83,23 +78,16 @@ fn test_encrypted_record_roundtrip() { let encryption_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = signing_key.verifying_key().to_bytes(); - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign( - topic, - unix_minute, - node_id, - record_content.clone(), - &signing_key, - ) - .expect("Failed to sign record"); + let record = Record::sign(topic, unix_minute, record_content.clone(), &signing_key) + .expect("Failed to sign record"); // Test encryption/decryption roundtrip let encrypted = record.encrypt(&encryption_key); @@ -128,16 +116,15 @@ fn test_encrypted_record_serialization() { let encryption_key = SigningKey::generate(&mut rand::rng()); let topic = [1u8; 32]; let unix_minute = 12345u64; - let node_id = signing_key.verifying_key().to_bytes(); - let active_peers = [[3u8; 32]; 5]; - let last_message_hashes = [[4u8; 32]; 5]; + let active_peers = [[3u8; 32]; MAX_RECORD_PEERS]; + let last_message_hashes = [[4u8; 32]; MAX_MESSAGE_HASHES]; let record_content = GossipRecordContent { active_peers, last_message_hashes, }; - let record = Record::sign(topic, unix_minute, node_id, record_content, &signing_key) + let record = Record::sign(topic, unix_minute, record_content, &signing_key) .expect("Failed to sign record"); let encrypted = record.encrypt(&encryption_key); @@ -194,58 +181,206 @@ fn test_unix_minute_function() { #[test] fn test_topic_signing_keypair_deterministic() { - let topic_id = RecordTopic::from_str("test-topic").unwrap(); - let record_topic = topic_id; + let topic_id = TopicId::from_str("test-topic").unwrap(); let unix_minute = 12345u64; - let key1 = signing_keypair(record_topic, unix_minute); - let key2 = signing_keypair(record_topic, unix_minute); + let key1 = signing_keypair(&topic_id, unix_minute); + let key2 = signing_keypair(&topic_id, unix_minute); // Same inputs should produce same keypair assert_eq!(key1.to_bytes(), key2.to_bytes()); // Different unix_minute should produce different keypair - let key3 = signing_keypair(record_topic, unix_minute + 1); + let key3 = signing_keypair(&topic_id, unix_minute + 1); assert_ne!(key1.to_bytes(), key3.to_bytes()); } #[test] fn test_topic_encryption_keypair_deterministic() { - let topic_id = RecordTopic::from_str("test-topic").unwrap(); - let record_topic = topic_id; + let topic_id = TopicId::from_str("test-topic").unwrap(); let unix_minute = 12345u64; let initial_secret_hash = [1u8; 32]; let rotation = RotationHandle::new(DefaultSecretRotation); - let key1 = encryption_keypair(record_topic, &rotation, initial_secret_hash, unix_minute); - let key2 = encryption_keypair(record_topic, &rotation, initial_secret_hash, unix_minute); + let key1 = encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute); + let key2 = encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute); // Same inputs should produce same keypair assert_eq!(key1.to_bytes(), key2.to_bytes()); // Different unix_minute should produce different keypair - let key3 = crate::encryption_keypair( - record_topic, - &rotation, - initial_secret_hash, - unix_minute + 1, - ); + let key3 = + crate::encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute + 1); assert_ne!(key1.to_bytes(), key3.to_bytes()); } #[test] fn test_topic_salt_deterministic() { - let topic_id = RecordTopic::from_str("test-topic").unwrap(); - let record_topic = topic_id; + let topic_id = TopicId::from_str("test-topic").unwrap(); let unix_minute = 12345u64; - let salt1 = salt(record_topic, unix_minute); - let salt2 = salt(record_topic, unix_minute); + let salt1 = salt(&topic_id, unix_minute); + let salt2 = salt(&topic_id, unix_minute); // Same inputs should produce same salt assert_eq!(salt1, salt2); // Different unix_minute should produce different salt - let salt3 = salt(record_topic, unix_minute + 1); + let salt3 = salt(&topic_id, unix_minute + 1); assert_ne!(salt1, salt3); } + +#[test] +fn test_topic_id_creation() { + let topic_id = TopicId::new("test-topic".to_string()); + assert_eq!(topic_id.hash().len(), 32); + + // Same input should produce same hash + let topic_id2 = TopicId::new("test-topic".to_string()); + assert_eq!(topic_id.hash(), topic_id2.hash()); + + // Different input should produce different hash + let topic_id3 = TopicId::new("different-topic".to_string()); + assert_ne!(topic_id.hash(), topic_id3.hash()); +} + +#[cfg(feature = "iroh-gossip")] +#[tokio::test] +async fn test_multiple_receivers_all_get_events() { + const N: usize = 3; + const MSG_COUNT: usize = 3; + + let config = Config::builder() + .publisher_config(PublisherConfig::Disabled) + .merge_config(MergeConfig::new( + BubbleMergeConfig::Disabled, + MessageOverlapMergeConfig::Disabled, + )) + .build(); + + let topic_id = TopicId::new("test-multi-receiver".to_string()); + + // Peer A + let secret_a = iroh::SecretKey::generate(&mut rand::rng()); + let signing_a = mainline::SigningKey::from_bytes(&secret_a.to_bytes()); + let endpoint_a = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_a) + .bind() + .await + .unwrap(); + let gossip_a = iroh_gossip::net::Gossip::builder().spawn(endpoint_a.clone()); + let _router_a = iroh::protocol::Router::builder(endpoint_a.clone()) + .accept(iroh_gossip::ALPN, gossip_a.clone()) + .spawn(); + + let rp_a = RecordPublisher::new( + topic_id.clone(), + signing_a, + None, + b"secret".to_vec(), + config.clone(), + ); + + let topic_a = gossip_a + .subscribe_and_join_with_auto_discovery_no_wait(rp_a) + .await + .unwrap(); + let (sender_a, mut receiver_a) = topic_a.split().await.unwrap(); + + // Peer B + let secret_b = iroh::SecretKey::generate(&mut rand::rng()); + let signing_b = mainline::SigningKey::from_bytes(&secret_b.to_bytes()); + let endpoint_b = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_b) + .bind() + .await + .unwrap(); + let gossip_b = iroh_gossip::net::Gossip::builder().spawn(endpoint_b.clone()); + let _router_b = iroh::protocol::Router::builder(endpoint_b.clone()) + .accept(iroh_gossip::ALPN, gossip_b.clone()) + .spawn(); + + let rp_b = RecordPublisher::new( + topic_id.clone(), + signing_b, + None, + b"secret".to_vec(), + config, + ); + + let topic_b = gossip_b + .subscribe_and_join_with_auto_discovery(rp_b) + .await + .unwrap(); + let (sender_b, mut receiver_b) = topic_b.split().await.unwrap(); + + // Join peers + sender_a + .join_peers(vec![endpoint_b.id()], None) + .await + .unwrap(); + sender_b + .join_peers(vec![endpoint_a.id()], None) + .await + .unwrap(); + + receiver_a.joined().await.unwrap(); + receiver_b.joined().await.unwrap(); + + let receivers: Vec = (0..N) + .map(|_| receiver_b.clone()) + .chain((0..N).map(|_| receiver_a.clone())) + .collect(); + let barrier = Arc::new(Barrier::new(receivers.len() + 1)); + + let handles = receivers + .into_iter() + .enumerate() + .map(|(i, mut rx)| { + let barrier = barrier.clone(); + tokio::spawn(async move { + barrier.wait().await; + + let mut received = Vec::new(); + while received.len() < MSG_COUNT { + match tokio::time::timeout(std::time::Duration::from_secs(30), rx.next()).await + { + Ok(Some(iroh_gossip::api::Event::Received(msg))) => { + received.push(msg.content.to_vec()); + } + Ok(Some(_)) => continue, + other => panic!("receiver {i}: unexpected result: {other:?}"), + } + } + received + }) + }) + .collect::>(); + + barrier.wait().await; + + for i in 0..MSG_COUNT { + sender_a + .broadcast(format!("msg-a-{i}").into_bytes()) + .await + .unwrap(); + sender_b + .broadcast(format!("msg-b-{i}").into_bytes()) + .await + .unwrap(); + } + + for (i, handle) in handles.into_iter().enumerate() { + let received = tokio::time::timeout(std::time::Duration::from_secs(60), handle) + .await + .unwrap_or_else(|_| panic!("receiver {i} timed out")) + .unwrap(); + + assert_eq!( + received.len(), + MSG_COUNT, + "receiver {i} got {} messages, expected {MSG_COUNT}", + received.len(), + ); + } +} From 7b210c37e9954c5f9c9ae0ac90d9cb4046c339a5 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 04:09:27 +0200 Subject: [PATCH 11/30] fix: cargo test_gossip removed --- Cargo.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dc2e8c5..50ebaed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,10 +42,6 @@ serde = { version = "1", default-features = false, features = ["std"] } tracing = { version = "0.1", default-features = false, features = ["std"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "env-filter", "ansi"] } -[[test]] -name = "test_gossip" -required-features = ["iroh-gossip"] - [[test]] name = "tests" From 341ff8c2af6f3ba4deb93dce959cef8bf11a78c9 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 18:29:25 +0200 Subject: [PATCH 12/30] fix: next_interval across project min(1000) sec to prevent hot loops and reduced to sec -> ms + docs string fixes + receiver cancel on failure + some correctness fixes --- examples/full_config.rs | 2 +- examples/without_mergers.rs | 2 +- src/dht.rs | 16 +++++++++------- src/gossip/merge/bubble.rs | 6 +++--- src/gossip/merge/message_overlap.rs | 6 +++--- src/gossip/merge/mod.rs | 2 +- src/gossip/mod.rs | 4 ++-- src/gossip/receiver.rs | 2 ++ src/gossip/topic/publisher.rs | 6 +++--- src/gossip/topic/topic.rs | 8 +++----- tests/tests.rs | 3 +-- 11 files changed, 29 insertions(+), 28 deletions(-) diff --git a/examples/full_config.rs b/examples/full_config.rs index 4ad2b54..188ca7d 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -5,7 +5,7 @@ use ed25519_dalek::SigningKey; use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; -// Imports from distrubuted-topic-tracker +// Imports from distributed-topic-tracker use distributed_topic_tracker::{ AutoDiscoveryGossip, BootstrapConfig, BubbleMergeConfig, Config, DefaultSecretRotation, DhtConfig, MergeConfig, MessageOverlapMergeConfig, PublisherConfig, RecordPublisher, diff --git a/examples/without_mergers.rs b/examples/without_mergers.rs index 98fb9d2..28f4abd 100644 --- a/examples/without_mergers.rs +++ b/examples/without_mergers.rs @@ -3,7 +3,7 @@ use ed25519_dalek::SigningKey; use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; -// Imports from distrubuted-topic-tracker +// Imports from distributed-topic-tracker use distributed_topic_tracker::{AutoDiscoveryGossip, BubbleMergeConfig, Config, MergeConfig, MessageOverlapMergeConfig, RecordPublisher, TopicId}; #[tokio::main] diff --git a/src/dht.rs b/src/dht.rs index ae0ccf7..9b352ff 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -115,7 +115,7 @@ impl DhtActor { self.reset().await?; } - for i in 0..1+self.config.retries() { + for i in 0..1 + self.config.retries() { let dht = self.dht.as_mut().context("DHT not initialized")?; let most_recent_result = tokio::time::timeout( @@ -153,19 +153,21 @@ impl DhtActor { self.reset().await?; - let retry_interval = self.config.base_retry_interval() - + Duration::from_millis(if self.config.max_retry_jitter().as_millis() > 0 { - rand::random::() % self.config.max_retry_jitter().as_millis() as u64 + let retry_interval = (self.config.base_retry_interval().as_millis() + + if self.config.max_retry_jitter().as_millis() > 0 { + rand::random::() % self.config.max_retry_jitter().as_millis() } else { 0 - }); + }) + .min(1000); + tracing::debug!( - "DHTActor: put_mutable attempt {}/{} failed, retrying in {:?}", + "DHTActor: put_mutable attempt {}/{} failed, retrying in {}ms", i + 1, 1 + self.config.retries(), retry_interval ); - tokio::time::sleep(retry_interval).await; + tokio::time::sleep(Duration::from_millis(retry_interval as u64)).await; } Ok(()) } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index a4f048b..0872ac9 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -83,9 +83,9 @@ impl BubbleMergeActor { _ = self.ticker.tick() => { tracing::debug!("BubbleMerge: tick fired, checking for bubbles"); let _ = self.merge().await; - let next_interval = self.base_interval.as_secs() + if self.max_jitter.as_secs() > 0 { rand::random::() % self.max_jitter.as_secs() } else {0}; - tracing::debug!("BubbleMerge: next check in {}s", next_interval); - self.ticker.reset_after(Duration::from_secs(next_interval)); + let next_interval = (self.base_interval.as_millis() + if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { 0 }).min(1000); + tracing::debug!("BubbleMerge: next check in {}ms", next_interval); + self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } _ = self.cancel_token.cancelled() => { break Ok(()); diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 66dea54..4f0deb7 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -74,9 +74,9 @@ impl MessageOverlapMergeActor { _ = self.ticker.tick() => { tracing::debug!("MessageOverlapMerge: tick fired, checking for split-brain"); let _ = self.merge().await; - let next_interval = self.base_interval.as_secs() + if self.max_jitter.as_secs() > 0 { rand::random::() % self.max_jitter.as_secs() } else {0}; - tracing::debug!("MessageOverlapMerge: next check in {}s", next_interval); - self.ticker.reset_after(Duration::from_secs(next_interval)); + let next_interval = (self.base_interval.as_millis() + if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { 0 }).min(1000); + tracing::debug!("MessageOverlapMerge: next check in {}ms", next_interval); + self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } _ = self.cancel_token.cancelled() => { break Ok(()); diff --git a/src/gossip/merge/mod.rs b/src/gossip/merge/mod.rs index 87052c6..9e66f5d 100644 --- a/src/gossip/merge/mod.rs +++ b/src/gossip/merge/mod.rs @@ -1,7 +1,7 @@ //! Peer merging strategies for recovering from network partitions. //! //! Two complementary strategies detect and heal split-brain scenarios: -//! - **Bubble Merge**: Joins small clusters (< 4 peers) with peers advertised in DHT +//! - **Bubble Merge**: Joins small clusters (< BubbleMergeConfig.min_neighbors() peers) with peers advertised in DHT //! - **Message Overlap**: Detects when isolated clusters share common message hashes, //! indicating they have seen the same messages and can be merged diff --git a/src/gossip/mod.rs b/src/gossip/mod.rs index d3740ba..15c0ea2 100644 --- a/src/gossip/mod.rs +++ b/src/gossip/mod.rs @@ -23,9 +23,9 @@ use crate::{MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher}; /// and recently seen message hashes for cluster merging. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GossipRecordContent { - /// Fixed array of 5 peer node IDs (as 32-byte arrays, empty slots are zero-filled) + /// Fixed array of MAX_RECORD_PEERS peer node IDs (as 32-byte arrays, empty slots are zero-filled) pub active_peers: [[u8; 32]; MAX_RECORD_PEERS], - /// Fixed array of 5 recent message hashes for overlap detection (empty slots are zero-filled) + /// Fixed array of MAX_MESSAGE_HASHES recent message hashes for overlap detection (empty slots are zero-filled) pub last_message_hashes: [[u8; 32]; MAX_MESSAGE_HASHES], } diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 04c4e57..e2b33b2 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -161,12 +161,14 @@ impl GossipReceiverActor { tracing::debug!("GossipReceiver: gossip receiver closed"); self.join_channel_sender.send(None).ok(); self.next_channel_sender.send(None).ok(); + self.cancel_token.cancel(); break Ok(()); } Some(Err(err)) => { tracing::warn!("GossipReceiver: error receiving gossip event: {err}"); self.next_channel_sender.send(None).ok(); self.join_channel_sender.send(None).ok(); + self.cancel_token.cancel(); break Ok(()); } Some(Ok(ref event)) => { diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 2e2d5aa..3e2d615 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -71,9 +71,9 @@ impl PublisherActor { _ = self.ticker.tick() => { tracing::debug!("Publisher: tick fired, attempting to publish"); let _ = self.publish().await; - let next_interval = self.base_interval.as_secs() + if self.max_jitter.as_secs() > 0 { rand::random::() % self.max_jitter.as_secs() } else {0}; - tracing::debug!("Publisher: next publish in {}s", next_interval); - self.ticker.reset_after(Duration::from_secs(next_interval)); + let next_interval = (self.base_interval.as_millis() + if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { 0 }).min(1000); + tracing::debug!("Publisher: next publish in {}ms", next_interval); + self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } _ = self.cancel_token.cancelled() => { break Ok(()); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 4bd5ff3..7f9e060 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -140,7 +140,7 @@ impl Topic { PublisherConfig::Enabled { .. } ) { tracing::debug!("Topic: starting publisher"); - let _ = api.call(act!(actor => actor.start_publishing())).await; + api.call(act!(actor => actor.start_publishing())).await?; } if matches!( @@ -148,7 +148,7 @@ impl Topic { BubbleMergeConfig::Enabled { .. } ) { tracing::debug!("Topic: starting bubble merge"); - let _ = api.call(act!(actor => actor.start_bubble_merge())).await; + api.call(act!(actor => actor.start_bubble_merge())).await?; } if matches!( @@ -159,9 +159,7 @@ impl Topic { MessageOverlapMergeConfig::Enabled { .. } ) { tracing::debug!("Topic: starting message overlap merge"); - let _ = api - .call(act!(actor => actor.start_message_overlap_merge())) - .await; + api.call(act!(actor => actor.start_message_overlap_merge())).await?; } tracing::debug!("Topic: fully initialized"); diff --git a/tests/tests.rs b/tests/tests.rs index 3e8ee0e..d0d5fbf 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -209,8 +209,7 @@ fn test_topic_encryption_keypair_deterministic() { assert_eq!(key1.to_bytes(), key2.to_bytes()); // Different unix_minute should produce different keypair - let key3 = - crate::encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute + 1); + let key3 = encryption_keypair(&topic_id, &rotation, initial_secret_hash, unix_minute + 1); assert_ne!(key1.to_bytes(), key3.to_bytes()); } From 24c8bb5648615d88204391ca76b7efbdc6646dbd Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 19:27:26 +0200 Subject: [PATCH 13/30] fix: review fixes --- examples/full_config.rs | 2 +- examples/without_mergers.rs | 3 +-- src/config.rs | 8 +++++--- src/crypto/keys.rs | 4 ++-- src/crypto/record.rs | 2 +- src/dht.rs | 15 +++++++-------- src/gossip/merge/bubble.rs | 7 ++++++- src/gossip/merge/message_overlap.rs | 7 ++++++- src/gossip/topic/bootstrap.rs | 6 +++--- src/gossip/topic/publisher.rs | 7 ++++++- src/lib.rs | 8 +++++--- 11 files changed, 43 insertions(+), 26 deletions(-) diff --git a/examples/full_config.rs b/examples/full_config.rs index 188ca7d..09c66e1 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -85,13 +85,13 @@ async fn main() -> Result<()> { let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - // Split into sink (sending) and stream (receiving) let record_publisher = RecordPublisher::builder(topic_id.clone(), signing_key.clone(), initial_secret) .config(config_builder()) .secret_rotation(RotationHandle::new(DefaultSecretRotation)) .build(); + // Split into sink (sending) and stream (receiving) let topic = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await?; diff --git a/examples/without_mergers.rs b/examples/without_mergers.rs index 28f4abd..7b27dd4 100644 --- a/examples/without_mergers.rs +++ b/examples/without_mergers.rs @@ -29,8 +29,6 @@ async fn main() -> Result<()> { let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - // Split into sink (sending) and stream (receiving) - let record_publisher = RecordPublisher::new( topic_id.clone(), signing_key.clone(), @@ -41,6 +39,7 @@ async fn main() -> Result<()> { Config::builder().merge_config(MergeConfig::new(BubbleMergeConfig::Disabled, MessageOverlapMergeConfig::Disabled)).build(), ); + // Split into sink (sending) and stream (receiving) let topic = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await?; diff --git a/src/config.rs b/src/config.rs index 81cdc2d..c16d00f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -164,6 +164,7 @@ impl Default for DhtConfig { } } +/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. #[derive(Debug, Clone)] pub enum BubbleMergeConfig { Enabled { @@ -174,6 +175,7 @@ pub enum BubbleMergeConfig { Disabled, } +/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. #[derive(Debug, Clone)] pub enum MessageOverlapMergeConfig { Enabled { @@ -193,7 +195,7 @@ pub enum PublisherConfig { Disabled, } -/// total_interval = base_interval + rand::random::() % max_jitter; +/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. #[derive(Debug, Clone)] pub struct MergeConfig { bubble_merge: BubbleMergeConfig, @@ -264,7 +266,7 @@ pub struct BootstrapConfigBuilder { impl BootstrapConfigBuilder { /// Max bootstrap records per topic per minute slot (min 1). Default: 5. pub fn max_bootstrap_records(mut self, max_records: usize) -> Self { - self.config.max_bootstrap_records = max_records; + self.config.max_bootstrap_records = max_records.max(1); self } @@ -446,7 +448,7 @@ impl ConfigBuilder { /// Max peers to join simultaneously (min 1). Default: 4. pub fn max_join_peer_count(mut self, max_peers: usize) -> Self { - self.config.max_join_peer_count = max_peers; + self.config.max_join_peer_count = max_peers.max(1); self } diff --git a/src/crypto/keys.rs b/src/crypto/keys.rs index adb1659..79e3855 100644 --- a/src/crypto/keys.rs +++ b/src/crypto/keys.rs @@ -98,7 +98,7 @@ impl RotationHandle { /// ```ignore /// let topic = TopicId::from_str("my-topic")?; /// let unix_minute = crate::unix_minute(0); -/// let signing_key = signing_keypair(topic, unix_minute); +/// let signing_key = signing_keypair(&topic, unix_minute); /// ``` pub fn signing_keypair(topic_id: &TopicId, unix_minute: u64) -> ed25519_dalek::SigningKey { let mut sign_keypair_hash = sha2::Sha512::new(); @@ -119,7 +119,7 @@ pub fn signing_keypair(topic_id: &TopicId, unix_minute: u64) -> ed25519_dalek::S /// ```ignore /// let topic = TopicId::from_str("my-topic")?; /// let rotation = RotationHandle::default(); -/// let enc_key = encryption_keypair(topic, &rotation, initial_hash, 0); +/// let enc_key = encryption_keypair(&topic, &rotation, initial_hash, 0); /// ``` pub fn encryption_keypair( topic_id: &TopicId, diff --git a/src/crypto/record.rs b/src/crypto/record.rs index fca18c4..38e08b5 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -390,7 +390,7 @@ impl Record { /// Deserialize from bytes. pub fn from_bytes(buf: Vec) -> Result { - if buf.len() < 32 + 8 + 32 + 64 + 1 { + if buf.len() < 32 + 8 + 32 + 64 { bail!("buffer too short for Record deserialization") } let (topic, buf) = buf.split_at(32); diff --git a/src/dht.rs b/src/dht.rs index 9b352ff..f1058cc 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -153,14 +153,13 @@ impl DhtActor { self.reset().await?; - let retry_interval = (self.config.base_retry_interval().as_millis() - + if self.config.max_retry_jitter().as_millis() > 0 { - rand::random::() % self.config.max_retry_jitter().as_millis() - } else { - 0 - }) - .min(1000); - + let jitter_ms = if self.config.max_retry_jitter().as_millis() > 0 { + rand::random::() % self.config.max_retry_jitter().as_millis() + } else { + 0 + }; + let retry_interval = (self.config.base_retry_interval().as_millis() + jitter_ms).max(1000); + tracing::debug!( "DHTActor: put_mutable attempt {}/{} failed, retrying in {}ms", i + 1, diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 0872ac9..e7a953c 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -83,7 +83,12 @@ impl BubbleMergeActor { _ = self.ticker.tick() => { tracing::debug!("BubbleMerge: tick fired, checking for bubbles"); let _ = self.merge().await; - let next_interval = (self.base_interval.as_millis() + if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { 0 }).min(1000); + let jitter_ms = if self.max_jitter.as_millis() > 0 { + rand::random::() % self.max_jitter.as_millis() + } else { + 0 + }; + let next_interval = (self.base_interval.as_millis() + jitter_ms).max(1000); tracing::debug!("BubbleMerge: next check in {}ms", next_interval); self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 4f0deb7..f23d8c0 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -74,7 +74,12 @@ impl MessageOverlapMergeActor { _ = self.ticker.tick() => { tracing::debug!("MessageOverlapMerge: tick fired, checking for split-brain"); let _ = self.merge().await; - let next_interval = (self.base_interval.as_millis() + if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { 0 }).min(1000); + let jitter_ms = if self.max_jitter.as_millis() > 0 { + rand::random::() % self.max_jitter.as_millis() + } else { + 0 + }; + let next_interval = (self.base_interval.as_millis() + jitter_ms).max(1000); tracing::debug!("MessageOverlapMerge: next check in {}ms", next_interval); self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index a77a530..029508d 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -112,7 +112,7 @@ impl BootstrapActor { record_publisher.topic_id().hash(), unix_minute, record_content, - &record_publisher.signing_key(), + record_publisher.signing_key(), ) { tokio::spawn(async move { let _ = record_creator.publish_record(record).await; @@ -171,7 +171,7 @@ impl BootstrapActor { record_publisher.topic_id().hash(), unix_minute, record_content, - &record_publisher.signing_key(), + record_publisher.signing_key(), ) { tokio::spawn(async move { let _ = record_creator.publish_record(record).await; @@ -314,7 +314,7 @@ impl BootstrapActor { active_peers: [[0; 32]; MAX_RECORD_PEERS], last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], }, - &record_publisher.signing_key(), + record_publisher.signing_key(), ) { tokio::spawn(async move { let _ = record_creator.publish_record(record).await; diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 3e2d615..af03a70 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -71,7 +71,12 @@ impl PublisherActor { _ = self.ticker.tick() => { tracing::debug!("Publisher: tick fired, attempting to publish"); let _ = self.publish().await; - let next_interval = (self.base_interval.as_millis() + if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { 0 }).min(1000); + let jitter_ms = if self.max_jitter.as_millis() > 0 { + rand::random::() % self.max_jitter.as_millis() + } else { + 0 + }; + let next_interval = (self.base_interval.as_millis() + jitter_ms).max(1000); tracing::debug!("Publisher: next publish in {}ms", next_interval); self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } diff --git a/src/lib.rs b/src/lib.rs index 555748f..1277556 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,8 +13,8 @@ pub use gossip::{ }; pub use config::{ - BubbleMergeConfig, Config, ConfigBuilder, DhtConfig, MergeConfig, MessageOverlapMergeConfig, - TimeoutConfig, TimeoutConfigBuilder, PublisherConfig, BootstrapConfig, + BootstrapConfig, BubbleMergeConfig, Config, ConfigBuilder, DhtConfig, MergeConfig, + MessageOverlapMergeConfig, PublisherConfig, TimeoutConfig, TimeoutConfigBuilder, }; pub use crypto::{ DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RotationHandle, @@ -39,5 +39,7 @@ pub const MAX_MESSAGE_HASHES: usize = 5; /// ``` #[doc(hidden)] pub fn unix_minute(minute_offset: i64) -> u64 { - ((chrono::Utc::now().timestamp() / 60) + minute_offset) as u64 + ((chrono::Utc::now().timestamp() / 60) + minute_offset) + .try_into() + .expect("timestamp overflow") } From 66158a547cefaf9507f31ef3c8430734cac7bf04 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 22:11:17 +0200 Subject: [PATCH 14/30] fix: explicitly set bootstrap first record unix_minute with config --- examples/full_config.rs | 1 + src/config.rs | 19 +++++++++++++++++++ src/gossip/topic/bootstrap.rs | 21 +++++++++++++++------ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/examples/full_config.rs b/examples/full_config.rs index 09c66e1..78e2985 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -29,6 +29,7 @@ fn config_builder() -> Config { BootstrapConfig::builder() .max_bootstrap_records(5) .publish_record_on_startup(true) + .check_last_minute_record_first_on_startup(false) .discovery_poll_interval(Duration::from_millis(2000)) .no_peers_retry_interval(Duration::from_millis(1500)) .per_peer_join_settle_time(Duration::from_millis(100)) diff --git a/src/config.rs b/src/config.rs index c16d00f..c6693e5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -243,6 +243,7 @@ pub struct BootstrapConfig { join_confirmation_wait_time: Duration, discovery_poll_interval: Duration, publish_record_on_startup: bool, + check_last_minute_record_first_on_startup: bool, } impl Default for BootstrapConfig { @@ -254,6 +255,7 @@ impl Default for BootstrapConfig { join_confirmation_wait_time: Duration::from_millis(500), discovery_poll_interval: Duration::from_millis(2000), publish_record_on_startup: true, + check_last_minute_record_first_on_startup: false, } } } @@ -300,6 +302,15 @@ impl BootstrapConfigBuilder { self } + /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup. Default: false. + /// + /// If joining longer running, existing topics is priority, set to true. + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. + pub fn check_last_minute_record_first_on_startup(mut self, check: bool) -> Self { + self.config.check_last_minute_record_first_on_startup = check; + self + } + pub fn build(self) -> BootstrapConfig { self.config } @@ -341,6 +352,14 @@ impl BootstrapConfig { pub fn publish_record_on_startup(&self) -> bool { self.publish_record_on_startup } + + /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup. Default: false. + /// + /// If joining longer running, existing topics is priority, set to true. + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. + pub fn check_last_minute_record_first_on_startup(&self) -> bool { + self.check_last_minute_record_first_on_startup + } } #[derive(Debug, Clone)] diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 029508d..ae9e502 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -102,7 +102,12 @@ impl BootstrapActor { if self.config.publish_record_on_startup() { let unix_minute = crate::unix_minute(0); tracing::debug!("Bootstrap: initial startup record publish {}", unix_minute); - last_published_unix_minute = unix_minute; + last_published_unix_minute = + if self.config.check_last_minute_record_first_on_startup() { + 0 + } else { + unix_minute + }; let record_creator = record_publisher.clone(); let record_content = GossipRecordContent { active_peers: [[0; 32]; MAX_RECORD_PEERS], @@ -137,11 +142,15 @@ impl BootstrapActor { } // On the first try we check the prev unix minute, after that the current one - let unix_minute = crate::unix_minute(if last_published_unix_minute == 0 { - -1 - } else { - 0 - }); + let unix_minute = crate::unix_minute( + if last_published_unix_minute == 0 + && bootstrap_config.check_last_minute_record_first_on_startup() + { + -1 + } else { + 0 + }, + ); // Unique, verified records for the unix minute let mut records = record_publisher.get_records(unix_minute - 1).await; From 44163f8dd8f9cd1e44a056e040d6048696c61db4 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 22:57:13 +0200 Subject: [PATCH 15/30] fix: next and join sender reference counting + changed base_interval to a minimum of 1 sec, could remove .max(1000) call on combined (base_interval + jitter).max(1000ms) call since base_interval > 1000ms + initial_delay now enforced --- README.md | 6 +-- src/config.rs | 11 +++++ src/gossip/merge/bubble.rs | 6 +-- src/gossip/merge/message_overlap.rs | 6 +-- src/gossip/receiver.rs | 51 ++++++++++++-------- src/gossip/topic/publisher.rs | 9 ++-- src/gossip/topic/topic.rs | 74 ++++++++++++++++++++++++++++- 7 files changed, 131 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5142d44..f076710 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ distributed-topic-tracker = "0.2" ``` Basic iroh-gossip integration: -```rust,no_run,ignore +```rust,no_run use anyhow::Result; use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; use ed25519_dalek::SigningKey; // Imports from distributed-topic-tracker -use distributed_topic_tracker::{TopicId, AutoDiscoveryGossip, RecordPublisher}; +use distributed_topic_tracker::{TopicId, AutoDiscoveryGossip, RecordPublisher, Config}; #[tokio::main] async fn main() -> Result<()> { @@ -55,10 +55,10 @@ async fn main() -> Result<()> { let initial_secret = b"my-initial-secret".to_vec(); let record_publisher = RecordPublisher::new( topic_id.clone(), - signing_key.verifying_key(), signing_key.clone(), None, initial_secret, + Config::default(), ); // Use new `subscribe_and_join_with_auto_discovery` on Gossip diff --git a/src/config.rs b/src/config.rs index c6693e5..bab3278 100644 --- a/src/config.rs +++ b/src/config.rs @@ -165,6 +165,7 @@ impl Default for DhtConfig { } /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. +/// base_interval is minimum 1s, max_jitter is minimum 0s. Default: enabled, 60s base, 120s jitter, 4 min neighbors. #[derive(Debug, Clone)] pub enum BubbleMergeConfig { Enabled { @@ -176,6 +177,7 @@ pub enum BubbleMergeConfig { } /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. +/// base_interval is minimum 1s, max_jitter is minimum 0s. Default: enabled, 60s base, 120s jitter. #[derive(Debug, Clone)] pub enum MessageOverlapMergeConfig { Enabled { @@ -185,6 +187,8 @@ pub enum MessageOverlapMergeConfig { Disabled, } +/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. +/// base_interval is minimum 1s, max_jitter is minimum 0s. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. #[derive(Debug, Clone)] pub enum PublisherConfig { Enabled { @@ -196,6 +200,7 @@ pub enum PublisherConfig { } /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. +/// base_interval is minimum 1s, max_jitter is minimum 0s. #[derive(Debug, Clone)] pub struct MergeConfig { bubble_merge: BubbleMergeConfig, @@ -214,6 +219,8 @@ impl Default for PublisherConfig { impl MergeConfig { /// Defaults: bubble_merge=Enabled(60s base, 120s jitter, 4 min neighbors), message_overlap_merge=Enabled(60s base, 120s jitter). + /// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes for message overlap detection and merging. + /// base_interval is minimum 1s, max_jitter is minimum 0s. pub fn new( bubble_merge: BubbleMergeConfig, message_overlap_merge: MessageOverlapMergeConfig, @@ -225,11 +232,13 @@ impl MergeConfig { } /// Bubble merge strategy config. Default: enabled, 60s base, 120s jitter, 4 min neighbors. + /// base_interval is minimum 1s, max_jitter is minimum 0s. pub fn bubble_merge(&self) -> &BubbleMergeConfig { &self.bubble_merge } /// Message overlap merge strategy config. Default: enabled, 60s base, 120s jitter. + /// base_interval is minimum 1s, max_jitter is minimum 0s. pub fn message_overlap_merge(&self) -> &MessageOverlapMergeConfig { &self.message_overlap_merge } @@ -382,6 +391,7 @@ impl Config { } /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. + /// base_interval is minimum 1s, max_jitter is minimum 0s. pub fn publisher_config(&self) -> &PublisherConfig { &self.publisher_config } @@ -448,6 +458,7 @@ impl ConfigBuilder { } /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. + /// base_interval and initial_delay minimum is 1s, max_jitter is minimum 0s. pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { self.config.publisher_config = publisher_config; self diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index e7a953c..d2a93ef 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -46,7 +46,7 @@ impl BubbleMerge { max_jitter: Duration, min_neighbors: usize, ) -> Result { - let mut ticker = tokio::time::interval(base_interval); + let mut ticker = tokio::time::interval(base_interval.max(Duration::from_secs(1))); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -57,7 +57,7 @@ impl BubbleMerge { ticker, cancel_token, max_join_peers, - base_interval, + base_interval: base_interval.max(Duration::from_secs(1)), max_jitter, min_neighbors, }, @@ -88,7 +88,7 @@ impl BubbleMergeActor { } else { 0 }; - let next_interval = (self.base_interval.as_millis() + jitter_ms).max(1000); + let next_interval = self.base_interval.as_millis() + jitter_ms; tracing::debug!("BubbleMerge: next check in {}ms", next_interval); self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index f23d8c0..bb4a46b 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -39,7 +39,7 @@ impl MessageOverlapMerge { base_interval: Duration, max_jitter: Duration, ) -> Result { - let mut ticker = tokio::time::interval(base_interval); + let mut ticker = tokio::time::interval(base_interval.max(Duration::from_secs(1))); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -50,7 +50,7 @@ impl MessageOverlapMerge { ticker, cancel_token, max_join_peers, - base_interval, + base_interval: base_interval.max(Duration::from_secs(1)), max_jitter, }, |mut actor, rx| async move { actor.run(rx).await }, @@ -79,7 +79,7 @@ impl MessageOverlapMergeActor { } else { 0 }; - let next_interval = (self.base_interval.as_millis() + jitter_ms).max(1000); + let next_interval = self.base_interval.as_millis() + jitter_ms; tracing::debug!("MessageOverlapMerge: next check in {}ms", next_interval); self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index e2b33b2..17e0c09 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -21,24 +21,37 @@ use crate::{MAX_MESSAGE_HASHES, Topic}; pub struct GossipReceiver { api: Handle, pub(crate) _topic_keep_alive: Option>, - _next_channel_sender: tokio::sync::broadcast::Sender< - Option>, - next_channel_receiver: tokio::sync::broadcast::Receiver< - Option, - >, - _join_channel_sender: tokio::sync::broadcast::Sender>, + _next_channel_sender: tokio::sync::broadcast::WeakSender>, + next_channel_receiver: tokio::sync::broadcast::Receiver>, + _join_channel_sender: tokio::sync::broadcast::WeakSender>, join_channel_receiver: tokio::sync::broadcast::Receiver>, } impl Clone for GossipReceiver { fn clone(&self) -> Self { + let next_rx = match self._next_channel_sender.upgrade() { + Some(sender) => sender.subscribe(), + None => { + let (tx, rx) = tokio::sync::broadcast::channel(1); + drop(tx); + rx + } + }; + let join_rx = match self._join_channel_sender.upgrade() { + Some(sender) => sender.subscribe(), + None => { + let (tx, rx) = tokio::sync::broadcast::channel(1); + drop(tx); + rx + } + }; Self { api: self.api.clone(), _topic_keep_alive: self._topic_keep_alive.clone(), _next_channel_sender: self._next_channel_sender.clone(), - next_channel_receiver: self._next_channel_sender.subscribe(), + next_channel_receiver: next_rx, _join_channel_sender: self._join_channel_sender.clone(), - join_channel_receiver: self._join_channel_sender.subscribe(), + join_channel_receiver: join_rx, } } } @@ -48,9 +61,7 @@ pub struct GossipReceiverActor { gossip_receiver: iroh_gossip::api::GossipReceiver, last_message_hashes: VecDeque<[u8; 32]>, cancel_token: tokio_util::sync::CancellationToken, - next_channel_sender: tokio::sync::broadcast::Sender< - Option, - >, + next_channel_sender: tokio::sync::broadcast::Sender>, join_channel_sender: tokio::sync::broadcast::Sender>, } @@ -78,9 +89,9 @@ impl GossipReceiver { api, _topic_keep_alive: None, next_channel_receiver: next_rx, - _next_channel_sender: next_tx, + _next_channel_sender: next_tx.downgrade(), join_channel_receiver: join_rx, - _join_channel_sender: join_tx, + _join_channel_sender: join_tx.downgrade(), } } @@ -103,15 +114,15 @@ impl GossipReceiver { /// Receive the next gossip event. /// /// Returns `None` if the receiver is closed. - pub async fn next( - &mut self, - ) -> Option { + pub async fn next(&mut self) -> Option { match self.next_channel_receiver.recv().await { Ok(event) => event, Err(err) => match err { tokio::sync::broadcast::error::RecvError::Closed => None, tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { - tracing::warn!("GossipReceiver: event stream lagged, {skipped} events may have been missed"); + tracing::warn!( + "GossipReceiver: event stream lagged, {skipped} events may have been missed" + ); Box::pin(self.next()).await } }, @@ -127,7 +138,9 @@ impl GossipReceiver { Err(err) => match err { tokio::sync::broadcast::error::RecvError::Closed => None, tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { - tracing::warn!("GossipReceiver: join event stream lagged, {skipped} events may have been missed"); + tracing::warn!( + "GossipReceiver: join event stream lagged, {skipped} events may have been missed" + ); Box::pin(self.joined()).await } }, @@ -211,4 +224,4 @@ impl GossipReceiverActor { } } } -} \ No newline at end of file +} diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index af03a70..1394fd0 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -38,7 +38,10 @@ impl Publisher { base_interval: Duration, max_jitter: Duration, ) -> Result { - let mut ticker = tokio::time::interval(initial_delay); + let mut ticker = tokio::time::interval_at( + tokio::time::Instant::now() + initial_delay, + base_interval.max(Duration::from_secs(1)), + ); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( PublisherActor { @@ -46,7 +49,7 @@ impl Publisher { gossip_receiver, ticker, cancel_token, - base_interval, + base_interval: base_interval.max(Duration::from_secs(1)), max_jitter, }, |mut actor, rx| async move { actor.run(rx).await }, @@ -76,7 +79,7 @@ impl PublisherActor { } else { 0 }; - let next_interval = (self.base_interval.as_millis() + jitter_ms).max(1000); + let next_interval = self.base_interval.as_millis() + jitter_ms; tracing::debug!("Publisher: next publish in {}ms", next_interval); self.ticker.reset_after(Duration::from_millis(next_interval as u64)); } diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 7f9e060..c4f7b38 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -159,7 +159,8 @@ impl Topic { MessageOverlapMergeConfig::Enabled { .. } ) { tracing::debug!("Topic: starting message overlap merge"); - api.call(act!(actor => actor.start_message_overlap_merge())).await?; + api.call(act!(actor => actor.start_message_overlap_merge())) + .await?; } tracing::debug!("Topic: fully initialized"); @@ -279,6 +280,77 @@ impl TopicActor { #[cfg(test)] mod tests { + #[tokio::test] + async fn test_receiver_returns_none_after_shutdown() { + let secret_key = iroh::SecretKey::generate(&mut rand::rng()); + let signing_key = mainline::SigningKey::from_bytes(&secret_key.to_bytes()); + let endpoint = iroh::Endpoint::builder(iroh::endpoint::presets::N0) + .secret_key(secret_key.clone()) + .bind() + .await + .unwrap(); + let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); + + let topic_id = crate::TopicId::new("shutdown-receiver-test".to_string()); + let initial_secret = b"my-initial-secret".to_vec(); + + let record_publisher = crate::RecordPublisher::new( + topic_id.clone(), + signing_key.clone(), + None, + initial_secret, + crate::config::Config::default(), + ); + + let topic = crate::Topic::new(record_publisher, gossip.clone(), true) + .await + .unwrap(); + + let cancel_token = topic.cancel_token(); + let (_sender, receiver) = topic.split().await.unwrap(); + + // Clone the receiver before shutdown, this survivor must not hang + let mut survivor = receiver.clone(); + + cancel_token.cancel(); + + // next() on a receiver that was alive before shutdown must return None, + // not hang. If the broadcast channel didn't close, this would block forever + let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.next()) + .await + .expect("next() hung after shutdown — broadcast channel didn't close"); + assert!(result.is_none(), "expected None from next() after shutdown"); + + // joined() must also return None after shutdown + let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.joined()) + .await + .expect("joined() hung after shutdown — broadcast channel didn't close"); + assert!( + result.is_none(), + "expected None from joined() after shutdown" + ); + + // A clone made after shutdown must also return None immediately + // (WeakSender::upgrade fails -> gets an already closed channel) + let mut late_clone = survivor.clone(); + + let result = tokio::time::timeout(std::time::Duration::from_secs(5), late_clone.next()) + .await + .expect("next() hung on post shutdown clone, WeakSender upgrade should fail"); + assert!( + result.is_none(), + "expected None from next() on post shutdown clone" + ); + + let result = tokio::time::timeout(std::time::Duration::from_secs(5), late_clone.joined()) + .await + .expect("joined() hung on post shutdown clone, WeakSender upgrade should fail"); + assert!( + result.is_none(), + "expected None from joined() on post shutdown clone" + ); + } + #[tokio::test] async fn test_topic_full_shutdown_on_drop() { let secret_key = iroh::SecretKey::generate(&mut rand::rng()); From a9d0ce152370a3ed5a46a1f5eacb6bdef5bb554e Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Thu, 16 Apr 2026 23:12:27 +0200 Subject: [PATCH 16/30] chore: review cleanup --- src/gossip/merge/bubble.rs | 5 ++-- src/gossip/merge/message_overlap.rs | 5 ++-- src/gossip/receiver.rs | 38 ++++++++++++++--------------- src/gossip/topic/publisher.rs | 9 +++---- 4 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index d2a93ef..5c97e1f 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -46,7 +46,8 @@ impl BubbleMerge { max_jitter: Duration, min_neighbors: usize, ) -> Result { - let mut ticker = tokio::time::interval(base_interval.max(Duration::from_secs(1))); + let base_interval = base_interval.max(Duration::from_secs(1)); + let mut ticker = tokio::time::interval(base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -57,7 +58,7 @@ impl BubbleMerge { ticker, cancel_token, max_join_peers, - base_interval: base_interval.max(Duration::from_secs(1)), + base_interval, max_jitter, min_neighbors, }, diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index bb4a46b..e17f31e 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -39,7 +39,8 @@ impl MessageOverlapMerge { base_interval: Duration, max_jitter: Duration, ) -> Result { - let mut ticker = tokio::time::interval(base_interval.max(Duration::from_secs(1))); + let base_interval = base_interval.max(Duration::from_secs(1)); + let mut ticker = tokio::time::interval(base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( @@ -50,7 +51,7 @@ impl MessageOverlapMerge { ticker, cancel_token, max_join_peers, - base_interval: base_interval.max(Duration::from_secs(1)), + base_interval, max_jitter, }, |mut actor, rx| async move { actor.run(rx).await }, diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 17e0c09..dbcdb8a 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -115,17 +115,16 @@ impl GossipReceiver { /// /// Returns `None` if the receiver is closed. pub async fn next(&mut self) -> Option { - match self.next_channel_receiver.recv().await { - Ok(event) => event, - Err(err) => match err { - tokio::sync::broadcast::error::RecvError::Closed => None, - tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { - tracing::warn!( + loop { + match self.next_channel_receiver.recv().await { + Ok(event) => return event, + Err(err) => match err { + tokio::sync::broadcast::error::RecvError::Closed => return None, + tokio::sync::broadcast::error::RecvError::Lagged(skipped) => tracing::warn!( "GossipReceiver: event stream lagged, {skipped} events may have been missed" - ); - Box::pin(self.next()).await - } - }, + ), + }, + } } } @@ -133,17 +132,16 @@ impl GossipReceiver { if self.is_joined().await.unwrap_or(false) { return Some(()); } - match self.join_channel_receiver.recv().await { - Ok(event) => event, - Err(err) => match err { - tokio::sync::broadcast::error::RecvError::Closed => None, - tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { - tracing::warn!( + loop { + match self.join_channel_receiver.recv().await { + Ok(event) => return event, + Err(err) => match err { + tokio::sync::broadcast::error::RecvError::Closed => return None, + tokio::sync::broadcast::error::RecvError::Lagged(skipped) => tracing::warn!( "GossipReceiver: join event stream lagged, {skipped} events may have been missed" - ); - Box::pin(self.joined()).await - } - }, + ), + }, + } } } diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 1394fd0..880c0e0 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -38,10 +38,9 @@ impl Publisher { base_interval: Duration, max_jitter: Duration, ) -> Result { - let mut ticker = tokio::time::interval_at( - tokio::time::Instant::now() + initial_delay, - base_interval.max(Duration::from_secs(1)), - ); + let base_interval = base_interval.max(Duration::from_secs(1)); + let mut ticker = + tokio::time::interval_at(tokio::time::Instant::now() + initial_delay, base_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); let api = Handle::spawn_with( PublisherActor { @@ -49,7 +48,7 @@ impl Publisher { gossip_receiver, ticker, cancel_token, - base_interval: base_interval.max(Duration::from_secs(1)), + base_interval, max_jitter, }, |mut actor, rx| async move { actor.run(rx).await }, From 4d7bad74de3d93bd2054f33f91f2e46d3bfbbf4c Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Fri, 17 Apr 2026 23:59:25 +0200 Subject: [PATCH 17/30] chore: impl review suggestions + fix and add docs --- ARCHITECTURE.md | 121 ++++++---- PROTOCOL.md | 353 +++++++++++++--------------- README.md | 164 ++++++++++++- examples/chat.rs | 2 +- examples/chat_no_wait.rs | 5 +- examples/e2e_test.rs | 2 +- examples/secret_rotation.rs | 4 +- examples/simple.rs | 4 +- src/config.rs | 34 ++- src/crypto/keys.rs | 5 +- src/crypto/record.rs | 22 +- src/dht.rs | 17 +- src/gossip/merge/bubble.rs | 4 +- src/gossip/merge/message_overlap.rs | 8 +- src/gossip/merge/mod.rs | 2 +- src/gossip/receiver.rs | 6 +- src/gossip/sender.rs | 1 + src/gossip/topic/bootstrap.rs | 22 +- src/gossip/topic/topic.rs | 32 ++- src/lib.rs | 2 +- tests/tests.rs | 63 ++--- 21 files changed, 531 insertions(+), 342 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 8ecac1d..204dff1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -17,7 +17,9 @@ Contents: Components: - iroh endpoint and gossip - Auto-discovery (bootstrap loop) -- Publisher (background task) +- Publisher (background actor) +- Bubble merge (background actor) +- Message overlap merge (background actor) - DHT client (mutable records) - Crypto (signing, encryption, secret rotation) @@ -35,6 +37,8 @@ flowchart LR subgraph AutoDiscovery B[Bootstrap Loop] P[Publisher] + BM[Bubble Merge] + MO[Message Overlap Merge] end subgraph DHT @@ -44,8 +48,12 @@ flowchart LR A --> E --> G G <---> B G <---> P + G <---> BM + G <---> MO B <--> D P <--> D + BM <--> D + MO <--> D ``` Node lifecycle: @@ -53,7 +61,7 @@ Node lifecycle: - Start gossip - Auto-discovery: - Join topic, attempt bootstrap, connect - - Spawn publisher on success + - Spawn publisher, bubble merge, and message overlap merge actors on success State machine: @@ -64,7 +72,7 @@ stateDiagram-v2 Discovering --> Joining Joining --> Joined Joined --> Publishing - Publishing --> Joined : backoff or success loop + Publishing --> Joined : interval + jitter Discovering --> Discovering : retry/jitter Joining --> Discovering : no peers ``` @@ -82,25 +90,28 @@ sequenceDiagram participant Gossip Node->>Gossip: subscribe(topic_hash) - Node->>DHT: get_mutable(signing_pub, salt, 10s) + Note over Node: optionally publish startup record + Node->>DHT: get_mutable(signing_pub, salt, 10s timeout) DHT-->>Node: encrypted records (0..N) Node->>Node: decrypt, verify, filter(not self) alt candidates exist loop each candidate Node->>Gossip: join_peers([node_id]) - Node->>Node: sleep 100ms + Node->>Node: sleep per_peer_join_settle_time (100ms) Gossip-->>Node: NeighborUp? end - Node->>Node: final wait 500ms + Node->>Node: final wait join_confirmation_wait_time (500ms) else no candidates Node->>Node: maybe publish own (rate-limited) + Node->>Node: sleep no_peers_retry_interval (1500ms) end - Node->>Node: joined? if yes, spawn publisher + Node->>Node: joined? if yes, spawn publisher + merge actors ``` Key points: -- First iteration: also check previous unix minute. -- Pacing avoids bursts and “bubbles.” +- First iteration: optionally also check previous unix minute (`check_last_minute_record_first_on_startup`). +- Both `unix_minute` and `unix_minute - 1` records are always fetched. +- Pacing avoids bursts and "bubbles." - Keep trying until joined. Pseudocode: @@ -109,23 +120,23 @@ Pseudocode: loop: if joined(): return sender, receiver - minute = first_attempt ? -1 : 0 - recs = get_unix_minute_records(minute) + minute = first_attempt && check_last_minute_first ? -1 : 0 + recs = get_records(unix_minute(minute) - 1) + get_records(unix_minute(minute)) if recs.is_empty(): maybe_publish_this_minute() - sleep(100ms) + sleep(no_peers_retry_interval = 1500ms) continue for peer in extract_bootstrap_nodes(recs): if joined(): break join_peer(peer) - sleep(100ms) + sleep(per_peer_join_settle_time = 100ms) - sleep(500ms) + sleep(join_confirmation_wait_time = 500ms) if joined(): return maybe_publish_this_minute() - sleep(100ms) + sleep(discovery_poll_interval = 2000ms) ``` ## Publishing @@ -136,36 +147,36 @@ Flow: ```mermaid flowchart TD - A[Start Cycle] --> B[Get minute=now] - B --> C[Discover existing records] - C --> D[Filter active participants] - D --> E{>= 10 active?} - E -- Yes --> F[Stop rate-limited] + A[Tick] --> B[Get minute=now] + B --> C[Get existing records] + C --> E{>= max_bootstrap_records = 5?} + E -- Yes --> F[Skip - rate-limited] E -- No --> G[Build record: peers + msg hashes] G --> H[Sign + Encrypt] - H --> I[Publish with retries + jitter] - I --> J[Return records, including own on success] + H --> I[Publish to DHT] + I --> J[Reset ticker: base_interval + random jitter] ``` Pseudocode: ```text -records = get_unix_minute_records(now) -active = filter_active(records) -if active.len >= 10: return records - -rec = make_record(neighbors(<=5), last_hashes(<=5)) -enc = encrypt(sign(rec)) -publish_with_retry(enc, retries=3, jitter=0..2000ms) -return records + [rec_if_success] +// Publisher actor loop (interval: base_interval + random jitter) +on tick: + records = get_records(unix_minute(0)) + if records.len >= max_bootstrap_records(5): return + + rec = make_record(neighbors(<=5), last_hashes(<=5)) + enc = encrypt(sign(rec)) + publish(enc) + reset_ticker(base_interval + random(0, max_jitter)) ``` ## Bubble detection and merging -Signal 1: small cluster \(neighbors < 4\). +Signal 1: small cluster \(neighbors < min\_neighbors, default 4\). - Extract peer ids from discovered records. - Exclude zeros, self, current neighbors. -- Join up to MAX_JOIN_PEERS_COUNT. +- Join up to max_join_peer_count (default 4). Signal 2: non-overlapping message sets. - Compare local last_message_hashes with others. @@ -176,12 +187,12 @@ Decision graph: ```mermaid flowchart LR - A[Post-Publish Records] --> B{neighbors < 4?} + A[Tick] --> B{neighbors < min_neighbors?} B -- Yes --> C[Join peers from records] B -- No --> D{local_msgs >= 1?} - D -- No --> E[Sleep random 0..60s] + D -- No --> E[Sleep until next tick] D -- Yes --> F{overlap with others?} - F -- No --> G[Join from non-overlap records] + F -- No --> G[Join from non-overlapping records] F -- Yes --> E ``` @@ -190,9 +201,8 @@ flowchart LR Record (summary): - topic hash (32) - unix_minute (u64) -- node_id (publisher) -- active_peers[5] (node ids) -- last_message_hashes[5] +- pub_key (publisher ed25519 public key) +- content (serialized RecordContent: active_peers + last_message_hashes) - signature (64) EncryptedRecord: @@ -206,16 +216,22 @@ classDiagram class Record { +topic: [u8;32] +unix_minute: u64 - +node_id: [u8;32] + +pub_key: [u8;32] + +content: RecordContent + +signature: [u8;64] + } + + class GossipRecordContent { +active_peers: [[u8;32];5] +last_message_hashes: [[u8;32];5] - +signature: [u8;64] } class EncryptedRecord { +encrypted_record: Vec +encrypted_decryption_key: Vec } + + Record --* GossipRecordContent : content deserializes to ``` Key derivation: @@ -239,23 +255,24 @@ flowchart TD - Decrypt/verify failure: - Drop record; proceed. - Publish failure: - - Exponential backoff (1..60 s), then retry. + - DHT layer retries with jittered intervals (3 retries, 5s base + 0-10s jitter). - Join failure: - - Continue to next peer; final 500 ms wait; loop. + - Continue to next peer; final 500ms wait; loop. ## Tuning -- Per-minute cap \(N_{active} \ge 10\) gates publishing. -- Pacing (100 ms) reduces bursts. -- Backoff (1..60 s) stabilizes DHT load. +- Per-minute cap \(records \ge max\_bootstrap\_records, default 5\) gates publishing. +- Per-peer pacing (100ms) reduces bursts. +- No-peers retry (1500ms) and discovery poll (2000ms) stabilize DHT load. - Message window size (5 peers, 5 hashes) is a trade-off: - Larger window = better visibility, larger records. - Smaller window = lower bandwidth, less overlap detection. -Parameters: -- MAX_BOOTSTRAP_RECORDS -- MAX_JOIN_PEERS_COUNT -- DHT timeout -- Retry count and jitter -- Join pacing and final wait -- Publisher backoff and success jitter \ No newline at end of file +Parameters (all configurable): +- `max_bootstrap_records` (default 5) +- `max_join_peer_count` (default 4) +- `min_neighbors` for bubble merge (default 4) +- DHT timeouts, retry count, and jitter +- Bootstrap timing: no_peers_retry, per_peer_settle, join_confirmation, discovery_poll +- Publisher timing: initial_delay, base_interval, max_jitter +- Merge timing: base_interval, max_jitter (separate for bubble and overlap) diff --git a/PROTOCOL.md b/PROTOCOL.md index 171c5e1..a5c3e38 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -1,54 +1,48 @@ # How this works: Protocol (spec) -## Publishing procedure (refined implementation) +## Publishing procedure -The publishing procedure is a rate-limited mechanism that prevents DHT overload while ensuring active participation in the gossip network. `publish_proc` function in `p01_refined.rs`: +The publishing procedure is a rate-limited mechanism that prevents DHT overload while ensuring active participation in the gossip network. Implemented in `RecordPublisher::publish_record()`: -- `MAX_BOOTSTRAP_RECORDS`: 10 (maximum active participant records allowed per unix minute) -- DHT timeout: 10 seconds for get_mutable operations -- Retry count: 3 attempts for DHT publishing operations -- Random delay: 0-2000ms between retry attempts +- `max_bootstrap_records`: 5 (default, configurable via `BootstrapConfig`) +- DHT get timeout: 10s (default, configurable via `DhtConfig`) +- DHT put retries: 3 (default, configurable via `DhtConfig`) +- DHT retry interval: 5s base + random 0-10s jitter (default, configurable via `DhtConfig`) ### Publishing Procedure 1. **Record Discovery** - - Call `get_unix_minute_records()` to fetch, decrypt, and verify existing records for the current unix minute + - Call `get_records()` to fetch, decrypt, and verify existing records for the current unix minute - Use the same key derivation as bootstrap: - - Derive signing keypair: `keypair_seed = hash(topic + unix_minute)` - - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic, unix_minute, initial_secret_hash)` - - Calculate salt: `salt = hash(topic + unix_minute)` + - Derive signing keypair: `keypair_seed = SHA512(topic_hash + unix_minute)[..32]` + - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic_hash, unix_minute, initial_secret_hash)` + - Calculate salt: `salt = SHA512(topic_hash + unix_minute)[..32]` - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout -2. **Active Participant Filtering** - - Filter records to include only "active participants" - records that have: - - Non-zero entries in `active_peers[5]` array, OR - - Non-zero entries in `last_message_hashes[5]` array - - This ensures only nodes actively participating in gossip are counted and network *Bubbles* are detected based on the `last_message_hashes` and merged. - -3. **Rate Limiting Check** - - If `active_participant_records.len() >= MAX_BOOTSTRAP_RECORDS` (10): - - **Do not publish** - return existing records to prevent DHT overload +2. **Rate Limiting Check** + - If `records.len() >= max_bootstrap_records` (default 5): + - **Do not publish** - return silently to prevent DHT overload - This implements the core rate limiting mechanism -4. **Record Creation** (if under limit) +3. **Record Creation** (if under limit) - Prepare `active_peers[5]` array: - - Fill with up to 5 current iroh-gossip neighbors + - Fill with up to 5 (`MAX_RECORD_PEERS`) current iroh-gossip neighbors - Remaining slots filled with zeros `[0; 32]` - Prepare `last_message_hashes[5]` array: - - Fill with up to 5 recent message hashes for proof of relay + - Fill with up to 5 (`MAX_MESSAGE_HASHES`) recent message hashes for proof of relay - Remaining slots filled with zeros `[0; 32]` -5. **Record Signing and Publishing** +4. **Record Signing and Publishing** - Create signed record using `Record::sign()`: - - Include: `topic_hash`, `unix_minute`, `node_id`, `active_peers`, `last_message_hashes` - - Sign with node's ed25519 signing key (`iroh::NodeId`) + - Include: `topic_hash`, `unix_minute`, `node_id`, content (serialized `active_peers` + `last_message_hashes`) + - Sign with node's ed25519 signing key - Encrypt record using one-time encryption key - - Publish to DHT via `publish_unix_minute_record()` with 3 retry attempts + - Publish to DHT via `Dht::put_mutable()` with retry support -6. **Error Handling** +5. **Error Handling** - DHT timeouts return empty record sets (non-fatal) - Failed record decryption/verification are ignored - - Random delays between retries prevent synchronized access patterns + - DHT put retries with jittered intervals prevent synchronized access patterns ### Publishing Flow Diagram @@ -56,101 +50,101 @@ The publishing procedure is a rate-limited mechanism that prevents DHT overload flowchart TD A[Start Publishing Procedure] --> B[Get Current Unix Minute] B --> C[Derive Keys: Signing & Encryption] - C --> D[Calculate Salt: hash = topic + unix_minute] + C --> D[Calculate Salt: SHA512 = topic_hash + unix_minute] D --> E[Query DHT: get_mutable = signing_pubkey, salt; Timeout: 10s] E --> F[Decrypt & Verify Records] - F --> G[Filter Active Participants] - G --> H{Active Records >= MAX_BOOTSTRAP_RECORDS = 10?} + F --> G{records.len >= max_bootstrap_records = default 5?} - H -- Yes --> I[Return Existing Records - Rate Limited] - H -- No --> J[Prepare Active Peers Array] + G -- Yes --> I[Return silently - Rate Limited] + G -- No --> J[Prepare Active Peers Array] J --> K[Fill active_peers with up to 5 gossip neighbors] K --> L[Prepare Last Message Hashes Array] L --> M[Fill last_message_hashes with up to 5 recent hashes] M --> N[Create Signed Record] - N --> O[Sign with: topic + unix_minute + node_id + active_peers + last_message_hashes] + N --> O[Sign with: topic + unix_minute + node_id + content] O --> P[Encrypt Record with One-Time Key] - P --> Q[Publish to DHT with 3 Retries] + P --> Q[Publish to DHT with retries] Q --> R{Publish Success?} - R -- Yes --> S[Return All Records Including Own] - R -- No --> T[Random Delay 0-2000ms] + R -- Yes --> S[Done] + R -- No --> T[Retry with jittered interval] T --> U{Retries Left?} U -- Yes --> Q - U -- No --> V[Return Error - Failed to Publish] + U -- No --> V[Return Error] style A fill:#f4f4f4,stroke:#333,stroke-width:1px style I fill:#ffcccc,stroke:#333,stroke-width:1px style S fill:#ccffcc,stroke:#333,stroke-width:1px style V fill:#ffcccc,stroke:#333,stroke-width:1px - style G fill:#f4f4f4,stroke:#333,stroke-width:1px style N fill:#f4f4f4,stroke:#333,stroke-width:1px ``` -## Bootstrap procedure (refined implementation) +## Bootstrap procedure -The bootstrap procedure is a continuous loop that attempts to discover and connect to existing nodes in the gossip network. Here's the detailed flow based on the `p01_refined.rs` implementation: +The bootstrap procedure is a continuous loop that attempts to discover and connect to existing nodes in the gossip network. Implemented in `BootstrapActor::start_bootstrap()`: -- `MAX_JOIN_PEERS_COUNT`: maximum peers to attempt joining -- `MAX_BOOTSTRAP_RECORDS`: maximum records allowed per unix minute -- DHT timeout: 10 seconds for get_mutable operations -- Connection retry delay: 100ms between attempts -- Final connection check delay: 500ms +- `max_join_peer_count`: 4 (default, configurable via `Config`) +- `max_bootstrap_records`: 5 (default, configurable via `BootstrapConfig`) +- DHT get timeout: 10s (default, configurable via `DhtConfig`) +- No peers retry interval: 1500ms (default, configurable via `BootstrapConfig`) +- Per-peer join settle time: 100ms (default, configurable via `BootstrapConfig`) +- Final join confirmation wait: 500ms (default, configurable via `BootstrapConfig`) +- Discovery poll interval: 2000ms (default, configurable via `BootstrapConfig`) +- Publish on startup: true (default, configurable via `BootstrapConfig`) +- Check last minute record first on startup: false (default, configurable via `BootstrapConfig`) ### Bootstrap Loop 1. **Initial Setup** - Subscribe to the gossip topic using `topic_id.hash` - - Initialize `last_published_unix_minute = 0` to track publishing state + - Optionally publish own record before the first DHT get (`publish_record_on_startup`) - Enter the main bootstrap loop 2. **Connection Check** - Check if already connected to at least one gossip peer via `gossip_receiver.is_joined()` - - If connected, return successfully with gossip sender/receiver pair + - If connected, exit bootstrap loop 3. **Time Window Selection** - - On first attempt: check previous unix minute (`unix_minute(-1)`) - - On subsequent attempts: check current unix minute (`unix_minute(0)`) - - This ensures we reliably discover existing gossip network records on the first try + - If `check_last_minute_record_first_on_startup` is enabled and this is the first attempt: check previous unix minute (`unix_minute(-1)`) + - Otherwise: check current unix minute (`unix_minute(0)`) + - Both `unix_minute` and `unix_minute - 1` records are always fetched 4. **Record Discovery** - - Call `get_unix_minute_records()` to fetch, decrypt, and verify records: - - Derive signing keypair: `keypair_seed = hash(topic + unix_minute)` - - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic, unix_minute, initial_secret_hash)` - - Calculate salt: `salt = hash(topic + unix_minute)` + - Call `get_records()` for both `unix_minute - 1` and `unix_minute`: + - Derive signing keypair: `keypair_seed = SHA512(topic_hash + unix_minute)[..32]` + - Derive encryption keypair from shared secret + - Calculate salt: `SHA512(topic_hash + unix_minute)[..32]` - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout - Decrypt each record using the encryption keypair - Verify signature, unix_minute, and topic hash - Filter out own records (matching node_id) 5. **If no valid Records Found** - - If no valid records found, attempt to publish own record via `publish_proc()` - - Only publish if haven't published in this unix minute - - Sleep 100ms and continue loop + - If no valid records found and haven't published in this unix minute, publish own record + - Wait for `no_peers_retry_interval` (default 1500ms) or a `joined()` event, then continue loop -6. **else if valid Records Found** +6. **If valid Records Found** - Extract bootstrap nodes from records: - Include `record.node_id` (the publisher) - Include all non-zero entries from `record.active_peers[5]` - - Convert byte arrays to valid `iroh::NodeId` instances + - Convert byte arrays to valid `iroh::EndpointId` instances 7. **Connection Attempts** - Check again if already connected (someone might have connected to us) - If not connected, attempt to join peers one by one: - Call `gossip_sender.join_peers(vec![node_id])` for each bootstrap node - - Sleep 100ms between attempts to minimize disruption + - Wait `per_peer_join_settle_time` (default 100ms) between attempts - Break early if connection established - - (findings showed connecting more too many nodes at once can cause the formation of netowrk *Bubbles*, isolated subnetworks that are not connected to the main network) 8. **Final Connection Verification** - - If still not connected, wait 500ms for iroh-gossip connection timeout + - If still not connected, wait `join_confirmation_wait_time` (default 500ms) - Check `gossip_receiver.is_joined()` one final time - - If connected: return successfully; spawn publisher task - - If not connected: attempt to publish own record (if not done this minute) - - Sleep 100ms and continue loop + - If connected: exit loop successfully + - If not connected: publish own record if not done this minute + - Wait `discovery_poll_interval` (default 2000ms) and continue loop ### Error Handling - DHT timeouts return empty record sets (non-fatal) @@ -162,54 +156,54 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne ```mermaid flowchart TD - A[Start Bootstrap] --> B[Subscribe to Gossip Topic] - B --> C[Initialize last_published_unix_minute = 0] - C --> D{Already Connected?} - + A[Start Bootstrap] --> AA{publish_record_on_startup?} + AA -- Yes --> AB[Publish startup record] + AA -- No --> B + AB --> B[Subscribe to Gossip Topic] + B --> D{Already Connected?} - D -- Yes --> Z[Return Success; Spawn Publisher] + D -- Yes --> Z[Return Success] D -- No --> E[Determine Unix Minute] - E --> F{First attempt?} + E --> F{First attempt AND check_last_minute_record_first?} F -- Yes --> G[Use Previous Minute unix_minute = -1] - F -- No --> H[Use Current Minute unix_minute = 0 ] + F -- No --> H[Use Current Minute unix_minute = 0] - G --> I[Get Unix Minute Records] + G --> I[Get Records for unix_minute-1 AND unix_minute] H --> I - I --> J[Derive Signing Keypair hash = topic + unix_minute ] + I --> J[Derive Signing Keypair] J --> K[Derive Encryption Keypair from shared secret] - K --> L[Calculate Salt hash = topic + unix_minute] + K --> L[Calculate Salt] L --> M[Query DHT: get_mutable = signing_pubkey, salt; Timeout: 10s] M --> N{Records Found?} N -- No --> O{Published This Minute?} - O -- No --> P[Publish Own Record via publish_proc] - O -- Yes --> Q[Sleep 100ms] + O -- No --> P[Publish Own Record] + O -- Yes --> Q[Wait no_peers_retry_interval = 1500ms or joined event] P --> Q Q --> D N -- Yes --> R[Decrypt & Verify Records] - R --> S[Filter Valid Records 1.Decrypt with encryption key 2.Verify signature 3.Check topic & unix_minute 4.Exclude own node_id] - S --> T[Extract Bootstrap Nodes 1.record.node_id 2.record.active_peers] - T --> U[Convert to iroh::NodeId] - U --> V{Already Connected?} + R --> S[Filter Valid Records] + S --> T[Extract Bootstrap Nodes: node_id + active_peers] + T --> V{Already Connected?} V -- Yes --> Z V -- No --> W[Join Peers One by One] - W --> X[For each bootstrap node: gossip_sender.join_peers = node_id] - X --> Y[Sleep 100ms] - Y --> AA{Connected?} - AA -- Yes --> Z - AA -- No --> BB{More Nodes?} + W --> X[For each bootstrap node: join_peers = node_id] + X --> Y[Wait per_peer_join_settle_time = 100ms] + Y --> AA2{Connected?} + AA2 -- Yes --> Z + AA2 -- No --> BB{More Nodes?} BB -- Yes --> X - BB -- No --> CC[Sleep 500ms Final connection timeout] + BB -- No --> CC[Wait join_confirmation_wait_time = 500ms] CC --> DD{Connected?} DD -- Yes --> Z DD -- No --> EE{Should Publish?} EE -- Yes --> FF[Publish Own Record] - EE -- No --> GG[Sleep 100ms] + EE -- No --> GG[Wait discovery_poll_interval = 2000ms] FF --> GG GG --> D @@ -220,131 +214,102 @@ flowchart TD style W fill:#f4f4f4,stroke:#333,stroke-width:1px ``` -## Spawn Publisher (refined implementation) +## Publisher -The Publisher is a background task that runs continuously after successful bootstrap to maintain topic presence on the DHT and detect/merge network *Bubbles*. `spawn_publisher` function in `p01_refined.rs`: +The Publisher is a separate background actor that runs after successful bootstrap to maintain topic presence on the DHT. Implemented in `PublisherActor`: -- `MAX_JOIN_PEERS_COUNT`: maximum peers to attempt joining during bubble merging -- Backoff mechanism: starts at 1 second, doubles on failure, caps at 60 seconds -- Random sleep interval: 0-60 seconds between successful publishing cycles -- Bubble detection threshold: less than 4 neighbors or non overlapping messages indicates potential network isolation +- Interval: `base_interval + random(0, max_jitter)` (default 10s base + 0-50s jitter) +- Initial delay: 10s (default, configurable via `PublisherConfig`) +- No exponential backoff; interval is constant with jitter ### Publisher Loop 1. **Initialization** - - Spawned as a background task after successful bootstrap - - Receives cloned gossip sender/receiver, topic configuration, and signing key - - Initializes exponential backoff counter starting at 1 second + - Spawned as a background actor after successful bootstrap + - Configures a ticker with `initial_delay` then repeating at `base_interval` 2. **Publishing Cycle** - - Get current unix minute: `unix_minute(0)` - - Call `publish_proc()` with live gossip data: - - Current neighbors from `gossip_receiver.neighbors()` - - Recent message hashes from `gossip_receiver.last_message_hashes()` - - This ensures published records contain real-time gossip network state + - On each tick: call `publish()` which creates a new record with current gossip state + - After publishing, reset ticker to `base_interval + random(0, max_jitter)` + - Record includes current neighbors (up to 5) and recent message hashes (up to 5) -3. **Bubble Detection and Merging** +### Publisher Flow Diagram - **Small Cluster Detection:** - - If `neighbors.len() < 4` AND valid records exist: - - Extract node IDs from `record.active_peers` in discovered records - - Filter out: zero entries, current neighbors, own node ID - - Attempt to join up to `MAX_JOIN_PEERS_COUNT` (100) new peers - - This helps merge small isolated clusters back into the main network +```mermaid +flowchart TD + A[Spawn Publisher Actor] --> B[Wait initial_delay = 10s] + B --> C[Tick] + C --> D[Get Live Gossip Data: neighbors + message_hashes] + D --> E[Create Record for current unix_minute] + E --> F[Publish via RecordPublisher::publish_record] + F --> G[Reset ticker: base_interval + random jitter] + G --> C - **Message Overlap Analysis:** - - If local node has received messages (`last_message_hashes.len() >= 1`): - - Compare local message hashes with `record.last_message_hashes` from other nodes - - Identify records with non-overlapping message sets (potential bubble or stale node indicator) - - Extract all node IDs (publisher + active_peers) from non-overlapping records - - Attempt to join these peers to merge network bubbles + style A fill:#f4f4f4,stroke:#333,stroke-width:1px +``` -4. **Error Handling and Backoff** - - On `publish_proc()` failure: - - Sleep for current backoff duration (1, 2, 4, 8, ..., 60 seconds) - - Double backoff duration, capped at 60 seconds - - Continue loop (retry publishing) +## Bubble detection and merging -5. **Success Handling** - - On successful publishing: - - Reset backoff to 1 second - - Sleep for random duration: 0-60 seconds - - Continue loop +Bubble detection and merging run as separate background actors alongside the publisher, each on their own interval timer. -### Network Bubble Detection Logic +### Bubble Merge (small cluster detection) -The publisher implements two bubble detection mechanisms: +Implemented in `BubbleMergeActor`. Interval: 60s base + 0-120s jitter (default). -1. **Cluster Size Analysis**: Small neighbor counts (< 4) suggest network fragmentation -2. **Message Flow Analysis**: Non-overlapping message hashes indicate isolated subnetworks or stale nodes +- If `neighbors.len() < min_neighbors` (default 4) AND DHT records exist: + - Extract node IDs from `record.active_peers` in discovered records + - Filter out: zero entries, current neighbors, own node ID + - Attempt to join up to `max_join_peer_count` (default 4) new peers -When bubbles are detected, the publisher proactively attempts to join peers from other network segments to restore connectivity. +### Message Overlap Merge (partition detection) -### Publisher Flow Diagram +Implemented in `MessageOverlapMergeActor`. Interval: 60s base + 0-120s jitter (default). -```mermaid -flowchart TD - A[Spawn Publisher Task] --> B[Initialize Backoff = 1s] - B --> C[Start Publisher Loop] - C --> D[Get Current Unix Minute] - D --> E[Get Live Gossip Data: neighbors + message_hashes] - E --> F[Call publish_proc = unix_minute, neighbors, message_hashes] - - F --> G{Publish Success?} - G -- No --> H[Failure] - H --> I[Sleep = backoff seconds] - I --> J[Backoff = min = backoff * 2, 60] - J --> C - - G -- Yes --> K[Bubble Detection Analysis] - K --> L{neighbors.len < 4 AND records exist?} - L -- Yes --> M[Small Cluster Detection] - M --> N[Extract active_peers from records] - N --> O[Filter: exclude zeros, current neighbors, self] - O --> P[Join up to MAX_JOIN_PEERS_COUNT peers] - P --> Q[Message Overlap Analysis] - - L -- No --> Q - Q --> R{Have local messages?} - R -- Yes --> S[Compare message_hashes with records] - S --> T{Non-overlapping messages found?} - T -- Yes --> U[Extract node_ids from non-overlapping records] - U --> V[Join peers to merge bubbles] - V --> W[Success] - - T -- No --> W - R -- No --> W - W --> X[Reset Backoff = 1s] - X --> Y[Sleep = random = 0-60s] - Y --> C +- If local node has received messages (`last_message_hashes.len() >= 1`): + - Compare local message hashes with `record.last_message_hashes` from other nodes + - Identify records with non-overlapping message sets (potential network partition) + - Extract all node IDs (publisher + active_peers) from non-overlapping records + - Attempt to join these peers to bridge partitions - style A fill:#f4f4f4,stroke:#333,stroke-width:1px - style K fill:#fff2cc,stroke:#333,stroke-width:1px - style M fill:#fff2cc,stroke:#333,stroke-width:1px - style S fill:#fff2cc,stroke:#333,stroke-width:1px - style W fill:#ccffcc,stroke:#333,stroke-width:1px - style H fill:#ffcccc,stroke:#333,stroke-width:1px +### Bubble Detection Decision Graph + +```mermaid +flowchart LR + A[Tick] --> B{neighbors < min_neighbors?} + B -- Yes --> C[Join peers from records] + B -- No --> D{local_msgs >= 1?} + D -- No --> E[Sleep until next tick] + D -- Yes --> F{overlap with others?} + F -- No --> G[Join from non-overlapping records] + F -- Yes --> E ``` -## Record structure (refined implementation) +## Record structure + +The record struct wraps a serialized `RecordContent`: ```rust -// 489 bytes total (S=5) #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Record { - // Record Content - topic: [u8; 32], // sha512(topic_string)[..32] + // Header + topic: [u8; 32], // SHA512(topic_string)[..32] unix_minute: u64, // floor(unixtime / 60) - node_id: [u8; 32], // publisher ed25519 public key - active_peers: [[u8; 32]; 5], // 5 node ids of active gossip peers - last_message_hashes: [[u8; 32]; 5], // 5 recent message hashes for proof of relay + pub_key: [u8; 32], // publisher ed25519 public key + + // Content (serialized via bincode) + content: RecordContent, // Vec wrapping serialized data + + // Signature + signature: [u8; 64], // ed25519 signature over topic + unix_minute + pub_key + content +} - // Record Signature - signature: [u8; 64], // ed25519 signature over above fields - // signed by the publisher's ed25519 private key +// Default content used by the gossip module: +pub struct GossipRecordContent { + pub active_peers: [[u8; 32]; 5], // MAX_RECORD_PEERS node ids + pub last_message_hashes: [[u8; 32]; 5], // MAX_MESSAGE_HASHES recent hashes } -// Variable size (>= 493 bytes at S=5) +// Variable size #[derive(Debug, Clone)] pub struct EncryptedRecord { encrypted_record: Vec, // encrypted Record using one-time key @@ -353,7 +318,7 @@ pub struct EncryptedRecord { } ``` -## Verification (refined implementation) +## Verification The `Record::verify()` method performs the following checks: @@ -361,8 +326,8 @@ The `Record::verify()` method performs the following checks: 2. **Time Verification**: Verify `record.unix_minute` matches the unix minute used for key derivation 3. **Signature Verification**: - Extract signature data: all record bytes except the last 64 bytes (signature) - - Signature data includes: `topic + unix_minute + node_id + active_peers + last_message_hashes` - - Verify ed25519 signature using `record.node_id` as the public key + - Signature data includes: `topic + unix_minute + pub_key + content` + - Verify ed25519 signature using `record.pub_key` as the public key - Use `verify_strict()` for enhanced security ### Additional Filtering @@ -377,7 +342,7 @@ A one-time key encryption scheme is used to protect record content while allowin **Signing Keypair (Public DHT Discovery):** - Purpose: Used for DHT mutable record signing and salt calculation -- Derivation: `signing_keypair_seed = SHA512(topic_hash + unix_minute)[..32]` +- Derivation: `signing_keypair_seed = SHA512("salt" + topic_hash + unix_minute)[..32]` - Key: `ed25519_dalek::SigningKey::from_bytes(signing_keypair_seed)` - Public: This keypair is deterministic and publicly derivable @@ -419,7 +384,6 @@ A one-time key encryption scheme is used to protect record content while allowin 1. **Decrypt One-Time Key** - Derive encryption keypair from shared secret (same as encryption) - Attempt decryption: `one_time_key_bytes = encryption_keypair.decrypt(encrypted_decryption_key)` - - Fallback to previous key if rotation occurred: `last_decryption_key.decrypt(encrypted_decryption_key)` - Reconstruct one-time key: `one_time_key = ed25519_dalek::SigningKey::from_bytes(one_time_key_bytes)` 2. **Decrypt Record Data** @@ -435,14 +399,14 @@ A one-time key encryption scheme is used to protect record content while allowin ``` [4 bytes: encrypted_record_length (little-endian u32)] [variable: encrypted_record data] -[variable: encrypted_decryption_key data] +[variable: encrypted_decryption_key data (88 bytes)] ``` ### Security Properties - **Forward Secrecy**: One-time keys are generated randomly for each record - **Access Control**: Only nodes with the shared secret can decrypt records -- **Key Rotation**: Supports fallback to previous encryption keys during rotation +- **Key Rotation**: Supports secret rotation via the `SecretRotation` trait - **Replay Protection**: Unix minute coupling prevents replay attacks - **Public Discovery**: DHT discovery remains public while content stays private @@ -464,11 +428,8 @@ flowchart TD J --> K[Derive Encryption Keypair from Shared Secret] K --> L[Decrypt One-Time Key] L --> M{Decryption Success?} - M -- No --> N[Try Previous Key] - N --> O{Success?} - O -- No --> P[Decryption Failed] - O -- Yes --> Q[Reconstruct One-Time Key] - M -- Yes --> Q + M -- No --> P[Decryption Failed] + M -- Yes --> Q[Reconstruct One-Time Key] Q --> R[Decrypt Record Data] R --> S[Deserialize Record] S --> T[Verify Signature & Metadata] diff --git a/README.md b/README.md index f076710..c03f871 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ rand = "0.9" iroh = "*" iroh-gossip = "*" -distributed-topic-tracker = "0.2" +distributed-topic-tracker = "0.3" ``` Basic iroh-gossip integration: @@ -110,15 +110,161 @@ Verify peer discovery across Docker containers: The E2E test confirms multiple nodes discover each other via DHT and join the same gossip topic. -## Roadmap +## Upgrading from 0.2 to 0.3 -- [x] Finalize crate name and publish to crates.io -- [x] Tests and CI -- [x] Add more examples -- [x] Optimize configuration -- [x] Major refactor -- [x] Make `iroh-gossip` integration a feature (repurposed for rustpatcher) -- [x] API docs +### Breaking changes + +**`RecordTopic` removed, use `TopicId` instead** + +`RecordTopic` has been removed. `TopicId` now serves as the unified topic identifier across the entire API and supports conversion from `&str`, `String`, `Vec`, and `FromStr`. + +```rust,ignore +// 0.2 +use distributed_topic_tracker::RecordTopic; +let topic = RecordTopic::from_str("my-topic")?; +let publisher = RecordPublisher::new(topic, signing_key, None, secret); + +// 0.3 +use distributed_topic_tracker::TopicId; +let topic = TopicId::new("my-topic".to_string()); +// or: let topic: TopicId = "my-topic".into(); +// or: let topic: TopicId = "my-topic".parse()?; +let publisher = RecordPublisher::new(topic, signing_key, None, secret, Config::default()); +``` + +**`RecordPublisher::new()` now requires a `Config` parameter** + +A 5th `Config` parameter was added. Use `Config::default()` for most use cases, tune as needed. + +```rust,ignore +// 0.2 +let publisher = RecordPublisher::new(topic, pub_key, signing_key, None, secret); + +// 0.3 +use distributed_topic_tracker::Config; +let publisher = RecordPublisher::new(topic, signing_key, None, secret, Config::default()); + +// or use the new builder: +let publisher = RecordPublisher::builder(topic, signing_key, secret) + .config(Config::builder().max_join_peer_count(8).build()) + .secret_rotation(rotation_handle) + .build(); +``` + +**`MAX_BOOTSTRAP_RECORDS` constant removed** + +The per-minute record cap is now configurable via `BootstrapConfig::max_bootstrap_records` (default: 5, was hardcoded 100). + +```rust,ignore +// 0.2 +use distributed_topic_tracker::MAX_BOOTSTRAP_RECORDS; // was 100 + +// 0.3 - configure via Config +let config = Config::builder() + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(10) + .build() + ) + .build(); +``` + +**`TopicId::raw()` removed** + +`TopicId` no longer stores the original string. Only the 32-byte hash is retained. + +```rust,ignore +// 0.2 +let topic = TopicId::new("my-topic".to_string()); +let original: &str = topic.raw(); // no longer available + +// 0.3 - store the raw string yourself if needed +let raw = "my-topic".to_string(); +let topic = TopicId::new(raw.clone()); +``` + +### New features + +**Full configuration system** + +All timing, retry, and threshold parameters are now configurable: + +```rust,ignore +use distributed_topic_tracker::{ + Config, BootstrapConfig, DhtConfig, TimeoutConfig, + PublisherConfig, BubbleMergeConfig, MessageOverlapMergeConfig, MergeConfig, +}; +use std::time::Duration; + +Config::builder() + .dht_config( + DhtConfig::builder() + .retries(3) + .base_retry_interval(Duration::from_secs(5)) + .max_retry_jitter(Duration::from_secs(10)) + .get_timeout(Duration::from_secs(10)) + .put_timeout(Duration::from_secs(10)) + .build(), + ) + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(5) + .publish_record_on_startup(true) + .check_last_minute_record_first_on_startup(false) + .discovery_poll_interval(Duration::from_millis(2000)) + .no_peers_retry_interval(Duration::from_millis(1500)) + .per_peer_join_settle_time(Duration::from_millis(100)) + .join_confirmation_wait_time(Duration::from_millis(500)) + .build(), + ) + .max_join_peer_count(4) + .publisher_config(PublisherConfig::Enabled { + initial_delay: Duration::from_secs(10), + base_interval: Duration::from_secs(10), + max_jitter: Duration::from_secs(50), + }) + .merge_config(MergeConfig::new( + BubbleMergeConfig::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + min_neighbors: 4, + }, + MessageOverlapMergeConfig::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + }, + )) + .timeouts( + TimeoutConfig::builder() + .join_peer_timeout(Duration::from_secs(5)) + .broadcast_neighbors_timeout(Duration::from_secs(5)) + .broadcast_timeout(Duration::from_secs(5)) + .build(), + ) + .build(); +``` + +**Disable merge strategies or publishing** + +```rust,ignore +// Run without any merge or publishing (bootstrap-only) +let config = Config::builder() + .publisher_config(PublisherConfig::Disabled) + .merge_config(MergeConfig::new( + BubbleMergeConfig::Disabled, + MessageOverlapMergeConfig::Disabled, + )) + .build(); +``` + +**`RecordPublisher::builder()` for ergonomic construction** + +```rust,ignore +let publisher = RecordPublisher::builder("my-topic", signing_key, b"secret") + .secret_rotation(rotation_handle) + .config(config) + .build(); +``` ## License diff --git a/examples/chat.rs b/examples/chat.rs index 38e5c3e..a3d2093 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -83,7 +83,7 @@ async fn main() -> Result<()> { print!("\n> "); stdin.read_line(&mut buffer).unwrap(); gossip_sender - .broadcast(buffer.clone().replace("\n", "").into()) + .broadcast(buffer.clone().replace("\n", "").replace("\r\n", "").into()) .await .unwrap(); println!(" - (sent)"); diff --git a/examples/chat_no_wait.rs b/examples/chat_no_wait.rs index 3537b5d..fd59556 100644 --- a/examples/chat_no_wait.rs +++ b/examples/chat_no_wait.rs @@ -2,9 +2,10 @@ use anyhow::Result; use iroh::{Endpoint, SecretKey}; use iroh_gossip::{api::Event, net::Gossip}; -// Imports from distrubuted-topic-tracker +use ed25519_dalek::SigningKey; + +// Imports from distributed-topic-tracker use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; -use mainline::SigningKey; #[tokio::main] async fn main() -> Result<()> { diff --git a/examples/e2e_test.rs b/examples/e2e_test.rs index 415a237..e9d177a 100644 --- a/examples/e2e_test.rs +++ b/examples/e2e_test.rs @@ -4,7 +4,7 @@ use iroh::{ }; use iroh_gossip::net::Gossip; -// Imports from distrubuted-topic-tracker +// Imports from distributed-topic-tracker use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; #[tokio::main] diff --git a/examples/secret_rotation.rs b/examples/secret_rotation.rs index 3003fd4..9acbbdb 100644 --- a/examples/secret_rotation.rs +++ b/examples/secret_rotation.rs @@ -4,7 +4,7 @@ use iroh::{Endpoint, SecretKey}; use iroh_gossip::{api::Event, net::Gossip}; use sha2::Digest; -// Imports from distrubuted-topic-tracker +// Imports from distributed-topic-tracker use distributed_topic_tracker::{ AutoDiscoveryGossip, Config, RecordPublisher, RotationHandle, SecretRotation, TopicId }; @@ -49,8 +49,6 @@ async fn main() -> Result<()> { let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - // Split into sink (sending) and stream (receiving) - let record_publisher = RecordPublisher::new( topic_id.clone(), signing_key.clone(), diff --git a/examples/simple.rs b/examples/simple.rs index db451af..976b1c5 100644 --- a/examples/simple.rs +++ b/examples/simple.rs @@ -3,7 +3,7 @@ use ed25519_dalek::SigningKey; use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; -// Imports from distrubuted-topic-tracker +// Imports from distributed-topic-tracker use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; #[tokio::main] @@ -29,8 +29,6 @@ async fn main() -> Result<()> { let topic_id = TopicId::new("my-iroh-gossip-topic".to_string()); let initial_secret = b"my-initial-secret".to_vec(); - // Split into sink (sending) and stream (receiving) - let record_publisher = RecordPublisher::new( topic_id.clone(), signing_key.clone(), diff --git a/src/config.rs b/src/config.rs index bab3278..833d00d 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ use std::time::Duration; +/// Timeout settings for gossip operations. #[derive(Debug, Clone)] pub struct TimeoutConfig { join_peer_timeout: Duration, @@ -8,6 +9,7 @@ pub struct TimeoutConfig { } impl TimeoutConfig { + /// Create a new `TimeoutConfigBuilder` with default values. pub fn builder() -> TimeoutConfigBuilder { TimeoutConfigBuilder { timeouts: TimeoutConfig::default(), @@ -40,6 +42,7 @@ impl Default for TimeoutConfig { } } +/// Builder for `TimeoutConfig`. #[derive(Debug)] pub struct TimeoutConfigBuilder { timeouts: TimeoutConfig, @@ -64,11 +67,13 @@ impl TimeoutConfigBuilder { self } + /// Build the `TimeoutConfig`. pub fn build(self) -> TimeoutConfig { self.timeouts } } +/// DHT operation settings including retry logic and timeouts. #[derive(Debug, Clone)] pub struct DhtConfig { retries: usize, @@ -78,6 +83,7 @@ pub struct DhtConfig { get_timeout: Duration, } +/// Builder for `DhtConfig`. #[derive(Debug, Clone)] pub struct DhtConfigBuilder { config: DhtConfig, @@ -114,12 +120,14 @@ impl DhtConfigBuilder { self } + /// Build the `DhtConfig`. pub fn build(self) -> DhtConfig { self.config } } impl DhtConfig { + /// Create a new `DhtConfigBuilder` with default values. pub fn builder() -> DhtConfigBuilder { DhtConfigBuilder { config: DhtConfig::default(), @@ -244,6 +252,7 @@ impl MergeConfig { } } +/// Bootstrap process settings for peer discovery. #[derive(Debug, Clone)] pub struct BootstrapConfig { max_bootstrap_records: usize, @@ -269,15 +278,16 @@ impl Default for BootstrapConfig { } } +/// Builder for `BootstrapConfig`. #[derive(Debug)] pub struct BootstrapConfigBuilder { config: BootstrapConfig, } impl BootstrapConfigBuilder { - /// Max bootstrap records per topic per minute slot (min 1). Default: 5. + /// Max bootstrap records per topic per minute slot. If zero, we don't publish (PublisherConfig will be set to Disabled). Default: 5. pub fn max_bootstrap_records(mut self, max_records: usize) -> Self { - self.config.max_bootstrap_records = max_records.max(1); + self.config.max_bootstrap_records = max_records; self } @@ -311,7 +321,7 @@ impl BootstrapConfigBuilder { self } - /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup. Default: false. + /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup (in this impl unix_minute and unix_minute-1 are both always fetched, if this is enabled than we first fetch unix_minute-2 and unix_minute-1). Default: false. /// /// If joining longer running, existing topics is priority, set to true. /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. @@ -320,19 +330,21 @@ impl BootstrapConfigBuilder { self } + /// Build the `BootstrapConfig`. pub fn build(self) -> BootstrapConfig { self.config } } impl BootstrapConfig { + /// Create a new `BootstrapConfigBuilder` with default values. pub fn builder() -> BootstrapConfigBuilder { BootstrapConfigBuilder { config: BootstrapConfig::default(), } } - /// Max bootstrap records per topic per minute slot (min 1). Default: 5. + /// Max bootstrap records per topic per minute slot. If zero, we don't publish (PublisherConfig will be set to Disabled, make sure to use config builder to enforce). Default: 5. pub fn max_bootstrap_records(&self) -> usize { self.max_bootstrap_records } @@ -362,7 +374,7 @@ impl BootstrapConfig { self.publish_record_on_startup } - /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup. Default: false. + /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup (in this impl unix_minute and unix_minute-1 are both always fetched, if this is enabled than we first fetch unix_minute-2 and unix_minute-1). Default: false. /// /// If joining longer running, existing topics is priority, set to true. /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. @@ -371,6 +383,7 @@ impl BootstrapConfig { } } +/// Top-level configuration combining all settings. #[derive(Debug, Clone)] pub struct Config { bootstrap_config: BootstrapConfig, @@ -384,6 +397,7 @@ pub struct Config { } impl Config { + /// Create a new `ConfigBuilder` with default values. pub fn builder() -> ConfigBuilder { ConfigBuilder { config: Config::default(), @@ -445,6 +459,7 @@ impl Default for Config { } } +/// Builder for `Config`. #[derive(Debug)] pub struct ConfigBuilder { config: Config, @@ -488,7 +503,14 @@ impl ConfigBuilder { self } + /// Build the `Config`. If `max_bootstrap_records` is zero, `PublisherConfig` is set to `Disabled`. pub fn build(self) -> Config { - self.config + let mut config = self.config; + if config.bootstrap_config.max_bootstrap_records == 0 { + // if max_bootstrap_records is zero, we don't publish, so disable publisher to avoid confusion + config.publisher_config = PublisherConfig::Disabled; + } + + config } } diff --git a/src/crypto/keys.rs b/src/crypto/keys.rs index 79e3855..07729f0 100644 --- a/src/crypto/keys.rs +++ b/src/crypto/keys.rs @@ -46,7 +46,7 @@ impl SecretRotation for DefaultSecretRotation { h.update(topic_hash); h.update(unix_minute.to_be_bytes()); h.update(initial_secret_hash); - h.finalize()[..32].try_into().unwrap() + h.finalize()[..32].try_into().expect("keys -> SecretRotation.derive() hash try into [..32] failed") } } @@ -136,10 +136,11 @@ pub fn encryption_keypair( /// Derive DHT salt for mutable record lookups. /// -/// Salt = SHA512(topic_hash || unix_minute.to_le_bytes())[..32] +/// Salt = SHA512("salt" || topic_hash || unix_minute.to_le_bytes())[..32] /// Ensures records are stored in different DHT slots per minute. pub fn salt(topic_id: &TopicId, unix_minute: u64) -> [u8; 32] { let mut slot_hash = sha2::Sha512::new(); + slot_hash.update(b"salt"); slot_hash.update(topic_id.hash()); slot_hash.update(unix_minute.to_le_bytes()); slot_hash.finalize()[..32] diff --git a/src/crypto/record.rs b/src/crypto/record.rs index 38e08b5..95068f5 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -72,6 +72,7 @@ pub struct RecordPublisher { initial_secret_hash: [u8; 32], } +/// Builder for `RecordPublisher`. #[derive(Debug)] pub struct RecordPublisherBuilder { topic_id: TopicId, @@ -82,16 +83,19 @@ pub struct RecordPublisherBuilder { } impl RecordPublisherBuilder { + /// Set a custom secret rotation strategy. pub fn secret_rotation(mut self, secret_rotation: crate::crypto::keys::RotationHandle) -> Self { self.secret_rotation = Some(secret_rotation); self } + /// Set the configuration. pub fn config(mut self, config: crate::config::Config) -> Self { self.config = config; self } + /// Build the `RecordPublisher`. pub fn build(self) -> RecordPublisher { RecordPublisher::new( self.topic_id, @@ -104,6 +108,7 @@ impl RecordPublisherBuilder { } impl RecordPublisher { + /// Create a new `RecordPublisherBuilder`. pub fn builder( topic_id: impl Into, signing_key: SigningKey, @@ -126,15 +131,16 @@ impl RecordPublisher { /// * `signing_key` - Ed25519 secret key (signing key) /// * `secret_rotation` - Optional custom key rotation strategy /// * `initial_secret` - Initial secret for key derivation + /// * `config` - Configuration settings pub fn new( topic_id: impl Into, signing_key: SigningKey, secret_rotation: Option, - initial_secret: Vec, + initial_secret: impl Into>, config: crate::config::Config, ) -> Self { let mut initial_secret_hash = sha2::Sha512::new(); - initial_secret_hash.update(initial_secret); + initial_secret_hash.update(initial_secret.into()); let initial_secret_hash: [u8; 32] = initial_secret_hash.finalize()[..32] .try_into() .expect("hashing failed"); @@ -174,7 +180,7 @@ impl RecordPublisher { self.pub_key } - /// Get the record topic. + /// Get TopicId. pub fn topic_id(&self) -> &TopicId { &self.topic_id } @@ -194,6 +200,7 @@ impl RecordPublisher { self.initial_secret_hash } + /// Get the configuration. pub fn config(&self) -> &Config { &self.config } @@ -203,7 +210,7 @@ impl RecordPublisher { /// Publish a record to the DHT if slot capacity allows. /// /// Checks existing record count for this time slot and skips publishing if - /// `MAX_BOOTSTRAP_RECORDS` limit reached. + /// `self.config.bootstrap_config().max_bootstrap_records()` limit reached. pub async fn publish_record(&self, record: Record) -> Result<()> { let records = self .get_records(record.unix_minute()) @@ -218,7 +225,7 @@ impl RecordPublisher { record.unix_minute() ); - if records.len() >= self.config.bootstrap_config().max_bootstrap_records().max(1) { + if records.len() >= self.config.bootstrap_config().max_bootstrap_records() { tracing::debug!( "RecordPublisher: max records reached ({}), skipping publish", self.config.bootstrap_config().max_bootstrap_records() @@ -296,10 +303,7 @@ impl RecordPublisher { Ok(encrypted_record) => match encrypted_record.decrypt(&encryption_key) { Ok(record) => match record.verify(&self.topic_id.hash(), unix_minute) { Ok(_) => match record.node_id().eq(self.pub_key.as_bytes()) { - true => { - tracing::debug!("RecordPublisher: filtered out self"); - None - } + true => None, false => Some(record), }, Err(_) => None, diff --git a/src/dht.rs b/src/dht.rs index f1058cc..cb73d7f 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -49,7 +49,6 @@ impl Dht { /// * `pub_key` - Ed25519 public key for the record /// * `salt` - Optional salt for record lookup /// * `more_recent_than` - Sequence number filter (get records newer than this) - /// * `timeout` - Maximum time to wait for results pub async fn get( &self, pub_key: VerifyingKey, @@ -69,8 +68,6 @@ impl Dht { /// * `pub_key` - Ed25519 public key (used for routing) /// * `salt` - Optional salt for record slot /// * `data` - Record value to publish - /// * `retry_count` - Number of retry attempts (default: 3) - /// * `timeout` - Per-request timeout pub async fn put_mutable( &self, signing_key: SigningKey, @@ -96,12 +93,19 @@ impl DhtActor { } let dht = self.dht.as_mut().context("DHT not initialized")?; - Ok(tokio::time::timeout( + match tokio::time::timeout( self.config.get_timeout(), dht.get_mutable(pub_key.as_bytes(), salt.as_deref(), more_recent_than) .collect::>(), ) - .await?) + .await + { + Ok(items) => Ok(items), + Err(_) => { + tracing::warn!("DHT get operation timed out"); + bail!("DHT get operation timed out") + } + } } pub async fn put_mutable( @@ -158,7 +162,8 @@ impl DhtActor { } else { 0 }; - let retry_interval = (self.config.base_retry_interval().as_millis() + jitter_ms).max(1000); + let retry_interval = + (self.config.base_retry_interval().as_millis() + jitter_ms).max(1000); tracing::debug!( "DHTActor: put_mutable attempt {}/{} failed, retrying in {}ms", diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 5c97e1f..b3a9df1 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -1,6 +1,6 @@ //! Bubble detection: merge isolated peer groups in the same topic. //! -//! If local peer count < 4, extract suggested peers from DHT records and join them. +//! If local peer count < `min_neighbors`, extract suggested peers from DHT records and join them. use actor_helper::{Action, Handle, Receiver}; use iroh::EndpointId; @@ -11,7 +11,7 @@ use anyhow::Result; /// Detects and merges small isolated peer groups (bubbles). /// -/// Triggers when local peer count < 4 and DHT records exist. Extracts `active_peers` +/// Triggers when local peer count < `min_neighbors` and DHT records exist. Extracts `active_peers` /// from DHT records and joins them. #[derive(Debug, Clone)] pub struct BubbleMerge { diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index e17f31e..900258b 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -9,7 +9,7 @@ use anyhow::Result; /// Detects network partitions by comparing message hashes across DHT records. /// -/// Joins peers when their published hashes match local message history. +/// Joins peers whose published hashes do not overlap with local message history. #[derive(Debug, Clone)] pub struct MessageOverlapMerge { _api: Handle, @@ -114,7 +114,7 @@ impl MessageOverlapMergeActor { if let Ok(content) = record.content::() { content.last_message_hashes.iter().any(|last_message_hash| { *last_message_hash != [0; 32] - && last_message_hashes.contains(last_message_hash) + && !last_message_hashes.contains(last_message_hash) }) } else { false @@ -123,7 +123,7 @@ impl MessageOverlapMergeActor { .collect::>(); tracing::debug!( - "MessageOverlapMerge: found {} peers with overlapping message hashes", + "MessageOverlapMerge: found {} peers with no overlapping message hashes", peers_to_join.len() ); @@ -160,7 +160,7 @@ impl MessageOverlapMergeActor { .collect::>(); tracing::debug!( - "MessageOverlapMerge: attempting to join {} node_ids with overlapping messages", + "MessageOverlapMerge: attempting to join {} node_ids with no overlapping messages", node_ids.len() ); diff --git a/src/gossip/merge/mod.rs b/src/gossip/merge/mod.rs index 9e66f5d..2e0f576 100644 --- a/src/gossip/merge/mod.rs +++ b/src/gossip/merge/mod.rs @@ -1,7 +1,7 @@ //! Peer merging strategies for recovering from network partitions. //! //! Two complementary strategies detect and heal split-brain scenarios: -//! - **Bubble Merge**: Joins small clusters (< BubbleMergeConfig.min_neighbors() peers) with peers advertised in DHT +//! - **Bubble Merge**: Joins small clusters (< `min_neighbors` peers, as configured in `BubbleMergeConfig::Enabled`) with peers advertised in DHT //! - **Message Overlap**: Detects when isolated clusters share common message hashes, //! indicating they have seen the same messages and can be merged diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index dbcdb8a..eae44b2 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -56,6 +56,7 @@ impl Clone for GossipReceiver { } } +/// Internal actor for gossip receive operations. #[derive(Debug)] pub struct GossipReceiverActor { gossip_receiver: iroh_gossip::api::GossipReceiver, @@ -128,6 +129,9 @@ impl GossipReceiver { } } + /// Waits for a NeighborUp or a message Received event then returns `Some(())`. + /// + /// Returns None if the receiver is closed. pub async fn joined(&mut self) -> Option<()> { if self.is_joined().await.unwrap_or(false) { return Some(()); @@ -211,7 +215,7 @@ impl GossipReceiverActor { } }; - self.next_channel_sender.send(Some(event.clone())).ok(); + self.next_channel_sender.send(Some(event)).ok(); } _ = self.cancel_token.cancelled() => { self.join_channel_sender.send(None).ok(); diff --git a/src/gossip/sender.rs b/src/gossip/sender.rs index 44d008e..f9f4939 100644 --- a/src/gossip/sender.rs +++ b/src/gossip/sender.rs @@ -20,6 +20,7 @@ pub struct GossipSender { pub(crate) timeout_config: TimeoutConfig, } +/// Internal actor for gossip send operations. #[derive(Debug)] pub struct GossipSenderActor { gossip_sender: iroh_gossip::api::GossipSender, diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index ae9e502..91215be 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -8,7 +8,7 @@ use iroh::EndpointId; use tokio::time::sleep; use crate::{ - GossipSender, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, + GossipSender, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher, config::BootstrapConfig, crypto::Record, gossip::{GossipRecordContent, receiver::GossipReceiver}, @@ -119,9 +119,7 @@ impl BootstrapActor { record_content, record_publisher.signing_key(), ) { - tokio::spawn(async move { - let _ = record_creator.publish_record(record).await; - }); + publish_record_fire_and_forget(record_creator, record); } } @@ -182,9 +180,7 @@ impl BootstrapActor { record_content, record_publisher.signing_key(), ) { - tokio::spawn(async move { - let _ = record_creator.publish_record(record).await; - }); + publish_record_fire_and_forget(record_creator, record); } } tokio::select! { @@ -325,9 +321,7 @@ impl BootstrapActor { }, record_publisher.signing_key(), ) { - tokio::spawn(async move { - let _ = record_creator.publish_record(record).await; - }); + publish_record_fire_and_forget(record_creator, record); } } tokio::select! { @@ -352,3 +346,11 @@ impl BootstrapActor { Ok(receiver) } } + +fn publish_record_fire_and_forget(record_publisher: RecordPublisher, record: Record) { + tokio::spawn(async move { + if let Err(err) = record_publisher.publish_record(record).await { + tracing::warn!("Failed to publish record: {:?}", err); + } + }); +} diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index c4f7b38..e0ec867 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -35,6 +35,26 @@ impl FromStr for TopicId { } } +impl From<&str> for TopicId { + fn from(s: &str) -> Self { + Self::new(s.to_string()) + } +} + +impl From for TopicId { + fn from(s: String) -> Self { + Self::new(s) + } +} + +impl TryFrom> for TopicId { + type Error = anyhow::Error; + + fn try_from(bytes: Vec) -> Result { + Ok(Self::new(String::from_utf8(bytes)?)) + } +} + impl TopicId { /// Create a new topic ID from a string. /// @@ -288,7 +308,7 @@ mod tests { .secret_key(secret_key.clone()) .bind() .await - .unwrap(); + .expect("failed to bind endpoint"); let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); let topic_id = crate::TopicId::new("shutdown-receiver-test".to_string()); @@ -304,10 +324,10 @@ mod tests { let topic = crate::Topic::new(record_publisher, gossip.clone(), true) .await - .unwrap(); + .expect("failed to create topic"); let cancel_token = topic.cancel_token(); - let (_sender, receiver) = topic.split().await.unwrap(); + let (_sender, receiver) = topic.split().await.expect("failed to split topic"); // Clone the receiver before shutdown, this survivor must not hang let mut survivor = receiver.clone(); @@ -359,7 +379,7 @@ mod tests { .secret_key(secret_key.clone()) .bind() .await - .unwrap(); + .expect("failed to bind edpoint"); let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); let topic_id = crate::TopicId::new("my-iroh-gossip-topic".to_string()); @@ -375,7 +395,7 @@ mod tests { let topic = crate::Topic::new(record_publisher, gossip.clone(), true) .await - .unwrap(); + .expect("failed to create Topic"); let cancel_token = topic.cancel_token(); @@ -389,7 +409,7 @@ mod tests { tokio::time::timeout(std::time::Duration::from_secs(5), cancel_token.cancelled()) .await - .unwrap(); + .expect("cancel token timed out"); assert!(cancel_token.is_cancelled()); } diff --git a/src/lib.rs b/src/lib.rs index 1277556..63bf066 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,7 @@ pub const MAX_MESSAGE_HASHES: usize = 5; /// ``` #[doc(hidden)] pub fn unix_minute(minute_offset: i64) -> u64 { - ((chrono::Utc::now().timestamp() / 60) + minute_offset) + ((chrono::Utc::now().timestamp() / 60).saturating_add(minute_offset)) .try_into() .expect("timestamp overflow") } diff --git a/tests/tests.rs b/tests/tests.rs index d0d5fbf..deb7675 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -23,14 +23,14 @@ fn test_record_serialization_roundtrip() { }; let record = Record::sign(topic, unix_minute, record_content.clone(), &signing_key) - .expect("Failed to sign record"); + .expect("failed to sign record"); // Test serialization roundtrip let bytes = record.to_bytes(); - let deserialized = Record::from_bytes(bytes).expect("Failed to deserialize record"); + let deserialized = Record::from_bytes(bytes).expect("failed to deserialize record"); let deserialized_content: GossipRecordContent = - deserialized.content().expect("Failed to get content"); + deserialized.content().expect("failed to get content"); assert_eq!(record.topic(), deserialized.topic()); assert_eq!(record.unix_minute(), deserialized.unix_minute()); @@ -59,7 +59,8 @@ fn test_record_verification() { last_message_hashes, }; - let record = Record::sign(topic, unix_minute, record_content, &signing_key).unwrap(); + let record = Record::sign(topic, unix_minute, record_content, &signing_key) + .expect("failed to sign record"); // Valid verification should pass assert!(record.verify(&topic, unix_minute).is_ok()); @@ -87,14 +88,16 @@ fn test_encrypted_record_roundtrip() { }; let record = Record::sign(topic, unix_minute, record_content.clone(), &signing_key) - .expect("Failed to sign record"); + .expect("failed to sign record"); // Test encryption/decryption roundtrip let encrypted = record.encrypt(&encryption_key); - let decrypted = encrypted.decrypt(&encryption_key).unwrap(); + let decrypted = encrypted + .decrypt(&encryption_key) + .expect("failed to decrypt record"); let deserialized_content: GossipRecordContent = - decrypted.content().expect("Failed to get content"); + decrypted.content().expect("failed to get content"); assert_eq!(record.topic(), decrypted.topic()); assert_eq!(record.unix_minute(), decrypted.unix_minute()); @@ -125,19 +128,19 @@ fn test_encrypted_record_serialization() { }; let record = Record::sign(topic, unix_minute, record_content, &signing_key) - .expect("Failed to sign record"); + .expect("failed to sign record"); let encrypted = record.encrypt(&encryption_key); // Test serialization roundtrip let bytes = encrypted.to_bytes(); let deserialized = - EncryptedRecord::from_bytes(bytes).expect("Failed to deserialize encrypted record"); + EncryptedRecord::from_bytes(bytes).expect("failed to deserialize encrypted record"); // Should be able to decrypt the deserialized version let decrypted = deserialized .decrypt(&encryption_key) - .expect("Failed to decrypt record"); + .expect("failed to decrypt record"); assert_eq!(record.topic(), decrypted.topic()); assert_eq!(record.unix_minute(), decrypted.unix_minute()); } @@ -181,7 +184,7 @@ fn test_unix_minute_function() { #[test] fn test_topic_signing_keypair_deterministic() { - let topic_id = TopicId::from_str("test-topic").unwrap(); + let topic_id = TopicId::from_str("test-topic").expect("failed to create TopicId from_str"); let unix_minute = 12345u64; let key1 = signing_keypair(&topic_id, unix_minute); @@ -197,7 +200,7 @@ fn test_topic_signing_keypair_deterministic() { #[test] fn test_topic_encryption_keypair_deterministic() { - let topic_id = TopicId::from_str("test-topic").unwrap(); + let topic_id = TopicId::from_str("test-topic").expect("failed to create TopicId from_str"); let unix_minute = 12345u64; let initial_secret_hash = [1u8; 32]; let rotation = RotationHandle::new(DefaultSecretRotation); @@ -215,7 +218,7 @@ fn test_topic_encryption_keypair_deterministic() { #[test] fn test_topic_salt_deterministic() { - let topic_id = TopicId::from_str("test-topic").unwrap(); + let topic_id = TopicId::from_str("test-topic").expect("failed to create TopicId from_str"); let unix_minute = 12345u64; let salt1 = salt(&topic_id, unix_minute); @@ -266,7 +269,7 @@ async fn test_multiple_receivers_all_get_events() { .secret_key(secret_a) .bind() .await - .unwrap(); + .expect("failed to bind endpoint A"); let gossip_a = iroh_gossip::net::Gossip::builder().spawn(endpoint_a.clone()); let _router_a = iroh::protocol::Router::builder(endpoint_a.clone()) .accept(iroh_gossip::ALPN, gossip_a.clone()) @@ -283,8 +286,8 @@ async fn test_multiple_receivers_all_get_events() { let topic_a = gossip_a .subscribe_and_join_with_auto_discovery_no_wait(rp_a) .await - .unwrap(); - let (sender_a, mut receiver_a) = topic_a.split().await.unwrap(); + .expect("failed to subscribe and join topic A"); + let (sender_a, mut receiver_a) = topic_a.split().await.expect("failed to split topic A"); // Peer B let secret_b = iroh::SecretKey::generate(&mut rand::rng()); @@ -293,7 +296,7 @@ async fn test_multiple_receivers_all_get_events() { .secret_key(secret_b) .bind() .await - .unwrap(); + .expect("failed to bind endpoint B"); let gossip_b = iroh_gossip::net::Gossip::builder().spawn(endpoint_b.clone()); let _router_b = iroh::protocol::Router::builder(endpoint_b.clone()) .accept(iroh_gossip::ALPN, gossip_b.clone()) @@ -310,21 +313,27 @@ async fn test_multiple_receivers_all_get_events() { let topic_b = gossip_b .subscribe_and_join_with_auto_discovery(rp_b) .await - .unwrap(); - let (sender_b, mut receiver_b) = topic_b.split().await.unwrap(); + .expect("failed to subscribe and join topic B"); + let (sender_b, mut receiver_b) = topic_b.split().await.expect("failed to split topic B"); // Join peers sender_a .join_peers(vec![endpoint_b.id()], None) .await - .unwrap(); + .expect("failed to join peers from sender A"); sender_b .join_peers(vec![endpoint_a.id()], None) .await - .unwrap(); + .expect("failed to join peers from sender B"); - receiver_a.joined().await.unwrap(); - receiver_b.joined().await.unwrap(); + receiver_a + .joined() + .await + .expect("failed to wait for receiver A to join"); + receiver_b + .joined() + .await + .expect("failed to wait for receiver B to join"); let receivers: Vec = (0..N) .map(|_| receiver_b.clone()) @@ -362,18 +371,18 @@ async fn test_multiple_receivers_all_get_events() { sender_a .broadcast(format!("msg-a-{i}").into_bytes()) .await - .unwrap(); + .expect("failed to broadcast from sender A"); sender_b .broadcast(format!("msg-b-{i}").into_bytes()) .await - .unwrap(); + .expect("failed to broadcast from sender B"); } for (i, handle) in handles.into_iter().enumerate() { let received = tokio::time::timeout(std::time::Duration::from_secs(60), handle) .await - .unwrap_or_else(|_| panic!("receiver {i} timed out")) - .unwrap(); + .expect(&format!("receiver {i} timed out")) + .expect(&format!("receiver {i} panicked")); assert_eq!( received.len(), From 1ba92ffd6aba08ace680307cdeb25fe1904f398b Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sat, 18 Apr 2026 00:45:24 +0200 Subject: [PATCH 18/30] chore: more review fixes --- PROTOCOL.md | 6 +-- examples/chat.rs | 2 +- examples/chat_no_wait.rs | 2 +- examples/e2e_test.rs | 2 +- examples/secret_rotation.rs | 2 +- src/config.rs | 18 ++++---- src/gossip/merge/bubble.rs | 4 +- src/gossip/merge/message_overlap.rs | 4 +- src/gossip/receiver.rs | 72 ++++++++++++++++++----------- src/gossip/topic/publisher.rs | 4 +- src/gossip/topic/topic.rs | 22 ++++----- tests/tests.rs | 4 +- 12 files changed, 83 insertions(+), 59 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index a5c3e38..cccf669 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -16,7 +16,7 @@ The publishing procedure is a rate-limited mechanism that prevents DHT overload - Use the same key derivation as bootstrap: - Derive signing keypair: `keypair_seed = SHA512(topic_hash + unix_minute)[..32]` - Derive encryption keypair: `enc_keypair_seed = secret_rotation_function.get_unix_minute_secret(topic_hash, unix_minute, initial_secret_hash)` - - Calculate salt: `salt = SHA512(topic_hash + unix_minute)[..32]` + - Calculate salt: `salt = SHA512("salt" + topic_hash + unix_minute)[..32]` - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout 2. **Rate Limiting Check** @@ -342,7 +342,7 @@ A one-time key encryption scheme is used to protect record content while allowin **Signing Keypair (Public DHT Discovery):** - Purpose: Used for DHT mutable record signing and salt calculation -- Derivation: `signing_keypair_seed = SHA512("salt" + topic_hash + unix_minute)[..32]` +- Derivation: `signing_keypair_seed = SHA512(topic_hash + unix_minute)[..32]` - Key: `ed25519_dalek::SigningKey::from_bytes(signing_keypair_seed)` - Public: This keypair is deterministic and publicly derivable @@ -354,7 +354,7 @@ A one-time key encryption scheme is used to protect record content while allowin **Salt Calculation:** - Purpose: Used as salt parameter for DHT mutable record storage -- Derivation: `salt = SHA512(topic_hash + unix_minute)[..32]` +- Derivation: `salt = SHA512("salt" + topic_hash + unix_minute)[..32]` ### Encryption Process diff --git a/examples/chat.rs b/examples/chat.rs index a3d2093..1bcad30 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -63,7 +63,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(event) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", diff --git a/examples/chat_no_wait.rs b/examples/chat_no_wait.rs index fd59556..45f0916 100644 --- a/examples/chat_no_wait.rs +++ b/examples/chat_no_wait.rs @@ -47,7 +47,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(event) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", diff --git a/examples/e2e_test.rs b/examples/e2e_test.rs index e9d177a..578e260 100644 --- a/examples/e2e_test.rs +++ b/examples/e2e_test.rs @@ -44,7 +44,7 @@ async fn main() -> Result<()> { .await?; tokio::spawn(async move { - while let Some(event) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { println!("event: {event:?}"); } }); diff --git a/examples/secret_rotation.rs b/examples/secret_rotation.rs index 9acbbdb..e2263b1 100644 --- a/examples/secret_rotation.rs +++ b/examples/secret_rotation.rs @@ -66,7 +66,7 @@ async fn main() -> Result<()> { // Spawn listener for incoming messages tokio::spawn(async move { - while let Some(event) = gossip_receiver.next().await { + while let Ok(event) = gossip_receiver.next().await { if let Event::Received(msg) = event { println!( "\nMessage from {}: {}", diff --git a/src/config.rs b/src/config.rs index 833d00d..5bc3004 100644 --- a/src/config.rs +++ b/src/config.rs @@ -90,7 +90,7 @@ pub struct DhtConfigBuilder { } impl DhtConfigBuilder { - /// Number of DHT operation retry attempts (first attempt + retries). Default: 3. + /// Number of retries after the initial attempt (total attempts = 1 + retries). Default: 3. pub fn retries(mut self, retries: usize) -> Self { self.config.retries = retries; self @@ -292,26 +292,26 @@ impl BootstrapConfigBuilder { } /// How long to wait when no peers are found before retrying. Default: 1500ms. - pub fn no_peers_retry_interval(mut self, interval_ms: Duration) -> Self { - self.config.no_peers_retry_interval = interval_ms; + pub fn no_peers_retry_interval(mut self, interval: Duration) -> Self { + self.config.no_peers_retry_interval = interval; self } /// How long to wait after joining a peer before attempting to join another. Default: 100ms. - pub fn per_peer_join_settle_time(mut self, interval_ms: Duration) -> Self { - self.config.per_peer_join_settle_time = interval_ms; + pub fn per_peer_join_settle_time(mut self, interval: Duration) -> Self { + self.config.per_peer_join_settle_time = interval; self } /// How long to wait after joining a peer before checking if joined successfully. Default: 500ms. - pub fn join_confirmation_wait_time(mut self, interval_ms: Duration) -> Self { - self.config.join_confirmation_wait_time = interval_ms; + pub fn join_confirmation_wait_time(mut self, interval: Duration) -> Self { + self.config.join_confirmation_wait_time = interval; self } /// How long to wait between DHT discovery attempts. Default: 2000ms. - pub fn discovery_poll_interval(mut self, interval_ms: Duration) -> Self { - self.config.discovery_poll_interval = interval_ms; + pub fn discovery_poll_interval(mut self, interval: Duration) -> Self { + self.config.discovery_poll_interval = interval; self } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index b3a9df1..6262942 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -83,7 +83,9 @@ impl BubbleMergeActor { } _ = self.ticker.tick() => { tracing::debug!("BubbleMerge: tick fired, checking for bubbles"); - let _ = self.merge().await; + if let Err(e) = self.merge().await { + tracing::warn!("BubbleMerge: error during merge: {:?}", e); + } let jitter_ms = if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 900258b..3e5f0bf 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -74,7 +74,9 @@ impl MessageOverlapMergeActor { } _ = self.ticker.tick() => { tracing::debug!("MessageOverlapMerge: tick fired, checking for split-brain"); - let _ = self.merge().await; + if let Err(e) = self.merge().await { + tracing::warn!("MessageOverlapMerge: error during merge: {:?}", e); + } let jitter_ms = if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index eae44b2..c9f614b 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashSet, VecDeque}, + fmt::Display, sync::Arc, }; @@ -66,6 +67,35 @@ pub struct GossipReceiverActor { join_channel_sender: tokio::sync::broadcast::Sender>, } +#[derive(Debug)] +pub enum ChannelError { + Closed, + Lagged(u64), +} + +impl From for ChannelError { + fn from(err: tokio::sync::broadcast::error::RecvError) -> Self { + match err { + tokio::sync::broadcast::error::RecvError::Closed => ChannelError::Closed, + tokio::sync::broadcast::error::RecvError::Lagged(skipped) => { + ChannelError::Lagged(skipped) + } + } + } +} + +impl std::error::Error for ChannelError {} +impl Display for ChannelError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ChannelError::Closed => write!(f, "channel closed"), + ChannelError::Lagged(skipped) => { + write!(f, "channel lagged, skipped {} messages", skipped) + } + } + } +} + impl GossipReceiver { /// Create a new gossip receiver from an iroh topic receiver. pub fn new( @@ -113,39 +143,27 @@ impl GossipReceiver { } /// Receive the next gossip event. - /// - /// Returns `None` if the receiver is closed. - pub async fn next(&mut self) -> Option { - loop { - match self.next_channel_receiver.recv().await { - Ok(event) => return event, - Err(err) => match err { - tokio::sync::broadcast::error::RecvError::Closed => return None, - tokio::sync::broadcast::error::RecvError::Lagged(skipped) => tracing::warn!( - "GossipReceiver: event stream lagged, {skipped} events may have been missed" - ), - }, + pub async fn next(&mut self) -> Result { + match self.next_channel_receiver.recv().await { + Ok(event) => match event { + Some(event) => Ok(event), + None => Err(ChannelError::Closed), } + Err(err) => Err(err.into()), } } - /// Waits for a NeighborUp or a message Received event then returns `Some(())`. - /// - /// Returns None if the receiver is closed. - pub async fn joined(&mut self) -> Option<()> { + /// Waits for a NeighborUp or a message Received event then returns `Ok(())`. + pub async fn joined(&mut self) -> Result<(), ChannelError> { if self.is_joined().await.unwrap_or(false) { - return Some(()); + return Ok(()); } - loop { - match self.join_channel_receiver.recv().await { - Ok(event) => return event, - Err(err) => match err { - tokio::sync::broadcast::error::RecvError::Closed => return None, - tokio::sync::broadcast::error::RecvError::Lagged(skipped) => tracing::warn!( - "GossipReceiver: join event stream lagged, {skipped} events may have been missed" - ), - }, - } + match self.join_channel_receiver.recv().await { + Ok(event) => match event { + Some(event) => Ok(event), + None => Err(ChannelError::Closed), + }, + Err(err) => Err(err.into()), } } diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 880c0e0..9bd7707 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -72,7 +72,9 @@ impl PublisherActor { } _ = self.ticker.tick() => { tracing::debug!("Publisher: tick fired, attempting to publish"); - let _ = self.publish().await; + if let Err(e) = self.publish().await { + tracing::warn!("Publisher: failed to publish record: {:?}", e); + } let jitter_ms = if self.max_jitter.as_millis() > 0 { rand::random::() % self.max_jitter.as_millis() } else { diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index e0ec867..97c7274 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -338,19 +338,19 @@ mod tests { // not hang. If the broadcast channel didn't close, this would block forever let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.next()) .await - .expect("next() hung after shutdown — broadcast channel didn't close"); - assert!(result.is_none(), "expected None from next() after shutdown"); + .expect("next() hung after shutdown - broadcast channel didn't close"); + assert!(result.is_err(), "expected Err from next() after shutdown"); - // joined() must also return None after shutdown + // joined() must also return Err after shutdown let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.joined()) .await - .expect("joined() hung after shutdown — broadcast channel didn't close"); + .expect("joined() hung after shutdown - broadcast channel didn't close"); assert!( - result.is_none(), - "expected None from joined() after shutdown" + result.is_err(), + "expected Err from joined() after shutdown" ); - // A clone made after shutdown must also return None immediately + // A clone made after shutdown must also return Err immediately // (WeakSender::upgrade fails -> gets an already closed channel) let mut late_clone = survivor.clone(); @@ -358,16 +358,16 @@ mod tests { .await .expect("next() hung on post shutdown clone, WeakSender upgrade should fail"); assert!( - result.is_none(), - "expected None from next() on post shutdown clone" + result.is_err(), + "expected Err from next() on post shutdown clone" ); let result = tokio::time::timeout(std::time::Duration::from_secs(5), late_clone.joined()) .await .expect("joined() hung on post shutdown clone, WeakSender upgrade should fail"); assert!( - result.is_none(), - "expected None from joined() on post shutdown clone" + result.is_err(), + "expected Err from joined() on post shutdown clone" ); } diff --git a/tests/tests.rs b/tests/tests.rs index deb7675..697b76f 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -353,10 +353,10 @@ async fn test_multiple_receivers_all_get_events() { while received.len() < MSG_COUNT { match tokio::time::timeout(std::time::Duration::from_secs(30), rx.next()).await { - Ok(Some(iroh_gossip::api::Event::Received(msg))) => { + Ok(Ok(iroh_gossip::api::Event::Received(msg))) => { received.push(msg.content.to_vec()); } - Ok(Some(_)) => continue, + Ok(Ok(_)) => continue, other => panic!("receiver {i}: unexpected result: {other:?}"), } } From 68557add9a85b7278e410ea7e3e94c347894e4e5 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sat, 18 Apr 2026 03:07:56 +0200 Subject: [PATCH 19/30] chore: more review stuff + one less dht get call on startup (cached records for dht put) --- ARCHITECTURE.md | 2 +- PROTOCOL.md | 14 +++--- examples/chat.rs | 18 ++++++-- src/crypto/record.rs | 68 +++++++++++++++++++---------- src/gossip/merge/bubble.rs | 20 ++++----- src/gossip/merge/message_overlap.rs | 22 +++++----- src/gossip/receiver.rs | 8 ++-- src/gossip/topic/bootstrap.rs | 56 +++++++++++++++++------- src/gossip/topic/topic.rs | 2 +- tests/tests.rs | 4 +- 10 files changed, 138 insertions(+), 76 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 204dff1..db014d8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -96,7 +96,7 @@ sequenceDiagram Node->>Node: decrypt, verify, filter(not self) alt candidates exist loop each candidate - Node->>Gossip: join_peers([node_id]) + Node->>Gossip: join_peers([pub_key]) Node->>Node: sleep per_peer_join_settle_time (100ms) Gossip-->>Node: NeighborUp? end diff --git a/PROTOCOL.md b/PROTOCOL.md index cccf669..0996ecc 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -34,7 +34,7 @@ The publishing procedure is a rate-limited mechanism that prevents DHT overload 4. **Record Signing and Publishing** - Create signed record using `Record::sign()`: - - Include: `topic_hash`, `unix_minute`, `node_id`, content (serialized `active_peers` + `last_message_hashes`) + - Include: `topic_hash`, `unix_minute`, `pub_key`, content (serialized `active_peers` + `last_message_hashes`) - Sign with node's ed25519 signing key - Encrypt record using one-time encryption key - Publish to DHT via `Dht::put_mutable()` with retry support @@ -64,7 +64,7 @@ flowchart TD L --> M[Fill last_message_hashes with up to 5 recent hashes] M --> N[Create Signed Record] - N --> O[Sign with: topic + unix_minute + node_id + content] + N --> O[Sign with: topic + unix_minute + pub_key + content] O --> P[Encrypt Record with One-Time Key] P --> Q[Publish to DHT with retries] @@ -120,7 +120,7 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout - Decrypt each record using the encryption keypair - Verify signature, unix_minute, and topic hash - - Filter out own records (matching node_id) + - Filter out own records (matching pub_key) 5. **If no valid Records Found** - If no valid records found and haven't published in this unix minute, publish own record @@ -128,14 +128,14 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne 6. **If valid Records Found** - Extract bootstrap nodes from records: - - Include `record.node_id` (the publisher) + - Include `record.pub_key` (the publisher) - Include all non-zero entries from `record.active_peers[5]` - Convert byte arrays to valid `iroh::EndpointId` instances 7. **Connection Attempts** - Check again if already connected (someone might have connected to us) - If not connected, attempt to join peers one by one: - - Call `gossip_sender.join_peers(vec![node_id])` for each bootstrap node + - Call `gossip_sender.join_peers(vec![pub_key])` for each bootstrap node - Wait `per_peer_join_settle_time` (default 100ms) between attempts - Break early if connection established @@ -186,12 +186,12 @@ flowchart TD N -- Yes --> R[Decrypt & Verify Records] R --> S[Filter Valid Records] - S --> T[Extract Bootstrap Nodes: node_id + active_peers] + S --> T[Extract Bootstrap Nodes: pub_key + active_peers] T --> V{Already Connected?} V -- Yes --> Z V -- No --> W[Join Peers One by One] - W --> X[For each bootstrap node: join_peers = node_id] + W --> X[For each bootstrap node: join_peers = pub_key] X --> Y[Wait per_peer_join_settle_time = 100ms] Y --> AA2{Connected?} AA2 -- Yes --> Z diff --git a/examples/chat.rs b/examples/chat.rs index 1bcad30..a687be7 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -6,7 +6,7 @@ use ed25519_dalek::SigningKey; // Imports from distributed-topic-tracker use distributed_topic_tracker::{ - AutoDiscoveryGossip, Config, RecordPublisher, TopicId, + AutoDiscoveryGossip, BootstrapConfig, Config, RecordPublisher, TopicId, }; #[tokio::main] @@ -49,7 +49,13 @@ async fn main() -> Result<()> { signing_key.clone(), None, initial_secret, - Config::default(), + Config::builder() + .bootstrap_config( + BootstrapConfig::builder() + .check_last_minute_record_first_on_startup(false) + .build(), + ) + .build(), ); // Split into sink (sending) and stream (receiving) @@ -83,7 +89,13 @@ async fn main() -> Result<()> { print!("\n> "); stdin.read_line(&mut buffer).unwrap(); gossip_sender - .broadcast(buffer.clone().replace("\n", "").replace("\r\n", "").into()) + .broadcast( + buffer + .clone() + .trim_end_matches(&['\r', '\n'][..]) + .as_bytes() + .to_vec(), + ) .await .unwrap(); println!(" - (sent)"); diff --git a/src/crypto/record.rs b/src/crypto/record.rs index 95068f5..171fbd3 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -37,6 +37,8 @@ pub struct Record { pub struct RecordContent(pub Vec); impl RecordContent { + const MAX_SIZE: usize = 2048; + /// Deserialize using postcard codec. /// /// # Example @@ -50,9 +52,14 @@ impl RecordContent { /// Serialize from an arbitrary type using postcard. pub fn from_arbitrary(from: &T) -> anyhow::Result { - Ok(Self( - postcard::to_allocvec(from).map_err(|e| anyhow::anyhow!(e))?, - )) + let serialized = postcard::to_allocvec(from).map_err(|e| anyhow::anyhow!(e))?; + if serialized.len() > Self::MAX_SIZE { + bail!( + "record content exceeds maximum size of {} bytes", + Self::MAX_SIZE + ) + } + Ok(Self(serialized)) } } @@ -212,12 +219,27 @@ impl RecordPublisher { /// Checks existing record count for this time slot and skips publishing if /// `self.config.bootstrap_config().max_bootstrap_records()` limit reached. pub async fn publish_record(&self, record: Record) -> Result<()> { - let records = self - .get_records(record.unix_minute()) - .await - .iter() - .cloned() - .collect::>(); + self.publish_record_cached_records(record, None).await + } + + /// Publish a record to the DHT (using cached get_records) if slot capacity allows. + /// + /// Checks existing record count for this time slot and skips publishing if + /// `self.config.bootstrap_config().max_bootstrap_records()` limit reached. + pub async fn publish_record_cached_records( + &self, + record: Record, + cached_records: Option>, + ) -> Result<()> { + let records = if let Some(records) = cached_records { + records + } else { + self.get_records(record.unix_minute()) + .await + .iter() + .cloned() + .collect::>() + }; tracing::debug!( "RecordPublisher: found {} existing records for unix_minute {}", @@ -283,11 +305,7 @@ impl RecordPublisher { // Get records, decrypt and verify let records_iter = self .dht - .get( - topic_sign.verifying_key(), - Some(salt.to_vec()), - None, - ) + .get(topic_sign.verifying_key(), Some(salt.to_vec()), None) .await .unwrap_or_default(); @@ -302,7 +320,7 @@ impl RecordPublisher { |record| match EncryptedRecord::from_bytes(record.value().to_vec()) { Ok(encrypted_record) => match encrypted_record.decrypt(&encryption_key) { Ok(record) => match record.verify(&self.topic_id.hash(), unix_minute) { - Ok(_) => match record.node_id().eq(self.pub_key.as_bytes()) { + Ok(_) => match record.pub_key().eq(self.pub_key.as_bytes()) { true => None, false => Some(record), }, @@ -354,8 +372,14 @@ impl EncryptedRecord { } let (encrypted_record_len, buf) = buf.split_at(4); let encrypted_record_len = u32::from_le_bytes(encrypted_record_len.try_into()?); - - if buf.len() != encrypted_record_len as usize + 88 /* encrypted_decryption_key.len() == 88 */ { + const ENCRYPTED_KEY_LENGTH: usize = 88; + let expected_payload_len = encrypted_record_len + .checked_add(ENCRYPTED_KEY_LENGTH as u32) + .ok_or_else(|| anyhow::anyhow!("encrypted record length overflow"))?; + if encrypted_record_len > RecordContent::MAX_SIZE as u32 { + bail!("encrypted record length exceeds maximum allowed size") + } + if buf.len() != expected_payload_len as usize { bail!("buffer length does not match expected encrypted record length") } let (encrypted_record, encrypted_decryption_key) = @@ -399,7 +423,7 @@ impl Record { } let (topic, buf) = buf.split_at(32); let (unix_minute, buf) = buf.split_at(8); - let (node_id, buf) = buf.split_at(32); + let (pub_key, buf) = buf.split_at(32); let (record_content, buf) = buf.split_at(buf.len() - 64); let (signature, buf) = buf.split_at(64); @@ -411,7 +435,7 @@ impl Record { Ok(Self { topic: topic.try_into()?, unix_minute: u64::from_le_bytes(unix_minute.try_into()?), - pub_key: node_id.try_into()?, + pub_key: pub_key.try_into()?, content: RecordContent(record_content.to_vec()), signature: signature.try_into()?, }) @@ -440,9 +464,9 @@ impl Record { let record_bytes = self.to_bytes(); let signature_data = record_bytes[..record_bytes.len() - 64].to_vec(); let signature = ed25519_dalek::Signature::from_bytes(&self.signature); - let node_id = ed25519_dalek::VerifyingKey::from_bytes(&self.pub_key)?; + let pub_key = ed25519_dalek::VerifyingKey::from_bytes(&self.pub_key)?; - node_id.verify_strict(signature_data.as_slice(), &signature)?; + pub_key.verify_strict(signature_data.as_slice(), &signature)?; Ok(()) } @@ -477,7 +501,7 @@ impl Record { } /// Get the node ID (publisher's public key). - pub fn node_id(&self) -> [u8; 32] { + pub fn pub_key(&self) -> [u8; 32] { self.pub_key } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 6262942..28c35ff 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -124,10 +124,10 @@ impl BubbleMergeActor { neighbors.len(), self.min_neighbors ); - let node_ids = records + let pub_keys = records .iter() .flat_map(|record| { - let mut endpoint_ids = if let Ok(content) = + let mut pub_keys = if let Ok(content) = record.content::() { content @@ -136,7 +136,7 @@ impl BubbleMergeActor { .filter_map(|&active_peer| { if active_peer == [0; 32] || neighbors.contains(&active_peer) - || active_peer == record.node_id() + || active_peer == record.pub_key() || active_peer.eq(self.record_publisher.pub_key().as_bytes()) { None @@ -148,25 +148,25 @@ impl BubbleMergeActor { } else { vec![] }; - if let Ok(endpoint_id) = EndpointId::from_bytes(&record.node_id()) - && endpoint_id + if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) + && pub_key != EndpointId::from_verifying_key(self.record_publisher.pub_key()) { - endpoint_ids.push(endpoint_id); + pub_keys.push(pub_key); } - endpoint_ids + pub_keys }) .collect::>(); tracing::debug!( "BubbleMerge: found {} potential peers to join", - node_ids.len() + pub_keys.len() ); - if !node_ids.is_empty() { + if !pub_keys.is_empty() { self.gossip_sender .join_peers( - node_ids.iter().cloned().collect::>(), + pub_keys.iter().cloned().collect::>(), Some(self.max_join_peers), ) .await?; diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 3e5f0bf..dff3516 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -131,44 +131,44 @@ impl MessageOverlapMergeActor { if !peers_to_join.is_empty() { let active_neighbors = self.gossip_receiver.neighbors().await?; - let node_ids = peers_to_join + let pub_keys = peers_to_join .iter() .flat_map(|&record| { let mut peers = vec![]; - if let Ok(endpoint_id) = EndpointId::from_bytes(&record.node_id()) - && endpoint_id + if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) + && pub_key != EndpointId::from_verifying_key(self.record_publisher.pub_key()) { - peers.push(endpoint_id); + peers.push(pub_key); } if let Ok(content) = record.content::() { for active_peer in content.active_peers { if active_peer == [0; 32] { continue; } - if let Ok(endpoint_id) = EndpointId::from_bytes(&active_peer) - && endpoint_id + if let Ok(pub_key) = EndpointId::from_bytes(&active_peer) + && pub_key != EndpointId::from_verifying_key( self.record_publisher.pub_key(), ) { - peers.push(endpoint_id); + peers.push(pub_key); } } } peers }) - .filter(|node_id| !active_neighbors.contains(node_id)) + .filter(|pub_key| !active_neighbors.contains(pub_key)) .collect::>(); tracing::debug!( - "MessageOverlapMerge: attempting to join {} node_ids with no overlapping messages", - node_ids.len() + "MessageOverlapMerge: attempting to join {} pub_keys with no overlapping messages", + pub_keys.len() ); self.gossip_sender .join_peers( - node_ids.iter().cloned().collect::>(), + pub_keys.iter().cloned().collect::>(), Some(self.max_join_peers), ) .await?; diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index c9f614b..2381219 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -218,12 +218,12 @@ impl GossipReceiverActor { } self.join_channel_sender.send(Some(())).ok(); } - iroh_gossip::api::Event::NeighborUp(node_id) => { - tracing::debug!("GossipReceiver: neighbor UP: {}", node_id); + iroh_gossip::api::Event::NeighborUp(pub_key) => { + tracing::debug!("GossipReceiver: neighbor UP: {}", pub_key); self.join_channel_sender.send(Some(())).ok(); } - iroh_gossip::api::Event::NeighborDown(node_id) => { - tracing::debug!("GossipReceiver: neighbor DOWN: {}", node_id); + iroh_gossip::api::Event::NeighborDown(pub_key) => { + tracing::debug!("GossipReceiver: neighbor DOWN: {}", pub_key); } iroh_gossip::api::Event::Lagged => { tracing::debug!("GossipReceiver: event stream lagged"); diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 91215be..5a5a2a1 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -119,7 +119,7 @@ impl BootstrapActor { record_content, record_publisher.signing_key(), ) { - publish_record_fire_and_forget(record_creator, record); + publish_record_fire_and_forget(record_creator, record, None); } } @@ -140,10 +140,12 @@ impl BootstrapActor { } // On the first try we check the prev unix minute, after that the current one + let mut use_cached_next = true; let unix_minute = crate::unix_minute( if last_published_unix_minute == 0 && bootstrap_config.check_last_minute_record_first_on_startup() { + use_cached_next = false; -1 } else { 0 @@ -152,7 +154,8 @@ impl BootstrapActor { // Unique, verified records for the unix minute let mut records = record_publisher.get_records(unix_minute - 1).await; - records.extend(record_publisher.get_records(unix_minute).await); + let current_records = record_publisher.get_records(unix_minute).await; + records.extend(current_records.clone()); tracing::debug!( "Bootstrap: fetched {} records for unix_minute {}", @@ -180,7 +183,15 @@ impl BootstrapActor { record_content, record_publisher.signing_key(), ) { - publish_record_fire_and_forget(record_creator, record); + publish_record_fire_and_forget( + record_creator, + record, + if use_cached_next { + Some(current_records.clone()) + } else { + None + }, + ); } } tokio::select! { @@ -193,11 +204,11 @@ impl BootstrapActor { // We found records - // Collect node ids from active_peers and record.node_id (of publisher) + // Collect node ids from active_peers and record.pub_key (of publisher) let bootstrap_nodes = records .iter() .flat_map(|record| { - let mut v = vec![record.node_id()]; + let mut v = vec![record.pub_key()]; if let Ok(record_content) = record.content::() { for peer in record_content.active_peers { if peer != [0; 32] { @@ -207,7 +218,7 @@ impl BootstrapActor { } v }) - .filter_map(|node_id| EndpointId::from_bytes(&node_id).ok()) + .filter_map(|pub_key| EndpointId::from_bytes(&pub_key).ok()) .collect::>(); tracing::debug!( @@ -230,12 +241,12 @@ impl BootstrapActor { break; } - // Instead of throwing everything into join_peers() at once we go node_id by node_id + // Instead of throwing everything into join_peers() at once we go pub_key by pub_key // again to disrupt as little nodes peer neighborhoods as possible. - for node_id in bootstrap_nodes.iter() { - match gossip_sender.join_peers(vec![*node_id], None).await { + for pub_key in bootstrap_nodes.iter() { + match gossip_sender.join_peers(vec![*pub_key], None).await { Ok(_) => { - tracing::debug!("Bootstrap: attempted to join peer {}", node_id); + tracing::debug!("Bootstrap: attempted to join peer {}", pub_key); tokio::select! { _ = sleep(bootstrap_config.per_peer_join_settle_time()) => {} @@ -248,7 +259,7 @@ impl BootstrapActor { { tracing::debug!( "Bootstrap: successfully joined via peer {}", - node_id + pub_key ); is_joined_ret = true; break; @@ -263,7 +274,7 @@ impl BootstrapActor { Err(e) => { tracing::debug!( "Bootstrap: failed to join peer {}: {:?}", - node_id, + pub_key, e ); continue; @@ -321,7 +332,15 @@ impl BootstrapActor { }, record_publisher.signing_key(), ) { - publish_record_fire_and_forget(record_creator, record); + publish_record_fire_and_forget( + record_creator, + record, + if use_cached_next { + Some(current_records) + } else { + None + }, + ); } } tokio::select! { @@ -347,9 +366,16 @@ impl BootstrapActor { } } -fn publish_record_fire_and_forget(record_publisher: RecordPublisher, record: Record) { +fn publish_record_fire_and_forget( + record_publisher: RecordPublisher, + record: Record, + cached_records: Option>, +) { tokio::spawn(async move { - if let Err(err) = record_publisher.publish_record(record).await { + if let Err(err) = record_publisher + .publish_record_cached_records(record, cached_records) + .await + { tracing::warn!("Failed to publish record: {:?}", err); } }); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 97c7274..976e701 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -71,7 +71,7 @@ impl TopicId { } /// Create from a pre-computed 32-byte hash. - pub fn from_bytes(bytes: &[u8; 32]) -> Self { + pub fn from_hash(bytes: &[u8; 32]) -> Self { Self(*bytes) } diff --git a/tests/tests.rs b/tests/tests.rs index 697b76f..dabd067 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -34,7 +34,7 @@ fn test_record_serialization_roundtrip() { assert_eq!(record.topic(), deserialized.topic()); assert_eq!(record.unix_minute(), deserialized.unix_minute()); - assert_eq!(record.node_id(), deserialized.node_id()); + assert_eq!(record.pub_key(), deserialized.pub_key()); assert_eq!( record_content.active_peers, deserialized_content.active_peers @@ -101,7 +101,7 @@ fn test_encrypted_record_roundtrip() { assert_eq!(record.topic(), decrypted.topic()); assert_eq!(record.unix_minute(), decrypted.unix_minute()); - assert_eq!(record.node_id(), decrypted.node_id()); + assert_eq!(record.pub_key(), decrypted.pub_key()); assert_eq!( record_content.active_peers, deserialized_content.active_peers From 44f2697585cedfd5cbbfd335aab938ae3906471e Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sat, 18 Apr 2026 20:12:45 +0200 Subject: [PATCH 20/30] fix: dht put seq_num + topic::new async_bootstrap + error passthrough + 'bootstrap loop break in start_bootstrap + bubble_merge neighbor aware --- Cargo.lock | 21 +++- Cargo.toml | 3 +- PROTOCOL.md | 6 +- src/crypto/record.rs | 108 +++++++++++------- src/dht.rs | 38 +++---- src/gossip/merge/bubble.rs | 8 +- src/gossip/merge/message_overlap.rs | 4 +- src/gossip/receiver.rs | 4 +- src/gossip/topic/bootstrap.rs | 14 ++- src/gossip/topic/publisher.rs | 6 +- src/gossip/topic/topic.rs | 169 ++++++++++++++++++++-------- 11 files changed, 253 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e69d3d..9c173d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,7 +685,7 @@ dependencies = [ [[package]] name = "distributed-topic-tracker" -version = "0.2.8" +version = "0.3.0" dependencies = [ "actor-helper", "anyhow", @@ -695,6 +695,7 @@ dependencies = [ "futures-lite", "iroh", "iroh-gossip", + "lru 0.17.0", "mainline", "postcard", "rand", @@ -1223,6 +1224,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapless" @@ -1800,7 +1806,7 @@ dependencies = [ "hyper-util", "iroh-base", "iroh-metrics", - "lru", + "lru 0.16.3", "n0-error", "n0-future", "noq", @@ -1946,6 +1952,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0b564323a0fb6d54b864f625ae139de9612e27edb944dda37c109f05aac531" +dependencies = [ + "hashbrown 0.17.0", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1971,7 +1986,7 @@ dependencies = [ "flume 0.11.1", "futures-lite", "getrandom 0.3.4", - "lru", + "lru 0.16.3", "serde", "serde_bencode", "serde_bytes", diff --git a/Cargo.toml b/Cargo.toml index 50ebaed..16cc181 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ homepage = "https://rustonbsd.github.io/2025/09/03/distributed-topic-tracker.htm readme = "README.md" keywords = ["networking"] categories = ["network-programming"] -version = "0.2.8" +version = "0.3.0" edition = "2024" [features] @@ -38,6 +38,7 @@ rand = { version = "0.9", default-features = false, features = ["std", "std_rng" actor-helper = { version = "0.3", features = ["tokio", "anyhow"] } postcard = "1" serde = { version = "1", default-features = false, features = ["std"] } +lru = { version = "0.17" } tracing = { version = "0.1", default-features = false, features = ["std"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "env-filter", "ansi"] } diff --git a/PROTOCOL.md b/PROTOCOL.md index 0996ecc..b304e3b 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -50,7 +50,7 @@ The publishing procedure is a rate-limited mechanism that prevents DHT overload flowchart TD A[Start Publishing Procedure] --> B[Get Current Unix Minute] B --> C[Derive Keys: Signing & Encryption] - C --> D[Calculate Salt: SHA512 = topic_hash + unix_minute] + C --> D[Calculate Salt: SHA512 = "salt" + topic_hash + unix_minute] D --> E[Query DHT: get_mutable = signing_pubkey, salt; Timeout: 10s] E --> F[Decrypt & Verify Records] @@ -116,7 +116,7 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne - Call `get_records()` for both `unix_minute - 1` and `unix_minute`: - Derive signing keypair: `keypair_seed = SHA512(topic_hash + unix_minute)[..32]` - Derive encryption keypair from shared secret - - Calculate salt: `SHA512(topic_hash + unix_minute)[..32]` + - Calculate salt: `SHA512("salt" + topic_hash + unix_minute)[..32]` - Query DHT: `get_mutable(signing_pubkey, salt)` with 10s timeout - Decrypt each record using the encryption keypair - Verify signature, unix_minute, and topic hash @@ -296,7 +296,7 @@ pub struct Record { unix_minute: u64, // floor(unixtime / 60) pub_key: [u8; 32], // publisher ed25519 public key - // Content (serialized via bincode) + // Content (serialized via postcard) content: RecordContent, // Vec wrapping serialized data // Signature diff --git a/src/crypto/record.rs b/src/crypto/record.rs index 171fbd3..26f4121 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -1,11 +1,13 @@ -use std::collections::HashSet; +use std::{collections::HashSet, sync::Arc}; use anyhow::{Result, bail}; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use ed25519_dalek_hpke::{Ed25519hpkeDecryption, Ed25519hpkeEncryption}; +use lru::LruCache; use serde::{Deserialize, Serialize}; use sha2::Digest; +use tokio::sync::Mutex; use crate::{Config, TopicId}; @@ -37,8 +39,6 @@ pub struct Record { pub struct RecordContent(pub Vec); impl RecordContent { - const MAX_SIZE: usize = 2048; - /// Deserialize using postcard codec. /// /// # Example @@ -52,14 +52,39 @@ impl RecordContent { /// Serialize from an arbitrary type using postcard. pub fn from_arbitrary(from: &T) -> anyhow::Result { - let serialized = postcard::to_allocvec(from).map_err(|e| anyhow::anyhow!(e))?; - if serialized.len() > Self::MAX_SIZE { - bail!( - "record content exceeds maximum size of {} bytes", - Self::MAX_SIZE - ) - } - Ok(Self(serialized)) + Ok(Self( + postcard::to_allocvec(from).map_err(|e| anyhow::anyhow!(e))?, + )) + } +} + +#[derive(Debug, Clone)] +struct NextDhtSeq { + inner: Arc>>, +} + +impl Default for NextDhtSeq { + fn default() -> Self { + Self::new(100).expect("capacity must be > 0") + } +} + +impl NextDhtSeq { + fn new(capacity: usize) -> Result { + Ok(Self { + inner: Arc::new(Mutex::new(LruCache::new( + std::num::NonZeroUsize::new(capacity) + .ok_or_else(|| anyhow::anyhow!("capacity must be > 0"))?, + ))), + }) + } + + async fn next(&self, unix_minute: u64) -> i64 { + let mut guard = self.inner.lock().await; + let current_seq = *guard.get(&unix_minute).unwrap_or(&0); + + guard.put(unix_minute, current_seq + 1); + current_seq } } @@ -69,6 +94,7 @@ impl RecordContent { #[derive(Debug, Clone)] pub struct RecordPublisher { dht: crate::dht::Dht, + dht_seq: NextDhtSeq, config: crate::config::Config, @@ -154,6 +180,7 @@ impl RecordPublisher { Self { dht: crate::dht::Dht::new(config.dht_config()), + dht_seq: NextDhtSeq::default(), config, topic_id: topic_id.into(), pub_key: signing_key.verifying_key(), @@ -235,7 +262,7 @@ impl RecordPublisher { records } else { self.get_records(record.unix_minute()) - .await + .await? .iter() .cloned() .collect::>() @@ -265,18 +292,19 @@ impl RecordPublisher { record.unix_minute, ); let encrypted_record = record.encrypt(&encryption_key); + let next_seq_num = self.dht_seq.next(record.unix_minute()).await; tracing::debug!( - "RecordPublisher: publishing record to DHT for unix_minute {}", + "RecordPublisher: publishing record to DHT for unix_minute {} with seq_num {next_seq_num}", record.unix_minute() ); self.dht .put_mutable( sign_key.clone(), - sign_key.verifying_key(), Some(salt.to_vec()), encrypted_record.to_bytes().to_vec(), + next_seq_num, ) .await?; @@ -287,7 +315,8 @@ impl RecordPublisher { /// Retrieve all verified records for a given time slot from the DHT. /// /// Filters out records from this publisher's own node ID. - pub async fn get_records(&self, unix_minute: u64) -> HashSet { + /// Dedub's records based on pub_key, keeping the newest sequence number per pub_key. + pub async fn get_records(&self, unix_minute: u64) -> Result> { tracing::debug!( "RecordPublisher: fetching records from DHT for unix_minute {}", unix_minute @@ -306,42 +335,43 @@ impl RecordPublisher { let records_iter = self .dht .get(topic_sign.verifying_key(), Some(salt.to_vec()), None) - .await - .unwrap_or_default(); + .await?; tracing::debug!( "RecordPublisher: received {} raw records from DHT", records_iter.len() ); - let verified_records = records_iter - .iter() - .filter_map( - |record| match EncryptedRecord::from_bytes(record.value().to_vec()) { - Ok(encrypted_record) => match encrypted_record.decrypt(&encryption_key) { - Ok(record) => match record.verify(&self.topic_id.hash(), unix_minute) { - Ok(_) => match record.pub_key().eq(self.pub_key.as_bytes()) { - true => None, - false => Some(record), - }, - Err(_) => None, - }, - Err(_) => None, - }, - Err(_) => None, - }, - ) - .collect::>(); - + let mut dedubed_records: Vec<(i64, Record)> = vec![]; + for item in records_iter.clone() { + if let Ok(encrypted_record) = EncryptedRecord::from_bytes(item.value().to_vec()) + && let Ok(record) = encrypted_record.decrypt(&encryption_key) + && record.verify(&self.topic_id.hash(), unix_minute).is_ok() + && !record.pub_key().eq(self.pub_key.as_bytes()) + { + if !dedubed_records + .iter() + .any(|(seq, r)| r.pub_key() == record.pub_key() && *seq >= item.seq()) + { + dedubed_records.push((item.seq(), record)); + } + } + } tracing::debug!( "RecordPublisher: verified {} records (filtered self)", - verified_records.len() + dedubed_records.len() ); - verified_records + + Ok(dedubed_records + .into_iter() + .map(|(_, record)| record) + .collect::>()) } } impl EncryptedRecord { + const MAX_SIZE: usize = 2048; + /// Decrypt using an Ed25519 HPKE private key. pub fn decrypt(&self, decryption_key: &ed25519_dalek::SigningKey) -> Result { let one_time_key_bytes: [u8; 32] = decryption_key @@ -376,7 +406,7 @@ impl EncryptedRecord { let expected_payload_len = encrypted_record_len .checked_add(ENCRYPTED_KEY_LENGTH as u32) .ok_or_else(|| anyhow::anyhow!("encrypted record length overflow"))?; - if encrypted_record_len > RecordContent::MAX_SIZE as u32 { + if encrypted_record_len > Self::MAX_SIZE as u32 { bail!("encrypted record length exceeds maximum allowed size") } if buf.len() != expected_payload_len as usize { diff --git a/src/dht.rs b/src/dht.rs index cb73d7f..b26f560 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -65,18 +65,18 @@ impl Dht { /// # Arguments /// /// * `signing_key` - Ed25519 secret key for signing - /// * `pub_key` - Ed25519 public key (used for routing) /// * `salt` - Optional salt for record slot /// * `data` - Record value to publish + /// * `next_unix_minute_seq` - Sequence number for the record (per unix_minute) pub async fn put_mutable( &self, signing_key: SigningKey, - pub_key: VerifyingKey, salt: Option>, data: Vec, + next_unix_minute_seq: i64, ) -> Result<()> { self.api - .call(act!(actor => actor.put_mutable(signing_key, pub_key, salt, data))) + .call(act!(actor => actor.put_mutable(signing_key, salt, data, next_unix_minute_seq))) .await } } @@ -111,9 +111,9 @@ impl DhtActor { pub async fn put_mutable( &mut self, signing_key: SigningKey, - pub_key: VerifyingKey, salt: Option>, data: Vec, + next_unix_minute_seq: i64, ) -> Result<()> { if self.dht.is_none() { self.reset().await?; @@ -122,22 +122,12 @@ impl DhtActor { for i in 0..1 + self.config.retries() { let dht = self.dht.as_mut().context("DHT not initialized")?; - let most_recent_result = tokio::time::timeout( - self.config.put_timeout(), - dht.get_mutable_most_recent(pub_key.as_bytes(), salt.as_deref()), - ) - .await?; - - let item = if let Some(mut_item) = most_recent_result { - MutableItem::new( - signing_key.clone(), - &data, - mut_item.seq() + 1, - salt.as_deref(), - ) - } else { - MutableItem::new(signing_key.clone(), &data, 0, salt.as_deref()) - }; + let item = MutableItem::new( + signing_key.clone(), + &data, + next_unix_minute_seq, + salt.as_deref(), + ); let put_result = match tokio::time::timeout( self.config.put_timeout(), @@ -145,7 +135,13 @@ impl DhtActor { ) .await { - Ok(result) => result.ok(), + Ok(result) => match result { + Ok(id) => Some(id), + Err(err) => { + tracing::warn!("DHT put_mutable operation failed: {err:?}"); + None + } + }, Err(_) => None, }; diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 28c35ff..6313594 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -108,8 +108,8 @@ impl BubbleMergeActor { // Cluster size as bubble indicator async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1).await; - records.extend(self.record_publisher.get_records(unix_minute).await); + let mut records = self.record_publisher.get_records(unix_minute - 1).await?; + records.extend(self.record_publisher.get_records(unix_minute).await?); let neighbors = self.gossip_receiver.neighbors().await?; tracing::debug!( @@ -127,8 +127,7 @@ impl BubbleMergeActor { let pub_keys = records .iter() .flat_map(|record| { - let mut pub_keys = if let Ok(content) = - record.content::() + let mut pub_keys = if let Ok(content) = record.content::() { content .active_peers @@ -151,6 +150,7 @@ impl BubbleMergeActor { if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) && pub_key != EndpointId::from_verifying_key(self.record_publisher.pub_key()) + && !neighbors.contains(&pub_key) { pub_keys.push(pub_key); } diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index dff3516..d83371b 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -98,8 +98,8 @@ impl MessageOverlapMergeActor { impl MessageOverlapMergeActor { async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1).await; - records.extend(self.record_publisher.get_records(unix_minute).await); + let mut records = self.record_publisher.get_records(unix_minute - 1).await?; + records.extend(self.record_publisher.get_records(unix_minute).await?); let local_hashes = self.gossip_receiver.last_message_hashes().await?; tracing::debug!( diff --git a/src/gossip/receiver.rs b/src/gossip/receiver.rs index 2381219..bdb5d92 100644 --- a/src/gossip/receiver.rs +++ b/src/gossip/receiver.rs @@ -148,14 +148,14 @@ impl GossipReceiver { Ok(event) => match event { Some(event) => Ok(event), None => Err(ChannelError::Closed), - } + }, Err(err) => Err(err.into()), } } /// Waits for a NeighborUp or a message Received event then returns `Ok(())`. pub async fn joined(&mut self) -> Result<(), ChannelError> { - if self.is_joined().await.unwrap_or(false) { + if self.is_joined().await.map_err(|_| ChannelError::Closed)? { return Ok(()); } match self.join_channel_receiver.recv().await { diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 5a5a2a1..e8a080b 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -125,7 +125,7 @@ impl BootstrapActor { async move { tracing::debug!("Bootstrap: starting bootstrap process"); - while !cancel_token.is_cancelled() { + 'bootstrap: while !cancel_token.is_cancelled() { // Check if we are connected to at least one node let is_joined = gossip_receiver.is_joined().await; if let Ok(is_joined) = is_joined @@ -153,8 +153,14 @@ impl BootstrapActor { ); // Unique, verified records for the unix minute - let mut records = record_publisher.get_records(unix_minute - 1).await; - let current_records = record_publisher.get_records(unix_minute).await; + let mut records = record_publisher + .get_records(unix_minute - 1) + .await + .unwrap_or_default(); + let current_records = record_publisher + .get_records(unix_minute) + .await + .unwrap_or_default(); records.extend(current_records.clone()); tracing::debug!( @@ -251,7 +257,7 @@ impl BootstrapActor { tokio::select! { _ = sleep(bootstrap_config.per_peer_join_settle_time()) => {} _ = gossip_receiver.joined() => {}, - _ = cancel_token.cancelled() => break, + _ = cancel_token.cancelled() => break 'bootstrap, } let is_joined = gossip_receiver.is_joined().await; if let Ok(is_joined) = is_joined diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 9bd7707..f4c2410 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -144,10 +144,10 @@ impl PublisherActor { let record = res?; let result = self.record_publisher.publish_record(record).await; - if result.is_ok() { - tracing::debug!("Publisher: successfully published record"); + if let Err(ref e) = result { + tracing::debug!("Publisher: failed to publish record: {:?}", e); } else { - tracing::debug!("Publisher: failed to publish record: {:?}", result); + tracing::debug!("Publisher: successfully published record"); } result diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 976e701..f877cb1 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -1,6 +1,9 @@ //! Main topic handle combining bootstrap, publishing, and merging. -use std::{str::FromStr, sync::Arc}; +use std::{ + str::FromStr, + sync::{Arc, Weak}, +}; use crate::{ BubbleMergeConfig, GossipSender, MessageOverlapMergeConfig, @@ -87,7 +90,7 @@ impl TopicId { /// Can be split into sender and receiver for message exchange. #[derive(Debug, Clone)] pub struct Topic { - api: Handle, + api: Arc>, cancel_token: CancellationToken, } @@ -136,54 +139,50 @@ impl Topic { .await?; tracing::debug!("Topic: bootstrap instance created"); - let api = Handle::spawn(TopicActor { - bootstrap: bootstrap.clone(), - record_publisher: record_publisher.clone(), - publisher: None, - bubble_merge: None, - message_overlap_merge: None, - cancel_token: cancel_token.clone(), - }) - .0; + let api = Arc::new( + Handle::spawn(TopicActor { + bootstrap: bootstrap.clone(), + record_publisher: record_publisher.clone(), + publisher: None, + bubble_merge: None, + message_overlap_merge: None, + cancel_token: cancel_token.clone(), + }) + .0, + ); let bootstrap_done = bootstrap.bootstrap().await?; if !async_bootstrap { tracing::debug!("Topic: waiting for bootstrap to complete"); - bootstrap_done.await??; + + wait_for_bootstrap_and_spawn_workers( + Arc::downgrade(&api), + bootstrap_done, + record_publisher.clone(), + cancel_token.clone(), + ) + .await?; tracing::debug!("Topic: bootstrap completed"); } else { tracing::debug!("Topic: bootstrap started asynchronously"); + tokio::spawn({ + let api = Arc::downgrade(&api); + let cancel_token = cancel_token.clone(); + async move { + if let Err(err) = wait_for_bootstrap_and_spawn_workers( + api, + bootstrap_done, + record_publisher.clone(), + cancel_token.clone(), + ) + .await + { + tracing::error!("failed to start topic: {}", err); + } + } + }); } - if matches!( - record_publisher.config().publisher_config(), - PublisherConfig::Enabled { .. } - ) { - tracing::debug!("Topic: starting publisher"); - api.call(act!(actor => actor.start_publishing())).await?; - } - - if matches!( - record_publisher.config().merge_config().bubble_merge(), - BubbleMergeConfig::Enabled { .. } - ) { - tracing::debug!("Topic: starting bubble merge"); - api.call(act!(actor => actor.start_bubble_merge())).await?; - } - - if matches!( - record_publisher - .config() - .merge_config() - .message_overlap_merge(), - MessageOverlapMergeConfig::Enabled { .. } - ) { - tracing::debug!("Topic: starting message overlap merge"); - api.call(act!(actor => actor.start_message_overlap_merge())) - .await?; - } - - tracing::debug!("Topic: fully initialized"); Ok(Self { api, cancel_token }) } @@ -224,6 +223,87 @@ impl Topic { } } +async fn wait_for_bootstrap_and_spawn_workers( + api: Weak>, + bootstrap_done: tokio::sync::oneshot::Receiver>, + record_publisher: crate::crypto::RecordPublisher, + cancel_token: CancellationToken, +) -> Result<()> { + if let Ok(Ok(_)) = bootstrap_done.await + && !cancel_token.is_cancelled() + { + if matches!( + record_publisher.config().publisher_config(), + PublisherConfig::Enabled { .. } + ) { + tracing::debug!("Topic: starting publisher"); + match api.upgrade() { + Some(api) => { + if let Err(err) = api.call(act!(actor => actor.start_publishing())).await { + cancel_token.cancel(); + return Err(anyhow::anyhow!("failed to start publisher: {err}")); + } + } + None => { + return Err(anyhow::anyhow!( + "failed to start publisher, topic actor dropped" + )); + } + } + } + + if matches!( + record_publisher.config().merge_config().bubble_merge(), + BubbleMergeConfig::Enabled { .. } + ) { + tracing::debug!("Topic: starting bubble merge"); + match api.upgrade() { + Some(api) => { + if let Err(err) = api.call(act!(actor => actor.start_bubble_merge())).await { + tracing::warn!("Topic: failed to start bubble merge: {err}"); + } + } + None => { + return Err(anyhow::anyhow!( + "failed to start bubble merge, topic actor dropped" + )); + } + } + } + + if matches!( + record_publisher + .config() + .merge_config() + .message_overlap_merge(), + MessageOverlapMergeConfig::Enabled { .. } + ) { + tracing::debug!("Topic: starting message overlap merge"); + match api.upgrade() { + Some(api) => { + if let Err(err) = api + .call(act!(actor => actor.start_message_overlap_merge())) + .await + { + tracing::warn!("Topic: failed to start message overlap merge: {err}"); + } + } + None => { + return Err(anyhow::anyhow!( + "failed to start message overlap merge, topic actor dropped" + )); + } + } + } + tracing::debug!("Topic: fully initialized"); + Ok(()) + } else { + tracing::error!("Topic: bootstrap failed or cancelled, shutting down topic"); + cancel_token.cancel(); + Err(anyhow::anyhow!("bootstrap failed or cancelled")) + } +} + impl TopicActor { pub async fn start_publishing(&mut self) -> Result<()> { if let PublisherConfig::Enabled { @@ -345,10 +425,7 @@ mod tests { let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.joined()) .await .expect("joined() hung after shutdown - broadcast channel didn't close"); - assert!( - result.is_err(), - "expected Err from joined() after shutdown" - ); + assert!(result.is_err(), "expected Err from joined() after shutdown"); // A clone made after shutdown must also return Err immediately // (WeakSender::upgrade fails -> gets an already closed channel) @@ -399,7 +476,7 @@ mod tests { let cancel_token = topic.cancel_token(); - let (sender, receiver) = topic.split().await.unwrap(); + let (sender, receiver) = topic.split().await.expect("failed to split topic"); assert!(!cancel_token.is_cancelled()); From 2d6d793e6353490f56b1ae06c0ca7b12b2236d7d Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sat, 18 Apr 2026 23:04:20 +0200 Subject: [PATCH 21/30] chore: messave_overlap changed from miss on single message to miss on all messages + dht seq now i64::MAX (at least in mainline this guarantees that overwrites are fair, last write wins per dht observer) this removed all CAS errors + added publisher and individual merge config builders + fixed dedub vec bug that retained lower seq numbers (in theory with i64 this should be obsolete) + added fail_topic_creation_on_publishing/merge_bubble/message_startup_failure to wait_for_bootstrap_and_spawn_workers making startup failure behaviour configurable --- Cargo.lock | 19 +-- Cargo.toml | 1 - examples/full_config.rs | 31 ++-- src/config.rs | 230 +++++++++++++++++++++++++--- src/crypto/record.rs | 55 ++----- src/gossip/merge/message_overlap.rs | 15 +- src/gossip/topic/topic.rs | 76 +++++++-- src/lib.rs | 5 +- 8 files changed, 315 insertions(+), 117 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c173d5..bf6626b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -695,7 +695,6 @@ dependencies = [ "futures-lite", "iroh", "iroh-gossip", - "lru 0.17.0", "mainline", "postcard", "rand", @@ -1224,11 +1223,6 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash 0.2.0", -] [[package]] name = "heapless" @@ -1806,7 +1800,7 @@ dependencies = [ "hyper-util", "iroh-base", "iroh-metrics", - "lru 0.16.3", + "lru", "n0-error", "n0-future", "noq", @@ -1952,15 +1946,6 @@ dependencies = [ "hashbrown 0.16.1", ] -[[package]] -name = "lru" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0b564323a0fb6d54b864f625ae139de9612e27edb944dda37c109f05aac531" -dependencies = [ - "hashbrown 0.17.0", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -1986,7 +1971,7 @@ dependencies = [ "flume 0.11.1", "futures-lite", "getrandom 0.3.4", - "lru 0.16.3", + "lru", "serde", "serde_bencode", "serde_bytes", diff --git a/Cargo.toml b/Cargo.toml index 16cc181..5bdf301 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,6 @@ rand = { version = "0.9", default-features = false, features = ["std", "std_rng" actor-helper = { version = "0.3", features = ["tokio", "anyhow"] } postcard = "1" serde = { version = "1", default-features = false, features = ["std"] } -lru = { version = "0.17" } tracing = { version = "0.1", default-features = false, features = ["std"] } tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "env-filter", "ansi"] } diff --git a/examples/full_config.rs b/examples/full_config.rs index 78e2985..aa1edb0 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -37,21 +37,24 @@ fn config_builder() -> Config { .build(), ) .max_join_peer_count(4) - .publisher_config(PublisherConfig::Enabled { - initial_delay: Duration::from_secs(10), - base_interval: Duration::from_secs(10), - max_jitter: Duration::from_secs(50), - }) + .publisher_config( + PublisherConfig::builder() + .initial_delay(Duration::from_secs(10)) + .base_interval(Duration::from_secs(10)) + .max_jitter(Duration::from_secs(50)) + .build(), + ) .merge_config(MergeConfig::new( - BubbleMergeConfig::Enabled { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - min_neighbors: 4, - }, - MessageOverlapMergeConfig::Enabled { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - }, + BubbleMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + MessageOverlapMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), )) .timeouts( TimeoutConfig::builder() diff --git a/src/config.rs b/src/config.rs index 5bc3004..b48f0ab 100644 --- a/src/config.rs +++ b/src/config.rs @@ -133,7 +133,7 @@ impl DhtConfig { config: DhtConfig::default(), } } - + /// Number of DHT operation retry attempts (first attempt + retries). Default: 3. pub fn retries(&self) -> usize { self.retries @@ -173,58 +173,250 @@ impl Default for DhtConfig { } /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s. Default: enabled, 60s base, 120s jitter, 4 min neighbors. +/// base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_merge_startup_failure. Default: enabled, 60s base, 120s jitter, 4 min neighbors, true. #[derive(Debug, Clone)] pub enum BubbleMergeConfig { Enabled { base_interval: Duration, max_jitter: Duration, min_neighbors: usize, + fail_topic_creation_on_merge_startup_failure: bool, }, Disabled, } +#[derive(Debug, Clone)] +pub struct BubbleMergeConfigBuilder { + base_interval: Duration, + max_jitter: Duration, + min_neighbors: usize, + fail_topic_creation_on_merge_startup_failure: bool, +} + +// NOTE: Also set in `BubbleMergeConfig::builder()` +impl Default for BubbleMergeConfig { + fn default() -> Self { + Self::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + min_neighbors: 4, + fail_topic_creation_on_merge_startup_failure: true, + } + } +} + +impl BubbleMergeConfig { + pub fn builder() -> BubbleMergeConfigBuilder { + BubbleMergeConfigBuilder { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + min_neighbors: 4, + fail_topic_creation_on_merge_startup_failure: true, + } + } +} + +impl BubbleMergeConfigBuilder { + /// Base interval for bubble merge attempts. Default: 60s. Minimum is 1s. + pub fn base_interval(mut self, interval: Duration) -> Self { + self.base_interval = interval; + self + } + + /// Max random jitter added to bubble merge interval. Default: 120s. Minimum is 0s. + pub fn max_jitter(mut self, jitter: Duration) -> Self { + self.max_jitter = jitter; + self + } + + /// Minimum number of neighbors required to attempt a bubble merge. Default: 4. + pub fn min_neighbors(mut self, min_neighbors: usize) -> Self { + self.min_neighbors = min_neighbors; + self + } + + /// Whether to fail topic creation if a bubble merge startup check fails (ret Err()) or just log and run topic without. Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { + self.fail_topic_creation_on_merge_startup_failure = fail; + self + } + + /// Build the `BubbleMergeConfig`. + pub fn build(self) -> BubbleMergeConfig { + BubbleMergeConfig::Enabled { + base_interval: self.base_interval, + max_jitter: self.max_jitter, + min_neighbors: self.min_neighbors, + fail_topic_creation_on_merge_startup_failure: self + .fail_topic_creation_on_merge_startup_failure, + } + } +} + /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s. Default: enabled, 60s base, 120s jitter. +/// base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_merge_startup_failure. Default: enabled, 60s base, 120s jitter, true. #[derive(Debug, Clone)] pub enum MessageOverlapMergeConfig { Enabled { base_interval: Duration, max_jitter: Duration, + fail_topic_creation_on_merge_startup_failure: bool, }, Disabled, } +#[derive(Debug, Clone)] +pub struct MessageOverlapMergeConfigBuilder { + base_interval: Duration, + max_jitter: Duration, + fail_topic_creation_on_merge_startup_failure: bool, +} + +// NOTE: Also set in `MessageOverlapMergeConfig::builder()` +impl Default for MessageOverlapMergeConfig { + fn default() -> Self { + Self::Enabled { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + fail_topic_creation_on_merge_startup_failure: true, + } + } +} + +impl MessageOverlapMergeConfig { + pub fn builder() -> MessageOverlapMergeConfigBuilder { + MessageOverlapMergeConfigBuilder { + base_interval: Duration::from_secs(60), + max_jitter: Duration::from_secs(120), + fail_topic_creation_on_merge_startup_failure: true, + } + } +} + +impl MessageOverlapMergeConfigBuilder { + /// Base interval for message overlap merge attempts. Default: 60s. Minimum is 1s. + pub fn base_interval(mut self, interval: Duration) -> Self { + self.base_interval = interval; + self + } + + /// Max random jitter added to message overlap merge interval. Default: 120s. Minimum is 0s. + pub fn max_jitter(mut self, jitter: Duration) -> Self { + self.max_jitter = jitter; + self + } + + /// Whether to fail topic creation if a message overlap merge startup check fails (ret Err()) or just log and run topic without. Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { + self.fail_topic_creation_on_merge_startup_failure = fail; + self + } + + /// Build the `MessageOverlapMergeConfig`. + pub fn build(self) -> MessageOverlapMergeConfig { + MessageOverlapMergeConfig::Enabled { + base_interval: self.base_interval, + max_jitter: self.max_jitter, + fail_topic_creation_on_merge_startup_failure: self + .fail_topic_creation_on_merge_startup_failure, + } + } +} + /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. +/// initial_delay is minimum 0s, base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_merge_startup_failure. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. #[derive(Debug, Clone)] pub enum PublisherConfig { Enabled { initial_delay: Duration, base_interval: Duration, max_jitter: Duration, + fail_topic_creation_on_publishing_startup_failure: bool, }, Disabled, } -/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s. #[derive(Debug, Clone)] -pub struct MergeConfig { - bubble_merge: BubbleMergeConfig, - message_overlap_merge: MessageOverlapMergeConfig, +pub struct PublisherConfigBuilder { + initial_delay: Duration, + base_interval: Duration, + max_jitter: Duration, + fail_topic_creation_on_merge_startup_failure: bool, } +/// NOTE: Also set in `PublisherConfig::builder()` impl Default for PublisherConfig { fn default() -> Self { Self::Enabled { initial_delay: Duration::from_secs(10), base_interval: Duration::from_secs(10), max_jitter: Duration::from_secs(50), + fail_topic_creation_on_publishing_startup_failure: true, + } + } +} + +impl PublisherConfig { + /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. + /// base_interval and initial_delay minimum is 1s, max_jitter is minimum 0s. + pub fn builder() -> PublisherConfigBuilder { + PublisherConfigBuilder { + initial_delay: Duration::from_secs(10), + base_interval: Duration::from_secs(10), + max_jitter: Duration::from_secs(50), + fail_topic_creation_on_merge_startup_failure: true, } } } +impl PublisherConfigBuilder { + /// Initial delay before starting publisher. Default: 10s. Minimum is 0s. + pub fn initial_delay(mut self, delay: Duration) -> Self { + self.initial_delay = delay; + self + } + + /// Base interval for publisher attempts. Default: 10s. Minimum is 1s. + pub fn base_interval(mut self, interval: Duration) -> Self { + self.base_interval = interval; + self + } + + /// Max random jitter added to publisher interval. Default: 50s. Minimum is 0s. + pub fn max_jitter(mut self, jitter: Duration) -> Self { + self.max_jitter = jitter; + self + } + + /// Whether to fail topic creation if a publisher startup check fails (ret Err()) or just log and run topic without. Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { + self.fail_topic_creation_on_merge_startup_failure = fail; + self + } + + /// Build the `PublisherConfig`. + pub fn build(self) -> PublisherConfig { + PublisherConfig::Enabled { + initial_delay: self.initial_delay, + base_interval: self.base_interval, + max_jitter: self.max_jitter, + fail_topic_creation_on_publishing_startup_failure: self + .fail_topic_creation_on_merge_startup_failure, + } + } +} + +/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. +/// base_interval is minimum 1s, max_jitter is minimum 0s. +#[derive(Debug, Clone)] +#[derive(Default)] +pub struct MergeConfig { + bubble_merge: BubbleMergeConfig, + message_overlap_merge: MessageOverlapMergeConfig, +} + + + impl MergeConfig { /// Defaults: bubble_merge=Enabled(60s base, 120s jitter, 4 min neighbors), message_overlap_merge=Enabled(60s base, 120s jitter). /// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes for message overlap detection and merging. @@ -239,13 +431,13 @@ impl MergeConfig { } } - /// Bubble merge strategy config. Default: enabled, 60s base, 120s jitter, 4 min neighbors. + /// Bubble merge strategy config. Default: enabled, 60s base, 120s jitter, 4 min neighbors, true. /// base_interval is minimum 1s, max_jitter is minimum 0s. pub fn bubble_merge(&self) -> &BubbleMergeConfig { &self.bubble_merge } - /// Message overlap merge strategy config. Default: enabled, 60s base, 120s jitter. + /// Message overlap merge strategy config. Default: enabled, 60s base, 120s jitter, true. /// base_interval is minimum 1s, max_jitter is minimum 0s. pub fn message_overlap_merge(&self) -> &MessageOverlapMergeConfig { &self.message_overlap_merge @@ -322,7 +514,7 @@ impl BootstrapConfigBuilder { } /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup (in this impl unix_minute and unix_minute-1 are both always fetched, if this is enabled than we first fetch unix_minute-2 and unix_minute-1). Default: false. - /// + /// /// If joining longer running, existing topics is priority, set to true. /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. pub fn check_last_minute_record_first_on_startup(mut self, check: bool) -> Self { @@ -375,7 +567,7 @@ impl BootstrapConfig { } /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup (in this impl unix_minute and unix_minute-1 are both always fetched, if this is enabled than we first fetch unix_minute-2 and unix_minute-1). Default: false. - /// + /// /// If joining longer running, existing topics is priority, set to true. /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. pub fn check_last_minute_record_first_on_startup(&self) -> bool { @@ -439,17 +631,7 @@ impl Config { impl Default for Config { fn default() -> Self { Self { - merge_config: MergeConfig { - bubble_merge: BubbleMergeConfig::Enabled { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - min_neighbors: 4, - }, - message_overlap_merge: MessageOverlapMergeConfig::Enabled { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - }, - }, + merge_config: MergeConfig::default(), bootstrap_config: BootstrapConfig::default(), publisher_config: PublisherConfig::default(), dht_config: DhtConfig::default(), diff --git a/src/crypto/record.rs b/src/crypto/record.rs index 26f4121..f6295a5 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -1,13 +1,11 @@ -use std::{collections::HashSet, sync::Arc}; +use std::collections::{HashMap, HashSet}; use anyhow::{Result, bail}; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use ed25519_dalek_hpke::{Ed25519hpkeDecryption, Ed25519hpkeEncryption}; -use lru::LruCache; use serde::{Deserialize, Serialize}; use sha2::Digest; -use tokio::sync::Mutex; use crate::{Config, TopicId}; @@ -58,43 +56,12 @@ impl RecordContent { } } -#[derive(Debug, Clone)] -struct NextDhtSeq { - inner: Arc>>, -} - -impl Default for NextDhtSeq { - fn default() -> Self { - Self::new(100).expect("capacity must be > 0") - } -} - -impl NextDhtSeq { - fn new(capacity: usize) -> Result { - Ok(Self { - inner: Arc::new(Mutex::new(LruCache::new( - std::num::NonZeroUsize::new(capacity) - .ok_or_else(|| anyhow::anyhow!("capacity must be > 0"))?, - ))), - }) - } - - async fn next(&self, unix_minute: u64) -> i64 { - let mut guard = self.inner.lock().await; - let current_seq = *guard.get(&unix_minute).unwrap_or(&0); - - guard.put(unix_minute, current_seq + 1); - current_seq - } -} - /// Publisher for creating and distributing signed DHT records. /// /// Checks existing DHT record count before publishing to respect capacity limits. #[derive(Debug, Clone)] pub struct RecordPublisher { dht: crate::dht::Dht, - dht_seq: NextDhtSeq, config: crate::config::Config, @@ -180,7 +147,6 @@ impl RecordPublisher { Self { dht: crate::dht::Dht::new(config.dht_config()), - dht_seq: NextDhtSeq::default(), config, topic_id: topic_id.into(), pub_key: signing_key.verifying_key(), @@ -292,10 +258,10 @@ impl RecordPublisher { record.unix_minute, ); let encrypted_record = record.encrypt(&encryption_key); - let next_seq_num = self.dht_seq.next(record.unix_minute()).await; + let next_seq_num = i64::MAX; tracing::debug!( - "RecordPublisher: publishing record to DHT for unix_minute {} with seq_num {next_seq_num}", + "RecordPublisher: publishing record to DHT for unix_minute {}", record.unix_minute() ); @@ -342,18 +308,19 @@ impl RecordPublisher { records_iter.len() ); - let mut dedubed_records: Vec<(i64, Record)> = vec![]; + let mut dedubed_records = HashMap::new(); for item in records_iter.clone() { if let Ok(encrypted_record) = EncryptedRecord::from_bytes(item.value().to_vec()) && let Ok(record) = encrypted_record.decrypt(&encryption_key) && record.verify(&self.topic_id.hash(), unix_minute).is_ok() && !record.pub_key().eq(self.pub_key.as_bytes()) { - if !dedubed_records - .iter() - .any(|(seq, r)| r.pub_key() == record.pub_key() && *seq >= item.seq()) - { - dedubed_records.push((item.seq(), record)); + let pub_key = record.pub_key(); + match dedubed_records.get(&pub_key) { + Some((seq, _)) if *seq >= item.seq() => {} + _ => { + dedubed_records.insert(pub_key, (item.seq(), record)); + } } } } @@ -363,7 +330,7 @@ impl RecordPublisher { ); Ok(dedubed_records - .into_iter() + .into_values() .map(|(_, record)| record) .collect::>()) } diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index d83371b..6360681 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -114,10 +114,17 @@ impl MessageOverlapMergeActor { .iter() .filter(|record| { if let Ok(content) = record.content::() { - content.last_message_hashes.iter().any(|last_message_hash| { - *last_message_hash != [0; 32] - && !last_message_hashes.contains(last_message_hash) - }) + let remote_hashes = content + .last_message_hashes + .iter() + .filter(|last_message_hash| **last_message_hash != [0; 32]) + .collect::>(); + + !remote_hashes.is_empty() + && !last_message_hashes.is_empty() + && remote_hashes.iter().all(|last_message_hash| { + !last_message_hashes.contains(*last_message_hash) + }) } else { false } diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index f877cb1..dc809fc 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -260,7 +260,8 @@ async fn wait_for_bootstrap_and_spawn_workers( match api.upgrade() { Some(api) => { if let Err(err) = api.call(act!(actor => actor.start_bubble_merge())).await { - tracing::warn!("Topic: failed to start bubble merge: {err}"); + cancel_token.cancel(); + return Err(anyhow::anyhow!("failed to start bubble merge: {err}")); } } None => { @@ -285,7 +286,10 @@ async fn wait_for_bootstrap_and_spawn_workers( .call(act!(actor => actor.start_message_overlap_merge())) .await { - tracing::warn!("Topic: failed to start message overlap merge: {err}"); + cancel_token.cancel(); + return Err(anyhow::anyhow!( + "failed to start message overlap merge: {err}" + )); } } None => { @@ -310,6 +314,7 @@ impl TopicActor { initial_delay, base_interval, max_jitter, + fail_topic_creation_on_publishing_startup_failure, } = self.record_publisher.config().publisher_config() { tracing::debug!("TopicActor: initializing publisher"); @@ -320,9 +325,24 @@ impl TopicActor { *initial_delay, *base_interval, *max_jitter, - )?; - self.publisher = Some(publisher); - tracing::debug!("TopicActor: publisher started"); + ); + + match publisher { + Ok(publisher) => { + self.publisher = Some(publisher); + tracing::debug!("TopicActor: publisher started"); + }, + Err(err) => { + if *fail_topic_creation_on_publishing_startup_failure { + return Err(anyhow::anyhow!("failed to start publisher: {}", err)); + } else { + tracing::warn!( + "TopicActor: failed to start publisher: {}, but continuing because Publisher.fail_topic_creation_on_publishing_startup_failure is false", + err + ); + } + } + } } Ok(()) } @@ -332,6 +352,7 @@ impl TopicActor { base_interval, max_jitter, min_neighbors, + fail_topic_creation_on_merge_startup_failure, } = self.record_publisher.config().merge_config().bubble_merge() { tracing::debug!("TopicActor: initializing bubble merge"); @@ -344,9 +365,24 @@ impl TopicActor { *base_interval, *max_jitter, *min_neighbors, - )?; - self.bubble_merge = Some(bubble_merge); - tracing::debug!("TopicActor: bubble merge started"); + ); + + match bubble_merge { + Ok(bubble_merge) => { + self.bubble_merge = Some(bubble_merge); + tracing::debug!("TopicActor: bubble merge started"); + } + Err(err) => { + if *fail_topic_creation_on_merge_startup_failure { + return Err(anyhow::anyhow!("failed to start bubble merge: {}", err)); + } else { + tracing::warn!( + "TopicActor: failed to start bubble merge: {}, but continuing because BubbleMerge.fail_topic_creation_on_merge_startup_failure is false", + err + ); + } + } + } } Ok(()) } @@ -355,6 +391,7 @@ impl TopicActor { if let MessageOverlapMergeConfig::Enabled { base_interval, max_jitter, + fail_topic_creation_on_merge_startup_failure, } = self .record_publisher .config() @@ -370,9 +407,26 @@ impl TopicActor { self.record_publisher.config().max_join_peer_count().max(1), *base_interval, *max_jitter, - )?; - self.message_overlap_merge = Some(message_overlap_merge); - tracing::debug!("TopicActor: message overlap merge started"); + ); + match message_overlap_merge { + Ok(message_overlap_merge) => { + self.message_overlap_merge = Some(message_overlap_merge); + tracing::debug!("TopicActor: message overlap merge started"); + } + Err(err) => { + if *fail_topic_creation_on_merge_startup_failure { + return Err(anyhow::anyhow!( + "failed to start message overlap merge: {}", + err + )); + } else { + tracing::warn!( + "TopicActor: failed to start message overlap merge: {}, but continuing because MessageOverlapMerge.fail_topic_creation_on_merge_startup_failure is false", + err + ); + } + } + } } Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 63bf066..ea6f69e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,8 +13,9 @@ pub use gossip::{ }; pub use config::{ - BootstrapConfig, BubbleMergeConfig, Config, ConfigBuilder, DhtConfig, MergeConfig, - MessageOverlapMergeConfig, PublisherConfig, TimeoutConfig, TimeoutConfigBuilder, + BootstrapConfig, BubbleMergeConfig, BubbleMergeConfigBuilder, Config, ConfigBuilder, DhtConfig, + MergeConfig, MessageOverlapMergeConfig, MessageOverlapMergeConfigBuilder, PublisherConfig, + PublisherConfigBuilder, TimeoutConfig, TimeoutConfigBuilder, }; pub use crypto::{ DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RotationHandle, From 07405ac430bc95e06e49f24d2cadcb80b79d0c2b Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sat, 18 Apr 2026 23:28:22 +0200 Subject: [PATCH 22/30] chore: update docs and correctness fixes --- ARCHITECTURE.md | 4 +- PROTOCOL.md | 2 +- README.md | 102 +++++++++++++++++++----------------- examples/full_config.rs | 2 +- examples/without_mergers.rs | 3 +- src/config.rs | 18 +++---- src/crypto/record.rs | 6 ++- src/gossip/merge/mod.rs | 4 +- 8 files changed, 75 insertions(+), 66 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index db014d8..fc5cc1a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -241,7 +241,9 @@ flowchart TD T[topic_hash] --> A[SHA512 topic+minute] M[unix_minute] --> A A --> S[signing_keypair seed -> Ed25519] - A --> L[salt = first 32 bytes] + + T --> L["salt = SHA512('salt' + topic + minute)[..32]"] + M --> L T --> R[secret_rotation topic,minute,initial_secret_hash] M --> R diff --git a/PROTOCOL.md b/PROTOCOL.md index b304e3b..7e96a03 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -40,7 +40,7 @@ The publishing procedure is a rate-limited mechanism that prevents DHT overload - Publish to DHT via `Dht::put_mutable()` with retry support 5. **Error Handling** - - DHT timeouts return empty record sets (non-fatal) + - DHT timeouts return error after retries are exhausted - Failed record decryption/verification are ignored - DHT put retries with jittered intervals prevent synchronized access patterns diff --git a/README.md b/README.md index c03f871..00ec283 100644 --- a/README.md +++ b/README.md @@ -189,59 +189,63 @@ let topic = TopicId::new(raw.clone()); All timing, retry, and threshold parameters are now configurable: -```rust,ignore +```rust,no_run +use std::time::Duration; use distributed_topic_tracker::{ - Config, BootstrapConfig, DhtConfig, TimeoutConfig, - PublisherConfig, BubbleMergeConfig, MessageOverlapMergeConfig, MergeConfig, + Config, DhtConfig, BootstrapConfig, PublisherConfig, MergeConfig, + BubbleMergeConfig, MessageOverlapMergeConfig, TimeoutConfig, }; -use std::time::Duration; Config::builder() - .dht_config( - DhtConfig::builder() - .retries(3) - .base_retry_interval(Duration::from_secs(5)) - .max_retry_jitter(Duration::from_secs(10)) - .get_timeout(Duration::from_secs(10)) - .put_timeout(Duration::from_secs(10)) - .build(), - ) - .bootstrap_config( - BootstrapConfig::builder() - .max_bootstrap_records(5) - .publish_record_on_startup(true) - .check_last_minute_record_first_on_startup(false) - .discovery_poll_interval(Duration::from_millis(2000)) - .no_peers_retry_interval(Duration::from_millis(1500)) - .per_peer_join_settle_time(Duration::from_millis(100)) - .join_confirmation_wait_time(Duration::from_millis(500)) - .build(), - ) - .max_join_peer_count(4) - .publisher_config(PublisherConfig::Enabled { - initial_delay: Duration::from_secs(10), - base_interval: Duration::from_secs(10), - max_jitter: Duration::from_secs(50), - }) - .merge_config(MergeConfig::new( - BubbleMergeConfig::Enabled { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - min_neighbors: 4, - }, - MessageOverlapMergeConfig::Enabled { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - }, - )) - .timeouts( - TimeoutConfig::builder() - .join_peer_timeout(Duration::from_secs(5)) - .broadcast_neighbors_timeout(Duration::from_secs(5)) - .broadcast_timeout(Duration::from_secs(5)) - .build(), - ) - .build(); + .dht_config( + DhtConfig::builder() + .retries(3) + .base_retry_interval(Duration::from_secs(5)) + .max_retry_jitter(Duration::from_secs(10)) + .get_timeout(Duration::from_secs(10)) + .put_timeout(Duration::from_secs(10)) + .build(), + ) + .bootstrap_config( + BootstrapConfig::builder() + .max_bootstrap_records(5) + .publish_record_on_startup(true) + .check_last_minute_record_first_on_startup(false) + .discovery_poll_interval(Duration::from_millis(2000)) + .no_peers_retry_interval(Duration::from_millis(1500)) + .per_peer_join_settle_time(Duration::from_millis(100)) + .join_confirmation_wait_time(Duration::from_millis(500)) + .build(), + ) + .max_join_peer_count(4) + .publisher_config( + PublisherConfig::builder() + .initial_delay(Duration::from_secs(10)) + .base_interval(Duration::from_secs(10)) + .max_jitter(Duration::from_secs(50)) + .build(), + ) + .merge_config(MergeConfig::new( + BubbleMergeConfig::builder() + .min_neighbors(4) + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + MessageOverlapMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + )) + .timeouts( + TimeoutConfig::builder() + .join_peer_timeout(Duration::from_secs(5)) + .broadcast_neighbors_timeout(Duration::from_secs(5)) + .broadcast_timeout(Duration::from_secs(5)) + .build(), + ) + .build(); ``` **Disable merge strategies or publishing** diff --git a/examples/full_config.rs b/examples/full_config.rs index aa1edb0..c022846 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -46,6 +46,7 @@ fn config_builder() -> Config { ) .merge_config(MergeConfig::new( BubbleMergeConfig::builder() + .min_neighbors(4) .base_interval(Duration::from_secs(60)) .max_jitter(Duration::from_secs(120)) .fail_topic_creation_on_merge_startup_failure(true) @@ -95,7 +96,6 @@ async fn main() -> Result<()> { .secret_rotation(RotationHandle::new(DefaultSecretRotation)) .build(); - // Split into sink (sending) and stream (receiving) let topic = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await?; diff --git a/examples/without_mergers.rs b/examples/without_mergers.rs index 7b27dd4..54bde39 100644 --- a/examples/without_mergers.rs +++ b/examples/without_mergers.rs @@ -38,8 +38,7 @@ async fn main() -> Result<()> { // [!] Disable merge workers (BubbleMerge and MessageOverlapMerge) Config::builder().merge_config(MergeConfig::new(BubbleMergeConfig::Disabled, MessageOverlapMergeConfig::Disabled)).build(), ); - - // Split into sink (sending) and stream (receiving) + let topic = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await?; diff --git a/src/config.rs b/src/config.rs index b48f0ab..dca2fd5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -134,7 +134,7 @@ impl DhtConfig { } } - /// Number of DHT operation retry attempts (first attempt + retries). Default: 3. + /// Number of retries after the initial attempt (total attempts = 1 + retries). Default: 3. pub fn retries(&self) -> usize { self.retries } @@ -324,7 +324,7 @@ impl MessageOverlapMergeConfigBuilder { } /// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// initial_delay is minimum 0s, base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_merge_startup_failure. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. +/// initial_delay is minimum 0s, base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_publishing_startup_failure. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. #[derive(Debug, Clone)] pub enum PublisherConfig { Enabled { @@ -341,7 +341,7 @@ pub struct PublisherConfigBuilder { initial_delay: Duration, base_interval: Duration, max_jitter: Duration, - fail_topic_creation_on_merge_startup_failure: bool, + fail_topic_creation_on_publishing_startup_failure: bool, } /// NOTE: Also set in `PublisherConfig::builder()` @@ -358,13 +358,13 @@ impl Default for PublisherConfig { impl PublisherConfig { /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. - /// base_interval and initial_delay minimum is 1s, max_jitter is minimum 0s. + /// base_interval minimum is 1s, initial_delay minimum is 0s, max_jitter is minimum 0s. pub fn builder() -> PublisherConfigBuilder { PublisherConfigBuilder { initial_delay: Duration::from_secs(10), base_interval: Duration::from_secs(10), max_jitter: Duration::from_secs(50), - fail_topic_creation_on_merge_startup_failure: true, + fail_topic_creation_on_publishing_startup_failure: true, } } } @@ -389,8 +389,8 @@ impl PublisherConfigBuilder { } /// Whether to fail topic creation if a publisher startup check fails (ret Err()) or just log and run topic without. Default: true. - pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { - self.fail_topic_creation_on_merge_startup_failure = fail; + pub fn fail_topic_creation_on_publishing_startup_failure(mut self, fail: bool) -> Self { + self.fail_topic_creation_on_publishing_startup_failure = fail; self } @@ -401,7 +401,7 @@ impl PublisherConfigBuilder { base_interval: self.base_interval, max_jitter: self.max_jitter, fail_topic_creation_on_publishing_startup_failure: self - .fail_topic_creation_on_merge_startup_failure, + .fail_topic_creation_on_publishing_startup_failure, } } } @@ -655,7 +655,7 @@ impl ConfigBuilder { } /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. - /// base_interval and initial_delay minimum is 1s, max_jitter is minimum 0s. + /// base_interval minimum is 1s, initial_delay minimum is 0s, max_jitter is minimum 0s. pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { self.config.publisher_config = publisher_config; self diff --git a/src/crypto/record.rs b/src/crypto/record.rs index f6295a5..e975f98 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -281,7 +281,7 @@ impl RecordPublisher { /// Retrieve all verified records for a given time slot from the DHT. /// /// Filters out records from this publisher's own node ID. - /// Dedub's records based on pub_key, keeping the newest sequence number per pub_key. + /// Dedup's records based on pub_key, keeping the highest sequence number per pub_key. pub async fn get_records(&self, unix_minute: u64) -> Result> { tracing::debug!( "RecordPublisher: fetching records from DHT for unix_minute {}", @@ -359,6 +359,10 @@ impl EncryptedRecord { buf.extend_from_slice(&encrypted_record_len.to_le_bytes()); buf.extend_from_slice(&self.encrypted_record); buf.extend_from_slice(&self.encrypted_decryption_key); + + if buf.len() > Self::MAX_SIZE { + unreachable!("EncryptedRecord serialization exceeds maximum size, the max is set generously so this should never happen, if so your code is using it manually"); + } buf } diff --git a/src/gossip/merge/mod.rs b/src/gossip/merge/mod.rs index 2e0f576..87622a5 100644 --- a/src/gossip/merge/mod.rs +++ b/src/gossip/merge/mod.rs @@ -2,8 +2,8 @@ //! //! Two complementary strategies detect and heal split-brain scenarios: //! - **Bubble Merge**: Joins small clusters (< `min_neighbors` peers, as configured in `BubbleMergeConfig::Enabled`) with peers advertised in DHT -//! - **Message Overlap**: Detects when isolated clusters share common message hashes, -//! indicating they have seen the same messages and can be merged +//! - **Message Overlap**: Detects when isolated clusters have no common message hashes, +//! indicating a network partition that should be healed by merging mod bubble; mod message_overlap; From b410eca5b4c119ba01442c8552c7b2fcf8aedae6 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 03:49:38 +0200 Subject: [PATCH 23/30] fix: topic error propagation + topic fail_on_* now fully enforced + config docs cleaned up + jitter calculation without info loss + topic bootstrap await using gossip_receiver.joined().await --- ARCHITECTURE.md | 4 +- PROTOCOL.md | 6 +- README.md | 38 +- examples/chat.rs | 1 - examples/full_config.rs | 32 +- examples/without_mergers.rs | 17 +- src/config.rs | 572 ++++++++++++++++++++-------- src/crypto/record.rs | 10 +- src/dht.rs | 13 +- src/gossip/merge/bubble.rs | 18 +- src/gossip/merge/message_overlap.rs | 22 +- src/gossip/topic/bootstrap.rs | 4 +- src/gossip/topic/publisher.rs | 12 +- src/gossip/topic/topic.rs | 154 ++++---- tests/tests.rs | 11 +- 15 files changed, 585 insertions(+), 329 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fc5cc1a..0577e63 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -202,7 +202,7 @@ Record (summary): - topic hash (32) - unix_minute (u64) - pub_key (publisher ed25519 public key) -- content (serialized RecordContent: active_peers + last_message_hashes) +- content (serialized GossipRecordContent: active_peers + last_message_hashes) - signature (64) EncryptedRecord: @@ -217,7 +217,7 @@ classDiagram +topic: [u8;32] +unix_minute: u64 +pub_key: [u8;32] - +content: RecordContent + +content: GossipRecordContent +signature: [u8;64] } diff --git a/PROTOCOL.md b/PROTOCOL.md index 7e96a03..12595fb 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -108,9 +108,9 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne - If connected, exit bootstrap loop 3. **Time Window Selection** - - If `check_last_minute_record_first_on_startup` is enabled and this is the first attempt: check previous unix minute (`unix_minute(-1)`) - - Otherwise: check current unix minute (`unix_minute(0)`) - - Both `unix_minute` and `unix_minute - 1` records are always fetched + - If `check_last_minute_record_first_on_startup` is enabled and this is the first attempt: check previous `unix minute-1` and `unix_minute-2` + - Otherwise: check current unix minute `unix_minute` and `unix_minute-1` + - Both `unix_minute` and `unix_minute-1` records are always fetched 4. **Record Discovery** - Call `get_records()` for both `unix_minute - 1` and `unix_minute`: diff --git a/README.md b/README.md index 00ec283..0cad8cb 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,15 @@ Add dependencies to `Cargo.toml`: anyhow = "1" tokio = "1" rand = "0.9" -iroh = "*" -iroh-gossip = "*" +ed25519-dalek = "3.0.0-pre.1" +iroh = "^0.97" +iroh-gossip = "^0.97" distributed-topic-tracker = "0.3" ``` Basic iroh-gossip integration: + ```rust,no_run use anyhow::Result; use iroh::{Endpoint, SecretKey}; @@ -217,7 +219,7 @@ Config::builder() .join_confirmation_wait_time(Duration::from_millis(500)) .build(), ) - .max_join_peer_count(4) + .max_join_peer_count(std::num::NonZeroU32::new(4).expect("must be > 0")) .publisher_config( PublisherConfig::builder() .initial_delay(Duration::from_secs(10)) @@ -225,19 +227,25 @@ Config::builder() .max_jitter(Duration::from_secs(50)) .build(), ) - .merge_config(MergeConfig::new( - BubbleMergeConfig::builder() - .min_neighbors(4) - .base_interval(Duration::from_secs(60)) - .max_jitter(Duration::from_secs(120)) - .fail_topic_creation_on_merge_startup_failure(true) - .build(), - MessageOverlapMergeConfig::builder() - .base_interval(Duration::from_secs(60)) - .max_jitter(Duration::from_secs(120)) - .fail_topic_creation_on_merge_startup_failure(true) + .merge_config( + MergeConfig::builder() + .bubble_merge( + BubbleMergeConfig::builder() + .min_neighbors(4) + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) + .message_overlap_merge( + MessageOverlapMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) .build(), - )) + ) .timeouts( TimeoutConfig::builder() .join_peer_timeout(Duration::from_secs(5)) diff --git a/examples/chat.rs b/examples/chat.rs index a687be7..fa7d08e 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -91,7 +91,6 @@ async fn main() -> Result<()> { gossip_sender .broadcast( buffer - .clone() .trim_end_matches(&['\r', '\n'][..]) .as_bytes() .to_vec(), diff --git a/examples/full_config.rs b/examples/full_config.rs index c022846..f1147a2 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -36,7 +36,7 @@ fn config_builder() -> Config { .join_confirmation_wait_time(Duration::from_millis(500)) .build(), ) - .max_join_peer_count(4) + .max_join_peer_count(std::num::NonZeroU32::new(4).expect("must be > 0")) .publisher_config( PublisherConfig::builder() .initial_delay(Duration::from_secs(10)) @@ -44,19 +44,25 @@ fn config_builder() -> Config { .max_jitter(Duration::from_secs(50)) .build(), ) - .merge_config(MergeConfig::new( - BubbleMergeConfig::builder() - .min_neighbors(4) - .base_interval(Duration::from_secs(60)) - .max_jitter(Duration::from_secs(120)) - .fail_topic_creation_on_merge_startup_failure(true) + .merge_config( + MergeConfig::builder() + .bubble_merge( + BubbleMergeConfig::builder() + .min_neighbors(4) + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) + .message_overlap_merge( + MessageOverlapMergeConfig::builder() + .base_interval(Duration::from_secs(60)) + .max_jitter(Duration::from_secs(120)) + .fail_topic_creation_on_merge_startup_failure(true) + .build(), + ) .build(), - MessageOverlapMergeConfig::builder() - .base_interval(Duration::from_secs(60)) - .max_jitter(Duration::from_secs(120)) - .fail_topic_creation_on_merge_startup_failure(true) - .build(), - )) + ) .timeouts( TimeoutConfig::builder() .join_peer_timeout(Duration::from_secs(5)) diff --git a/examples/without_mergers.rs b/examples/without_mergers.rs index 54bde39..3152523 100644 --- a/examples/without_mergers.rs +++ b/examples/without_mergers.rs @@ -4,7 +4,10 @@ use iroh::{Endpoint, SecretKey}; use iroh_gossip::net::Gossip; // Imports from distributed-topic-tracker -use distributed_topic_tracker::{AutoDiscoveryGossip, BubbleMergeConfig, Config, MergeConfig, MessageOverlapMergeConfig, RecordPublisher, TopicId}; +use distributed_topic_tracker::{ + AutoDiscoveryGossip, BubbleMergeConfig, Config, MergeConfig, MessageOverlapMergeConfig, + RecordPublisher, TopicId, +}; #[tokio::main] async fn main() -> Result<()> { @@ -34,11 +37,17 @@ async fn main() -> Result<()> { signing_key.clone(), None, initial_secret, - // [!] Disable merge workers (BubbleMerge and MessageOverlapMerge) - Config::builder().merge_config(MergeConfig::new(BubbleMergeConfig::Disabled, MessageOverlapMergeConfig::Disabled)).build(), + Config::builder() + .merge_config( + MergeConfig::builder() + .bubble_merge(BubbleMergeConfig::Disabled) + .message_overlap_merge(MessageOverlapMergeConfig::Disabled) + .build(), + ) + .build(), ); - + let topic = gossip .subscribe_and_join_with_auto_discovery(record_publisher) .await?; diff --git a/src/config.rs b/src/config.rs index dca2fd5..64a5b40 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{num::NonZeroU32, time::Duration}; /// Timeout settings for gossip operations. #[derive(Debug, Clone)] @@ -16,17 +16,23 @@ impl TimeoutConfig { } } - /// How long to wait when joining a peer. Default: 5s. + /// How long to wait when joining a peer. + /// + /// Default: 5s. pub fn join_peer_timeout(&self) -> Duration { self.join_peer_timeout } - /// How long to wait when broadcasting messages. Default: 5s. + /// How long to wait when broadcasting messages. + /// + /// Default: 5s. pub fn broadcast_timeout(&self) -> Duration { self.broadcast_timeout } - /// How long to wait when broadcasting to neighbors. Default: 5s. + /// How long to wait when broadcasting to neighbors. + /// + /// Default: 5s. pub fn broadcast_neighbors_timeout(&self) -> Duration { self.broadcast_neighbors_timeout } @@ -49,19 +55,25 @@ pub struct TimeoutConfigBuilder { } impl TimeoutConfigBuilder { - /// How long to wait when joining a peer. Default: 5s. + /// How long to wait when joining a peer. + /// + /// Default: 5s. pub fn join_peer_timeout(mut self, timeout: Duration) -> Self { self.timeouts.join_peer_timeout = timeout; self } - /// How long to wait when broadcasting messages. Default: 5s. + /// How long to wait when broadcasting messages. + /// + /// Default: 5s. pub fn broadcast_timeout(mut self, timeout: Duration) -> Self { self.timeouts.broadcast_timeout = timeout; self } - /// How long to wait when broadcasting to neighbors. Default: 5s. + /// How long to wait when broadcasting to neighbors. + /// + /// Default: 5s. pub fn broadcast_neighbors_timeout(mut self, timeout: Duration) -> Self { self.timeouts.broadcast_neighbors_timeout = timeout; self @@ -90,31 +102,48 @@ pub struct DhtConfigBuilder { } impl DhtConfigBuilder { - /// Number of retries after the initial attempt (total attempts = 1 + retries). Default: 3. + /// Number of retries after the initial attempt. + /// + /// Total attempts = 1 + retries. + /// + /// Default: 3. pub fn retries(mut self, retries: usize) -> Self { self.config.retries = retries; self } - /// Base delay between retries. Default: 5s. + /// Base delay between retries. + /// + /// If `base_retry_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_retry_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 5s. pub fn base_retry_interval(mut self, interval: Duration) -> Self { - self.config.base_retry_interval = interval; + if interval > Duration::ZERO { + self.config.base_retry_interval = interval; + } self } - /// Max random jitter added to retry interval. Default: 10s. + /// Max random jitter added to retry interval. + /// + /// Default: 10s. pub fn max_retry_jitter(mut self, jitter: Duration) -> Self { self.config.max_retry_jitter = jitter; self } - /// Timeout for DHT put operations. Default: 10s. + /// Timeout for DHT put operations. + /// + /// Default: 10s. pub fn put_timeout(mut self, timeout: Duration) -> Self { self.config.put_timeout = timeout; self } - /// Timeout for DHT get operations. Default: 10s. + /// Timeout for DHT get operations. + /// + /// Default: 10s. pub fn get_timeout(mut self, timeout: Duration) -> Self { self.config.get_timeout = timeout; self @@ -134,27 +163,39 @@ impl DhtConfig { } } - /// Number of retries after the initial attempt (total attempts = 1 + retries). Default: 3. + /// Number of retries after the initial attempt. + /// + /// Total attempts = 1 + retries. + /// + /// Default: 3. pub fn retries(&self) -> usize { self.retries } - /// Base delay between retries. Default: 5s. + /// Base delay between retries. + /// + /// Default: 5s. pub fn base_retry_interval(&self) -> Duration { self.base_retry_interval } - /// Max random jitter added to retry interval. Default: 10s. + /// Max random jitter added to retry interval. + /// + /// Default: 10s. pub fn max_retry_jitter(&self) -> Duration { self.max_retry_jitter } - /// Timeout for DHT put operations. Default: 10s. + /// Timeout for DHT put operations. + /// + /// Default: 10s. pub fn put_timeout(&self) -> Duration { self.put_timeout } - /// Timeout for DHT get operations. Default: 10s. + /// Timeout for DHT get operations. + /// + /// Default: 10s. pub fn get_timeout(&self) -> Duration { self.get_timeout } @@ -172,31 +213,35 @@ impl Default for DhtConfig { } } -/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_merge_startup_failure. Default: enabled, 60s base, 120s jitter, 4 min neighbors, true. +/// Bubble merge strategy config for detecting and healing split-brain scenarios by joining small clusters with peers advertised in DHT that are not our neighbors. #[derive(Debug, Clone)] pub enum BubbleMergeConfig { - Enabled { - base_interval: Duration, - max_jitter: Duration, - min_neighbors: usize, - fail_topic_creation_on_merge_startup_failure: bool, - }, + Enabled(BubbleMergeConfigInner), Disabled, } #[derive(Debug, Clone)] -pub struct BubbleMergeConfigBuilder { +pub struct BubbleMergeConfigInner { base_interval: Duration, max_jitter: Duration, min_neighbors: usize, fail_topic_creation_on_merge_startup_failure: bool, } -// NOTE: Also set in `BubbleMergeConfig::builder()` +#[derive(Debug, Clone)] +pub struct BubbleMergeConfigBuilder { + config: BubbleMergeConfigInner, +} + impl Default for BubbleMergeConfig { fn default() -> Self { - Self::Enabled { + Self::Enabled(BubbleMergeConfigInner::default()) + } +} + +impl Default for BubbleMergeConfigInner { + fn default() -> Self { + Self { base_interval: Duration::from_secs(60), max_jitter: Duration::from_secs(120), min_neighbors: 4, @@ -206,76 +251,114 @@ impl Default for BubbleMergeConfig { } impl BubbleMergeConfig { + /// Create a new `BubbleMergeConfigBuilder` with default values. pub fn builder() -> BubbleMergeConfigBuilder { BubbleMergeConfigBuilder { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - min_neighbors: 4, - fail_topic_creation_on_merge_startup_failure: true, + config: BubbleMergeConfigInner::default(), } } } +impl BubbleMergeConfigInner { + /// Base interval for bubble merge attempts. + /// + /// Default: 60s. + pub fn base_interval(&self) -> Duration { + self.base_interval + } + + /// Max random jitter added to bubble merge interval. + /// + /// Default: 120s. + pub fn max_jitter(&self) -> Duration { + self.max_jitter + } + + /// Minimum number of neighbors required to attempt a bubble merge. + /// + /// Default: 4. + pub fn min_neighbors(&self) -> usize { + self.min_neighbors + } + + /// Whether to fail topic creation + /// + /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(&self) -> bool { + self.fail_topic_creation_on_merge_startup_failure + } +} + impl BubbleMergeConfigBuilder { - /// Base interval for bubble merge attempts. Default: 60s. Minimum is 1s. + /// Base interval for bubble merge attempts. + /// + /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 60s. pub fn base_interval(mut self, interval: Duration) -> Self { - self.base_interval = interval; + if interval > Duration::ZERO { + self.config.base_interval = interval; + } self } - /// Max random jitter added to bubble merge interval. Default: 120s. Minimum is 0s. + /// Max random jitter added to bubble merge interval. + /// + /// Default: 120s pub fn max_jitter(mut self, jitter: Duration) -> Self { - self.max_jitter = jitter; + self.config.max_jitter = jitter; self } - /// Minimum number of neighbors required to attempt a bubble merge. Default: 4. + /// Minimum number of neighbors required to attempt a bubble merge. + /// + /// Default: 4. pub fn min_neighbors(mut self, min_neighbors: usize) -> Self { - self.min_neighbors = min_neighbors; + self.config.min_neighbors = min_neighbors; self } - /// Whether to fail topic creation if a bubble merge startup check fails (ret Err()) or just log and run topic without. Default: true. + /// Whether to fail topic creation + /// + /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { - self.fail_topic_creation_on_merge_startup_failure = fail; + self.config.fail_topic_creation_on_merge_startup_failure = fail; self } /// Build the `BubbleMergeConfig`. pub fn build(self) -> BubbleMergeConfig { - BubbleMergeConfig::Enabled { - base_interval: self.base_interval, - max_jitter: self.max_jitter, - min_neighbors: self.min_neighbors, - fail_topic_creation_on_merge_startup_failure: self - .fail_topic_creation_on_merge_startup_failure, - } + BubbleMergeConfig::Enabled(self.config) } } -/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_merge_startup_failure. Default: enabled, 60s base, 120s jitter, true. +/// Message overlap merge strategy config for detecting and healing split-brain scenarios by checking for overlapping message hashes with other cluster peers. #[derive(Debug, Clone)] pub enum MessageOverlapMergeConfig { - Enabled { - base_interval: Duration, - max_jitter: Duration, - fail_topic_creation_on_merge_startup_failure: bool, - }, + Enabled(MessageOverlapMergeConfigInner), Disabled, } #[derive(Debug, Clone)] -pub struct MessageOverlapMergeConfigBuilder { +pub struct MessageOverlapMergeConfigInner { base_interval: Duration, max_jitter: Duration, fail_topic_creation_on_merge_startup_failure: bool, } -// NOTE: Also set in `MessageOverlapMergeConfig::builder()` -impl Default for MessageOverlapMergeConfig { +#[derive(Debug, Clone)] +pub struct MessageOverlapMergeConfigBuilder { + config: MessageOverlapMergeConfigInner, +} + +impl Default for MessageOverlapMergeConfigInner { fn default() -> Self { - Self::Enabled { + Self { base_interval: Duration::from_secs(60), max_jitter: Duration::from_secs(120), fail_topic_creation_on_merge_startup_failure: true, @@ -283,71 +366,105 @@ impl Default for MessageOverlapMergeConfig { } } +impl Default for MessageOverlapMergeConfig { + fn default() -> Self { + Self::Enabled(MessageOverlapMergeConfigInner::default()) + } +} + impl MessageOverlapMergeConfig { + /// Create a new `MessageOverlapMergeConfigBuilder` with default values. pub fn builder() -> MessageOverlapMergeConfigBuilder { MessageOverlapMergeConfigBuilder { - base_interval: Duration::from_secs(60), - max_jitter: Duration::from_secs(120), - fail_topic_creation_on_merge_startup_failure: true, + config: MessageOverlapMergeConfigInner::default(), } } } +impl MessageOverlapMergeConfigInner { + /// Base interval for message overlap merge attempts. + /// + /// Default: 60s. + pub fn base_interval(&self) -> Duration { + self.base_interval + } + + /// Max random jitter added to message overlap merge interval. + /// + /// Default: 120s. + pub fn max_jitter(&self) -> Duration { + self.max_jitter + } + + /// Whether to fail topic creation + /// + /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_merge_startup_failure(&self) -> bool { + self.fail_topic_creation_on_merge_startup_failure + } +} + impl MessageOverlapMergeConfigBuilder { - /// Base interval for message overlap merge attempts. Default: 60s. Minimum is 1s. + /// Base interval for message overlap merge attempts. + /// + /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 60s. pub fn base_interval(mut self, interval: Duration) -> Self { - self.base_interval = interval; + self.config.base_interval = interval; self } - /// Max random jitter added to message overlap merge interval. Default: 120s. Minimum is 0s. + /// Max random jitter added to message overlap merge interval. + /// + /// Default: 120s. Minimum is 0s. pub fn max_jitter(mut self, jitter: Duration) -> Self { - self.max_jitter = jitter; + self.config.max_jitter = jitter; self } - /// Whether to fail topic creation if a message overlap merge startup check fails (ret Err()) or just log and run topic without. Default: true. + /// Whether to fail topic creation + /// + /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { - self.fail_topic_creation_on_merge_startup_failure = fail; + self.config.fail_topic_creation_on_merge_startup_failure = fail; self } /// Build the `MessageOverlapMergeConfig`. pub fn build(self) -> MessageOverlapMergeConfig { - MessageOverlapMergeConfig::Enabled { - base_interval: self.base_interval, - max_jitter: self.max_jitter, - fail_topic_creation_on_merge_startup_failure: self - .fail_topic_creation_on_merge_startup_failure, - } + MessageOverlapMergeConfig::Enabled(self.config) } } -/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// initial_delay is minimum 0s, base_interval is minimum 1s, max_jitter is minimum 0s, fail_topic_creation_on_publishing_startup_failure. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. +/// Publisher strategy config for publishing bootstrap records to DHT for peer discovery. #[derive(Debug, Clone)] pub enum PublisherConfig { - Enabled { - initial_delay: Duration, - base_interval: Duration, - max_jitter: Duration, - fail_topic_creation_on_publishing_startup_failure: bool, - }, + Enabled(PublisherConfigInner), Disabled, } #[derive(Debug, Clone)] -pub struct PublisherConfigBuilder { +pub struct PublisherConfigInner { initial_delay: Duration, base_interval: Duration, max_jitter: Duration, fail_topic_creation_on_publishing_startup_failure: bool, } -/// NOTE: Also set in `PublisherConfig::builder()` -impl Default for PublisherConfig { +#[derive(Debug, Clone)] +pub struct PublisherConfigBuilder { + config: PublisherConfigInner, +} + +impl Default for PublisherConfigInner { fn default() -> Self { - Self::Enabled { + Self { initial_delay: Duration::from_secs(10), base_interval: Duration::from_secs(10), max_jitter: Duration::from_secs(50), @@ -356,94 +473,159 @@ impl Default for PublisherConfig { } } +impl Default for PublisherConfig { + fn default() -> Self { + Self::Enabled(PublisherConfigInner::default()) + } +} + impl PublisherConfig { - /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter, true. - /// base_interval minimum is 1s, initial_delay minimum is 0s, max_jitter is minimum 0s. + /// Create a new `PublisherConfigBuilder` with default values. pub fn builder() -> PublisherConfigBuilder { PublisherConfigBuilder { - initial_delay: Duration::from_secs(10), - base_interval: Duration::from_secs(10), - max_jitter: Duration::from_secs(50), - fail_topic_creation_on_publishing_startup_failure: true, + config: PublisherConfigInner::default(), } } } +impl PublisherConfigInner { + /// Initial delay before starting publisher. + /// + /// Default: 10s. + pub fn initial_delay(&self) -> Duration { + self.initial_delay + } + + /// Base interval for publisher attempts. + /// + /// Default: 10s. + pub fn base_interval(&self) -> Duration { + self.base_interval + } + + /// Max random jitter added to publisher interval. + /// + /// Default: 50s. + pub fn max_jitter(&self) -> Duration { + self.max_jitter + } + + /// Whether to fail topic creation + /// + /// If a publisher startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. + pub fn fail_topic_creation_on_publishing_startup_failure(&self) -> bool { + self.fail_topic_creation_on_publishing_startup_failure + } +} + impl PublisherConfigBuilder { - /// Initial delay before starting publisher. Default: 10s. Minimum is 0s. + /// Initial delay before starting publisher. + /// + /// Default: 10s. pub fn initial_delay(mut self, delay: Duration) -> Self { - self.initial_delay = delay; + self.config.initial_delay = delay; self } - /// Base interval for publisher attempts. Default: 10s. Minimum is 1s. + /// Base interval for publisher attempts. + /// + /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. + /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. + /// + /// Default: 10s. pub fn base_interval(mut self, interval: Duration) -> Self { - self.base_interval = interval; + if interval > Duration::ZERO { + self.config.base_interval = interval; + } self } - /// Max random jitter added to publisher interval. Default: 50s. Minimum is 0s. + /// Max random jitter added to publisher interval. + /// + /// Default: 50s. pub fn max_jitter(mut self, jitter: Duration) -> Self { - self.max_jitter = jitter; + self.config.max_jitter = jitter; self } - /// Whether to fail topic creation if a publisher startup check fails (ret Err()) or just log and run topic without. Default: true. + /// Whether to fail topic creation + /// + /// If a publisher startup check fails (ret Err()) or just log and run topic without. + /// + /// Default: true. pub fn fail_topic_creation_on_publishing_startup_failure(mut self, fail: bool) -> Self { - self.fail_topic_creation_on_publishing_startup_failure = fail; + self.config.fail_topic_creation_on_publishing_startup_failure = fail; self } /// Build the `PublisherConfig`. pub fn build(self) -> PublisherConfig { - PublisherConfig::Enabled { - initial_delay: self.initial_delay, - base_interval: self.base_interval, - max_jitter: self.max_jitter, - fail_topic_creation_on_publishing_startup_failure: self - .fail_topic_creation_on_publishing_startup_failure, - } + PublisherConfig::Enabled(self.config) } } -/// Effective interval is `(base_interval + jitter).max(1000ms)`, where `jitter` is sampled in `[0, max_jitter)`. -/// base_interval is minimum 1s, max_jitter is minimum 0s. -#[derive(Debug, Clone)] -#[derive(Default)] +/// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records +/// and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes +/// for message overlap detection and merging. +#[derive(Debug, Clone, Default)] pub struct MergeConfig { bubble_merge: BubbleMergeConfig, message_overlap_merge: MessageOverlapMergeConfig, } - +/// Builder for `MergeConfig`. +pub struct MergeConfigBuilder { + config: MergeConfig, +} impl MergeConfig { - /// Defaults: bubble_merge=Enabled(60s base, 120s jitter, 4 min neighbors), message_overlap_merge=Enabled(60s base, 120s jitter). - /// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes for message overlap detection and merging. - /// base_interval is minimum 1s, max_jitter is minimum 0s. - pub fn new( - bubble_merge: BubbleMergeConfig, - message_overlap_merge: MessageOverlapMergeConfig, - ) -> Self { - Self { - bubble_merge, - message_overlap_merge, + /// Create a new `MergeConfigBuilder` with default values. + pub fn builder() -> MergeConfigBuilder { + MergeConfigBuilder { + config: MergeConfig::default(), } } - /// Bubble merge strategy config. Default: enabled, 60s base, 120s jitter, 4 min neighbors, true. - /// base_interval is minimum 1s, max_jitter is minimum 0s. + /// Bubble merge strategy config. + /// + /// Default: BubbleMergeConfig::default() pub fn bubble_merge(&self) -> &BubbleMergeConfig { &self.bubble_merge } - /// Message overlap merge strategy config. Default: enabled, 60s base, 120s jitter, true. - /// base_interval is minimum 1s, max_jitter is minimum 0s. + /// Message overlap merge strategy config. + /// + /// Default: MessageOverlapMergeConfig::default() pub fn message_overlap_merge(&self) -> &MessageOverlapMergeConfig { &self.message_overlap_merge } } +impl MergeConfigBuilder { + /// Bubble merge strategy config. + /// + /// Default: BubbleMergeConfig::default() + pub fn bubble_merge(mut self, bubble_merge: BubbleMergeConfig) -> Self { + self.config.bubble_merge = bubble_merge; + self + } + + /// Message overlap merge strategy config. + /// + /// Default: MessageOverlapMergeConfig::default() + pub fn message_overlap_merge(mut self, message_overlap_merge: MessageOverlapMergeConfig) -> Self { + self.config.message_overlap_merge = message_overlap_merge; + self + } + + /// Build the `MergeConfig`. + pub fn build(self) -> MergeConfig { + self.config + } +} + /// Bootstrap process settings for peer discovery. #[derive(Debug, Clone)] pub struct BootstrapConfig { @@ -477,46 +659,65 @@ pub struct BootstrapConfigBuilder { } impl BootstrapConfigBuilder { - /// Max bootstrap records per topic per minute slot. If zero, we don't publish (PublisherConfig will be set to Disabled). Default: 5. + /// Max bootstrap records per topic per minute slot. + /// + /// If zero, we don't publish (PublisherConfig will be set to Disabled). + /// + /// Default: 5. pub fn max_bootstrap_records(mut self, max_records: usize) -> Self { self.config.max_bootstrap_records = max_records; self } - /// How long to wait when no peers are found before retrying. Default: 1500ms. + /// How long to wait when no peers are found before retrying. + /// + /// Default: 1500ms. pub fn no_peers_retry_interval(mut self, interval: Duration) -> Self { self.config.no_peers_retry_interval = interval; self } - /// How long to wait after joining a peer before attempting to join another. Default: 100ms. + /// How long to wait after joining a peer before attempting to join another. + /// + /// Default: 100ms. pub fn per_peer_join_settle_time(mut self, interval: Duration) -> Self { self.config.per_peer_join_settle_time = interval; self } - /// How long to wait after joining a peer before checking if joined successfully. Default: 500ms. + /// How long to wait after joining a peer before checking if joined successfully. + /// + /// Default: 500ms. pub fn join_confirmation_wait_time(mut self, interval: Duration) -> Self { self.config.join_confirmation_wait_time = interval; self } - /// How long to wait between DHT discovery attempts. Default: 2000ms. + /// How long to wait between DHT discovery attempts. + /// + /// Default: 2000ms. pub fn discovery_poll_interval(mut self, interval: Duration) -> Self { self.config.discovery_poll_interval = interval; self } - /// Whether to publish a bootstrap record unconditionally on startup before dht get. Default: true. + /// Whether to publish a bootstrap record unconditionally on startup before dht get. + /// + /// Default: true. pub fn publish_record_on_startup(mut self, publish: bool) -> Self { self.config.publish_record_on_startup = publish; self } - /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup (in this impl unix_minute and unix_minute-1 are both always fetched, if this is enabled than we first fetch unix_minute-2 and unix_minute-1). Default: false. - /// + /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. + /// + /// If this is enabled than we first fetch `unix_minute-1` and `unix_minute-1`. + /// /// If joining longer running, existing topics is priority, set to true. - /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly + /// at the same time into a topic without peers), set to false. + /// + /// Default: false. pub fn check_last_minute_record_first_on_startup(mut self, check: bool) -> Self { self.config.check_last_minute_record_first_on_startup = check; self @@ -536,40 +737,59 @@ impl BootstrapConfig { } } - /// Max bootstrap records per topic per minute slot. If zero, we don't publish (PublisherConfig will be set to Disabled, make sure to use config builder to enforce). Default: 5. + /// Max bootstrap records per topic per minute slot. + /// + /// If zero, we don't publish (PublisherConfig will be set to Disabled). + /// + /// Default: 5. pub fn max_bootstrap_records(&self) -> usize { self.max_bootstrap_records } - /// How long to wait when no peers are found before retrying. Default: 1500ms. + /// How long to wait when no peers are found before retrying. + /// + /// Default: 1500ms. pub fn no_peers_retry_interval(&self) -> Duration { self.no_peers_retry_interval } - /// How long to wait after joining a peer before attempting to join another. Default: 100ms. + /// How long to wait after joining a peer before attempting to join another. + /// + /// Default: 100ms. pub fn per_peer_join_settle_time(&self) -> Duration { self.per_peer_join_settle_time } - /// How long to wait after joining a peer before checking if joined successfully. Default: 500ms. + /// How long to wait after joining a peer before checking if joined successfully. + /// + /// Default: 500ms. pub fn join_confirmation_wait_time(&self) -> Duration { self.join_confirmation_wait_time } - /// How long to wait between DHT discovery attempts. Default: 2000ms. + /// How long to wait between DHT discovery attempts. + /// + /// Default: 2000ms. pub fn discovery_poll_interval(&self) -> Duration { self.discovery_poll_interval } - /// Whether to publish a bootstrap record unconditionally on startup before dht get. Default: true. + /// Whether to publish a bootstrap record unconditionally on startup before dht get. + /// + /// Default: true. pub fn publish_record_on_startup(&self) -> bool { self.publish_record_on_startup } - /// Whether to check the last minute record unix_minute-1 before the current time window unix_minute on startup (in this impl unix_minute and unix_minute-1 are both always fetched, if this is enabled than we first fetch unix_minute-2 and unix_minute-1). Default: false. - /// + /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. + /// + /// If this is enabled than we first fetch `unix_minute-1` and `unix_minute-1`. + /// /// If joining longer running, existing topics is priority, set to true. - /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly at the same time into a topic without peers), set to false. + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly + /// at the same time into a topic without peers), set to false. + /// + /// Default: false. pub fn check_last_minute_record_first_on_startup(&self) -> bool { self.check_last_minute_record_first_on_startup } @@ -596,33 +816,46 @@ impl Config { } } - /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. - /// base_interval is minimum 1s, max_jitter is minimum 0s. + /// Publisher strategy config. + /// + /// Default: PublisherConfig::default(). pub fn publisher_config(&self) -> &PublisherConfig { &self.publisher_config } - /// DHT operation settings. Default: DhtConfig::default(). + /// DHT operation settings. + /// + /// Default: DhtConfig::default(). pub fn dht_config(&self) -> &DhtConfig { &self.dht_config } - /// Bootstrap strategy settings. Default: max 5 bootstrap records per topic per minute, 1500ms no peers retry interval, 100ms per peer join settle time, 500ms join confirmation wait time, 2000ms discovery poll interval. + /// Bootstrap strategy settings. + /// + /// Default: BootstrapConfig::default(). pub fn bootstrap_config(&self) -> &BootstrapConfig { &self.bootstrap_config } - /// Max peers to join simultaneously (min 1). Default: 4. + /// Max peers to join simultaneously. + /// + /// Minimum: 1. + /// + /// Default: 4. pub fn max_join_peer_count(&self) -> usize { self.max_join_peer_count } - /// Timeout settings. Default: TimeoutConfig::default(). + /// Timeout settings. + /// + /// Default: TimeoutConfig::default(). pub fn timeouts(&self) -> &TimeoutConfig { &self.timeouts } - /// Merge strategy settings. Default: bubble and overlap merges enabled. + /// Merge strategy settings. + /// + /// Default: bubble and overlap merges enabled. pub fn merge_config(&self) -> &MergeConfig { &self.merge_config } @@ -648,44 +881,61 @@ pub struct ConfigBuilder { } impl ConfigBuilder { - /// Merge strategy settings. Default: bubble and overlap merges enabled. + /// Merge strategy settings. + /// + /// Default: MergeConfig::default(). pub fn merge_config(mut self, merge_config: MergeConfig) -> Self { self.config.merge_config = merge_config; self } - /// Publisher strategy config. Default: enabled, 10s initial delay, 10s base interval, 50s jitter. - /// base_interval minimum is 1s, initial_delay minimum is 0s, max_jitter is minimum 0s. + /// Publisher strategy config. + /// + /// base_interval > `Duration::ZERO` + /// + /// Default: PublisherConfig::default(). pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { self.config.publisher_config = publisher_config; self } - /// DHT operation settings. Default: DhtConfig::default(). + /// DHT operation settings. + /// + /// Default: DhtConfig::default(). pub fn dht_config(mut self, dht_config: DhtConfig) -> Self { self.config.dht_config = dht_config; self } - /// Bootstrap strategy settings. Default: max 5 bootstrap records per topic per minute, 1500ms no peers retry interval, 100ms per peer join settle time, 500ms join confirmation wait time, 2000ms discovery poll interval. + /// Bootstrap strategy settings. + /// + /// Default: BootstrapConfig::default(). pub fn bootstrap_config(mut self, bootstrap_config: BootstrapConfig) -> Self { self.config.bootstrap_config = bootstrap_config; self } - /// Max peers to join simultaneously (min 1). Default: 4. - pub fn max_join_peer_count(mut self, max_peers: usize) -> Self { - self.config.max_join_peer_count = max_peers.max(1); + /// Max peers to join simultaneously. + /// + /// Minimum: 1. + /// + /// Default: 4. + pub fn max_join_peer_count(mut self, max_peers: NonZeroU32) -> Self { + self.config.max_join_peer_count = max_peers.get() as usize; self } - /// Timeout settings. Default: TimeoutConfig::default(). + /// Timeout settings. + /// + /// Default: TimeoutConfig::default(). pub fn timeouts(mut self, timeouts: TimeoutConfig) -> Self { self.config.timeouts = timeouts; self } - /// Build the `Config`. If `max_bootstrap_records` is zero, `PublisherConfig` is set to `Disabled`. + /// Build the `Config`. + /// + /// If `max_bootstrap_records` is zero, `PublisherConfig` is set to `Disabled`. pub fn build(self) -> Config { let mut config = self.config; if config.bootstrap_config.max_bootstrap_records == 0 { diff --git a/src/crypto/record.rs b/src/crypto/record.rs index e975f98..e484280 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -269,7 +269,7 @@ impl RecordPublisher { .put_mutable( sign_key.clone(), Some(salt.to_vec()), - encrypted_record.to_bytes().to_vec(), + encrypted_record.to_bytes()?, next_seq_num, ) .await?; @@ -353,7 +353,7 @@ impl EncryptedRecord { } /// Serialize to bytes (length-prefixed format). - pub fn to_bytes(&self) -> Vec { + pub fn to_bytes(&self) -> Result> { let mut buf = Vec::new(); let encrypted_record_len = self.encrypted_record.len() as u32; buf.extend_from_slice(&encrypted_record_len.to_le_bytes()); @@ -361,9 +361,9 @@ impl EncryptedRecord { buf.extend_from_slice(&self.encrypted_decryption_key); if buf.len() > Self::MAX_SIZE { - unreachable!("EncryptedRecord serialization exceeds maximum size, the max is set generously so this should never happen, if so your code is using it manually"); + bail!("EncryptedRecord serialization exceeds maximum size, the max is set generously so this should never happen, if so your code is using it manually"); } - buf + Ok(buf) } /// Deserialize from bytes. @@ -501,7 +501,7 @@ impl Record { self.unix_minute } - /// Get the node ID (publisher's public key). + /// Get the pub_key (publisher's public key). pub fn pub_key(&self) -> [u8; 32] { self.pub_key } diff --git a/src/dht.rs b/src/dht.rs index b26f560..99578d8 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -153,21 +153,20 @@ impl DhtActor { self.reset().await?; - let jitter_ms = if self.config.max_retry_jitter().as_millis() > 0 { - rand::random::() % self.config.max_retry_jitter().as_millis() + let jitter = if self.config.max_retry_jitter() > Duration::ZERO { + Duration::from_micros(rand::random::() % self.config.max_retry_jitter().as_micros() as u64) } else { - 0 + Duration::ZERO }; - let retry_interval = - (self.config.base_retry_interval().as_millis() + jitter_ms).max(1000); + let retry_interval = self.config.base_retry_interval() + jitter; tracing::debug!( "DHTActor: put_mutable attempt {}/{} failed, retrying in {}ms", i + 1, 1 + self.config.retries(), - retry_interval + retry_interval.as_millis() ); - tokio::time::sleep(Duration::from_millis(retry_interval as u64)).await; + tokio::time::sleep(retry_interval).await; } Ok(()) } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 6313594..6dfe93b 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -86,14 +86,14 @@ impl BubbleMergeActor { if let Err(e) = self.merge().await { tracing::warn!("BubbleMerge: error during merge: {:?}", e); } - let jitter_ms = if self.max_jitter.as_millis() > 0 { - rand::random::() % self.max_jitter.as_millis() + let jitter = if self.max_jitter > Duration::ZERO { + Duration::from_micros(rand::random::() % self.max_jitter.as_micros() as u64) } else { - 0 + Duration::ZERO }; - let next_interval = self.base_interval.as_millis() + jitter_ms; - tracing::debug!("BubbleMerge: next check in {}ms", next_interval); - self.ticker.reset_after(Duration::from_millis(next_interval as u64)); + let next_interval = self.base_interval + jitter; + tracing::debug!("BubbleMerge: next check in {}ms", next_interval.as_millis()); + self.ticker.reset_after(next_interval); } _ = self.cancel_token.cancelled() => { break Ok(()); @@ -124,6 +124,7 @@ impl BubbleMergeActor { neighbors.len(), self.min_neighbors ); + let self_pub_key = EndpointId::from_verifying_key(self.record_publisher.pub_key()); let pub_keys = records .iter() .flat_map(|record| { @@ -136,7 +137,7 @@ impl BubbleMergeActor { if active_peer == [0; 32] || neighbors.contains(&active_peer) || active_peer == record.pub_key() - || active_peer.eq(self.record_publisher.pub_key().as_bytes()) + || active_peer.eq(self_pub_key.as_bytes()) { None } else { @@ -148,8 +149,7 @@ impl BubbleMergeActor { vec![] }; if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) - && pub_key - != EndpointId::from_verifying_key(self.record_publisher.pub_key()) + && !pub_key.eq(&self_pub_key) && !neighbors.contains(&pub_key) { pub_keys.push(pub_key); diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 6360681..d6874cb 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -77,14 +77,14 @@ impl MessageOverlapMergeActor { if let Err(e) = self.merge().await { tracing::warn!("MessageOverlapMerge: error during merge: {:?}", e); } - let jitter_ms = if self.max_jitter.as_millis() > 0 { - rand::random::() % self.max_jitter.as_millis() + let jitter = if self.max_jitter > Duration::ZERO { + Duration::from_micros(rand::random::() % self.max_jitter.as_micros() as u64) } else { - 0 + Duration::ZERO }; - let next_interval = self.base_interval.as_millis() + jitter_ms; - tracing::debug!("MessageOverlapMerge: next check in {}ms", next_interval); - self.ticker.reset_after(Duration::from_millis(next_interval as u64)); + let next_interval = self.base_interval + jitter; + tracing::debug!("MessageOverlapMerge: next check in {}ms", next_interval.as_millis()); + self.ticker.reset_after(next_interval); } _ = self.cancel_token.cancelled() => { break Ok(()); @@ -121,7 +121,6 @@ impl MessageOverlapMergeActor { .collect::>(); !remote_hashes.is_empty() - && !last_message_hashes.is_empty() && remote_hashes.iter().all(|last_message_hash| { !last_message_hashes.contains(*last_message_hash) }) @@ -138,13 +137,13 @@ impl MessageOverlapMergeActor { if !peers_to_join.is_empty() { let active_neighbors = self.gossip_receiver.neighbors().await?; + let self_pub_key = EndpointId::from_verifying_key(self.record_publisher.pub_key()); let pub_keys = peers_to_join .iter() .flat_map(|&record| { let mut peers = vec![]; if let Ok(pub_key) = EndpointId::from_bytes(&record.pub_key()) - && pub_key - != EndpointId::from_verifying_key(self.record_publisher.pub_key()) + && pub_key != self_pub_key { peers.push(pub_key); } @@ -154,10 +153,7 @@ impl MessageOverlapMergeActor { continue; } if let Ok(pub_key) = EndpointId::from_bytes(&active_peer) - && pub_key - != EndpointId::from_verifying_key( - self.record_publisher.pub_key(), - ) + && pub_key != self_pub_key { peers.push(pub_key); } diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index e8a080b..ef7c8bf 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -217,7 +217,9 @@ impl BootstrapActor { let mut v = vec![record.pub_key()]; if let Ok(record_content) = record.content::() { for peer in record_content.active_peers { - if peer != [0; 32] { + if peer != [0; 32] + && !peer.eq(record_publisher.pub_key().as_bytes()) + { v.push(peer); } } diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index f4c2410..cc4ad9f 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -75,14 +75,14 @@ impl PublisherActor { if let Err(e) = self.publish().await { tracing::warn!("Publisher: failed to publish record: {:?}", e); } - let jitter_ms = if self.max_jitter.as_millis() > 0 { - rand::random::() % self.max_jitter.as_millis() + let jitter = if self.max_jitter > Duration::ZERO { + Duration::from_micros(rand::random::() % self.max_jitter.as_micros() as u64) } else { - 0 + Duration::ZERO }; - let next_interval = self.base_interval.as_millis() + jitter_ms; - tracing::debug!("Publisher: next publish in {}ms", next_interval); - self.ticker.reset_after(Duration::from_millis(next_interval as u64)); + let next_interval = self.base_interval + jitter; + tracing::debug!("Publisher: next publish in {}ms", next_interval.as_millis()); + self.ticker.reset_after(next_interval); } _ = self.cancel_token.cancelled() => { break Ok(()); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index dc809fc..ed70d99 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -50,11 +50,9 @@ impl From for TopicId { } } -impl TryFrom> for TopicId { - type Error = anyhow::Error; - - fn try_from(bytes: Vec) -> Result { - Ok(Self::new(String::from_utf8(bytes)?)) +impl From> for TopicId { + fn from(bytes: Vec) -> Self { + Self::new(bytes) } } @@ -62,9 +60,9 @@ impl TopicId { /// Create a new topic ID from a string. /// /// String is hashed with SHA512; the first 32 bytes produce the identifier. - pub fn new(raw: String) -> Self { + pub fn new(raw: impl Into>) -> Self { let mut raw_hash = sha2::Sha512::new(); - raw_hash.update(raw.as_bytes()); + raw_hash.update(raw.into()); Self( raw_hash.finalize()[..32] @@ -152,35 +150,29 @@ impl Topic { ); let bootstrap_done = bootstrap.bootstrap().await?; + tokio::spawn({ + let api = Arc::downgrade(&api); + let cancel_token = cancel_token.clone(); + async move { + if let Err(err) = wait_for_bootstrap_and_spawn_workers( + api, + bootstrap_done, + record_publisher.clone(), + cancel_token.clone(), + ) + .await + { + tracing::error!("failed to start topic: {}", err); + } + } + }); + if !async_bootstrap { tracing::debug!("Topic: waiting for bootstrap to complete"); - - wait_for_bootstrap_and_spawn_workers( - Arc::downgrade(&api), - bootstrap_done, - record_publisher.clone(), - cancel_token.clone(), - ) - .await?; + bootstrap.gossip_receiver().await?.joined().await?; tracing::debug!("Topic: bootstrap completed"); } else { tracing::debug!("Topic: bootstrap started asynchronously"); - tokio::spawn({ - let api = Arc::downgrade(&api); - let cancel_token = cancel_token.clone(); - async move { - if let Err(err) = wait_for_bootstrap_and_spawn_workers( - api, - bootstrap_done, - record_publisher.clone(), - cancel_token.clone(), - ) - .await - { - tracing::error!("failed to start topic: {}", err); - } - } - }); } Ok(Self { api, cancel_token }) @@ -310,30 +302,27 @@ async fn wait_for_bootstrap_and_spawn_workers( impl TopicActor { pub async fn start_publishing(&mut self) -> Result<()> { - if let PublisherConfig::Enabled { - initial_delay, - base_interval, - max_jitter, - fail_topic_creation_on_publishing_startup_failure, - } = self.record_publisher.config().publisher_config() + if let PublisherConfig::Enabled(config) = self.record_publisher.config().publisher_config() { tracing::debug!("TopicActor: initializing publisher"); - let publisher = Publisher::new( - self.record_publisher.clone(), - self.bootstrap.gossip_receiver().await?, - self.cancel_token.clone(), - *initial_delay, - *base_interval, - *max_jitter, - ); - - match publisher { + let publisher = async { + Publisher::new( + self.record_publisher.clone(), + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + config.initial_delay(), + config.base_interval(), + config.max_jitter(), + ) + }; + + match publisher.await { Ok(publisher) => { self.publisher = Some(publisher); tracing::debug!("TopicActor: publisher started"); - }, + } Err(err) => { - if *fail_topic_creation_on_publishing_startup_failure { + if config.fail_topic_creation_on_publishing_startup_failure() { return Err(anyhow::anyhow!("failed to start publisher: {}", err)); } else { tracing::warn!( @@ -348,32 +337,30 @@ impl TopicActor { } pub async fn start_bubble_merge(&mut self) -> Result<()> { - if let BubbleMergeConfig::Enabled { - base_interval, - max_jitter, - min_neighbors, - fail_topic_creation_on_merge_startup_failure, - } = self.record_publisher.config().merge_config().bubble_merge() + if let BubbleMergeConfig::Enabled(config) = + self.record_publisher.config().merge_config().bubble_merge() { tracing::debug!("TopicActor: initializing bubble merge"); - let bubble_merge = BubbleMerge::new( - self.record_publisher.clone(), - self.bootstrap.gossip_sender().await?, - self.bootstrap.gossip_receiver().await?, - self.cancel_token.clone(), - self.record_publisher.config().max_join_peer_count().max(1), - *base_interval, - *max_jitter, - *min_neighbors, - ); - - match bubble_merge { + let bubble_merge = async { + BubbleMerge::new( + self.record_publisher.clone(), + self.bootstrap.gossip_sender().await?, + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + self.record_publisher.config().max_join_peer_count().max(1), + config.base_interval(), + config.max_jitter(), + config.min_neighbors(), + ) + }; + + match bubble_merge.await { Ok(bubble_merge) => { self.bubble_merge = Some(bubble_merge); tracing::debug!("TopicActor: bubble merge started"); } Err(err) => { - if *fail_topic_creation_on_merge_startup_failure { + if config.fail_topic_creation_on_merge_startup_failure() { return Err(anyhow::anyhow!("failed to start bubble merge: {}", err)); } else { tracing::warn!( @@ -388,33 +375,32 @@ impl TopicActor { } pub async fn start_message_overlap_merge(&mut self) -> Result<()> { - if let MessageOverlapMergeConfig::Enabled { - base_interval, - max_jitter, - fail_topic_creation_on_merge_startup_failure, - } = self + if let MessageOverlapMergeConfig::Enabled(config) = self .record_publisher .config() .merge_config() .message_overlap_merge() { tracing::debug!("TopicActor: initializing message overlap merge"); - let message_overlap_merge = MessageOverlapMerge::new( - self.record_publisher.clone(), - self.bootstrap.gossip_sender().await?, - self.bootstrap.gossip_receiver().await?, - self.cancel_token.clone(), - self.record_publisher.config().max_join_peer_count().max(1), - *base_interval, - *max_jitter, - ); - match message_overlap_merge { + let message_overlap_merge = async { + MessageOverlapMerge::new( + self.record_publisher.clone(), + self.bootstrap.gossip_sender().await?, + self.bootstrap.gossip_receiver().await?, + self.cancel_token.clone(), + self.record_publisher.config().max_join_peer_count().max(1), + config.base_interval(), + config.max_jitter(), + ) + }; + + match message_overlap_merge.await { Ok(message_overlap_merge) => { self.message_overlap_merge = Some(message_overlap_merge); tracing::debug!("TopicActor: message overlap merge started"); } Err(err) => { - if *fail_topic_creation_on_merge_startup_failure { + if config.fail_topic_creation_on_merge_startup_failure() { return Err(anyhow::anyhow!( "failed to start message overlap merge: {}", err diff --git a/tests/tests.rs b/tests/tests.rs index dabd067..67b75cb 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -133,7 +133,7 @@ fn test_encrypted_record_serialization() { let encrypted = record.encrypt(&encryption_key); // Test serialization roundtrip - let bytes = encrypted.to_bytes(); + let bytes = encrypted.to_bytes().expect("record max size must be < EncryptedRecord::MAX_SIZE"); let deserialized = EncryptedRecord::from_bytes(bytes).expect("failed to deserialize encrypted record"); @@ -254,10 +254,11 @@ async fn test_multiple_receivers_all_get_events() { let config = Config::builder() .publisher_config(PublisherConfig::Disabled) - .merge_config(MergeConfig::new( - BubbleMergeConfig::Disabled, - MessageOverlapMergeConfig::Disabled, - )) + .merge_config(MergeConfig::builder() + .bubble_merge(BubbleMergeConfig::Disabled) + .message_overlap_merge(MessageOverlapMergeConfig::Disabled) + .build() + ) .build(); let topic_id = TopicId::new("test-multi-receiver".to_string()); From fa81706c83a177fa040e03ef10312c948f9206f2 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 17:17:41 +0200 Subject: [PATCH 24/30] chore: docs cleanup + wait for spawn_workers tasks --- README.md | 22 ++++++++---- src/config.rs | 16 ++++----- src/gossip/topic/topic.rs | 70 +++++++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 0cad8cb..bc3a98a 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,10 @@ The E2E test confirms multiple nodes discover each other via DHT and join the sa ## Upgrading from 0.2 to 0.3 +**0.3** resolves several **stability issues** present in **0.2**. Circular and dangling references between actors caused resource leaks and **tasks could outlive topic and channel handles** (after `Topic`, `GossipSender` and `GossipReceiver` were dropped). Reduced unneccessary DHT writes and reads, adjusted timeouts, **reduced time to bootstrap**. All actor lifecycles are now token-gated, references now work as expected (if all dropped, all background tasks shut down gracefully), and many more improvements. If you find any issues, please report them. + +**tldr: background tasks shutdown as expected, faster bootstrap time, better all around** + ### Breaking changes **`RecordTopic` removed, use `TopicId` instead** @@ -259,14 +263,15 @@ Config::builder() **Disable merge strategies or publishing** ```rust,ignore -// Run without any merge or publishing (bootstrap-only) +// Run without any merge strategies (bootstrap-only) let config = Config::builder() - .publisher_config(PublisherConfig::Disabled) - .merge_config(MergeConfig::new( - BubbleMergeConfig::Disabled, - MessageOverlapMergeConfig::Disabled, - )) - .build(); + .merge_config( + MergeConfig::builder() + .bubble_merge(BubbleMergeConfig::Disabled) + .message_overlap_merge(MessageOverlapMergeConfig::Disabled) + .build(), + ) + .build() ``` **`RecordPublisher::builder()` for ergonomic construction** @@ -277,6 +282,9 @@ let publisher = RecordPublisher::builder("my-topic", signing_key, b"secret") .config(config) .build(); ``` +## Todo's + + ## License diff --git a/src/config.rs b/src/config.rs index 64a5b40..c4b43d8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -112,7 +112,7 @@ impl DhtConfigBuilder { self } - /// Base delay between retries. + /// Base delay between retries. No-op if `interval` is `Duration::ZERO`. /// /// If `base_retry_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_retry_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. @@ -292,7 +292,7 @@ impl BubbleMergeConfigInner { } impl BubbleMergeConfigBuilder { - /// Base interval for bubble merge attempts. + /// Base interval for bubble merge attempts. No-op if `interval` is `Duration::ZERO`. /// /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. @@ -407,14 +407,16 @@ impl MessageOverlapMergeConfigInner { } impl MessageOverlapMergeConfigBuilder { - /// Base interval for message overlap merge attempts. + /// Base interval for message overlap merge attempts. No-op if `interval` is `Duration::ZERO`. /// /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. /// /// Default: 60s. pub fn base_interval(mut self, interval: Duration) -> Self { - self.config.base_interval = interval; + if interval > Duration::ZERO { + self.config.base_interval = interval; + } self } @@ -529,7 +531,7 @@ impl PublisherConfigBuilder { self } - /// Base interval for publisher attempts. + /// Base interval for publisher attempts. No-op if `interval` is `Duration::ZERO`. /// /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. @@ -889,9 +891,7 @@ impl ConfigBuilder { self } - /// Publisher strategy config. - /// - /// base_interval > `Duration::ZERO` + /// Publisher strategy config. /// /// Default: PublisherConfig::default(). pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index ed70d99..bbaab2b 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -6,7 +6,7 @@ use std::{ }; use crate::{ - BubbleMergeConfig, GossipSender, MessageOverlapMergeConfig, + BubbleMergeConfig, Config, GossipSender, MessageOverlapMergeConfig, config::PublisherConfig, gossip::{ merge::{BubbleMerge, MessageOverlapMerge}, @@ -27,7 +27,7 @@ use tokio_util::sync::CancellationToken; /// ```ignore /// let topic_id = TopicId::new("chat-room-1".to_string()); /// ``` -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct TopicId([u8; 32]); impl FromStr for TopicId { @@ -150,19 +150,22 @@ impl Topic { ); let bootstrap_done = bootstrap.bootstrap().await?; + let config = record_publisher.config().clone(); tokio::spawn({ let api = Arc::downgrade(&api); + let config = config.clone(); let cancel_token = cancel_token.clone(); async move { - if let Err(err) = wait_for_bootstrap_and_spawn_workers( - api, - bootstrap_done, - record_publisher.clone(), - cancel_token.clone(), - ) - .await - { - tracing::error!("failed to start topic: {}", err); + if let Err(err) = wait_for_bootstrap(bootstrap_done, cancel_token.clone()).await { + tracing::warn!("bootstrap failed: {}", err); + return; + } + + if async_bootstrap { + tracing::debug!("Bootstrap completed, now spawning workers"); + if let Err(err) = spawn_workers(api, config, cancel_token.clone()).await { + tracing::warn!("failed to spawn workers: {}", err); + } } } }); @@ -170,6 +173,11 @@ impl Topic { if !async_bootstrap { tracing::debug!("Topic: waiting for bootstrap to complete"); bootstrap.gossip_receiver().await?.joined().await?; + if let Err(err) = + spawn_workers(Arc::downgrade(&api), config, cancel_token.clone()).await + { + tracing::warn!("failed to spawn workers: {}", err); + } tracing::debug!("Topic: bootstrap completed"); } else { tracing::debug!("Topic: bootstrap started asynchronously"); @@ -215,19 +223,26 @@ impl Topic { } } -async fn wait_for_bootstrap_and_spawn_workers( - api: Weak>, +async fn wait_for_bootstrap( bootstrap_done: tokio::sync::oneshot::Receiver>, - record_publisher: crate::crypto::RecordPublisher, cancel_token: CancellationToken, ) -> Result<()> { - if let Ok(Ok(_)) = bootstrap_done.await - && !cancel_token.is_cancelled() - { - if matches!( - record_publisher.config().publisher_config(), - PublisherConfig::Enabled { .. } - ) { + if let Ok(Ok(_)) = bootstrap_done.await { + Ok(()) + } else { + tracing::error!("Topic: bootstrap failed or cancelled, shutting down topic"); + cancel_token.cancel(); + Err(anyhow::anyhow!("bootstrap failed or cancelled")) + } +} + +async fn spawn_workers( + api: Weak>, + config: Config, + cancel_token: CancellationToken, +) -> Result<()> { + if !cancel_token.is_cancelled() { + if matches!(config.publisher_config(), PublisherConfig::Enabled { .. }) { tracing::debug!("Topic: starting publisher"); match api.upgrade() { Some(api) => { @@ -245,7 +260,7 @@ async fn wait_for_bootstrap_and_spawn_workers( } if matches!( - record_publisher.config().merge_config().bubble_merge(), + config.merge_config().bubble_merge(), BubbleMergeConfig::Enabled { .. } ) { tracing::debug!("Topic: starting bubble merge"); @@ -265,10 +280,7 @@ async fn wait_for_bootstrap_and_spawn_workers( } if matches!( - record_publisher - .config() - .merge_config() - .message_overlap_merge(), + config.merge_config().message_overlap_merge(), MessageOverlapMergeConfig::Enabled { .. } ) { tracing::debug!("Topic: starting message overlap merge"); @@ -291,12 +303,12 @@ async fn wait_for_bootstrap_and_spawn_workers( } } } - tracing::debug!("Topic: fully initialized"); + tracing::debug!("Topic: spawn_worker finished"); Ok(()) } else { - tracing::error!("Topic: bootstrap failed or cancelled, shutting down topic"); + tracing::warn!("Topic: cancelled before workers could be spawned"); cancel_token.cancel(); - Err(anyhow::anyhow!("bootstrap failed or cancelled")) + Err(anyhow::anyhow!("cancelled before workers could be spawned")) } } From 77015838c078721e61c8f6966fbfc95c226cbba0 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 17:39:33 +0200 Subject: [PATCH 25/30] chore: review fixes --- README.md | 16 ++++++++++------ src/config.rs | 2 +- src/gossip/topic/topic.rs | 35 ++++++++++++++++++----------------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index bc3a98a..474f314 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The E2E test confirms multiple nodes discover each other via DHT and join the sa ## Upgrading from 0.2 to 0.3 -**0.3** resolves several **stability issues** present in **0.2**. Circular and dangling references between actors caused resource leaks and **tasks could outlive topic and channel handles** (after `Topic`, `GossipSender` and `GossipReceiver` were dropped). Reduced unneccessary DHT writes and reads, adjusted timeouts, **reduced time to bootstrap**. All actor lifecycles are now token-gated, references now work as expected (if all dropped, all background tasks shut down gracefully), and many more improvements. If you find any issues, please report them. +**0.3** resolves several **stability issues** present in **0.2**. Circular and dangling references between actors caused resource leaks and **tasks could outlive topic and channel handles** (after `Topic`, `GossipSender` and `GossipReceiver` were dropped). Reduced unnecessary DHT writes and reads, adjusted timeouts, **reduced time to bootstrap**. All actor lifecycles are now token-gated, references now work as expected (if all dropped, all background tasks shut down gracefully), and many more improvements. If you find any issues, please report them. **tldr: background tasks shutdown as expected, faster bootstrap time, better all around** @@ -151,9 +151,12 @@ use distributed_topic_tracker::Config; let publisher = RecordPublisher::new(topic, signing_key, None, secret, Config::default()); // or use the new builder: -let publisher = RecordPublisher::builder(topic, signing_key, secret) - .config(Config::builder().max_join_peer_count(8).build()) - .secret_rotation(rotation_handle) +let publisher = RecordPublisher::builder(topic_id, signing_key, initial_secret) + .config( + Config::builder() + .max_join_peer_count(std::num::NonZeroU32::new(8).expect("must be > 0")) + .build(), + ) .build(); ``` @@ -271,7 +274,7 @@ let config = Config::builder() .message_overlap_merge(MessageOverlapMergeConfig::Disabled) .build(), ) - .build() + .build(); ``` **`RecordPublisher::builder()` for ergonomic construction** @@ -282,9 +285,10 @@ let publisher = RecordPublisher::builder("my-topic", signing_key, b"secret") .config(config) .build(); ``` -## Todo's +## Todo's +- [] Network degradation testing ## License diff --git a/src/config.rs b/src/config.rs index c4b43d8..ae91099 100644 --- a/src/config.rs +++ b/src/config.rs @@ -713,7 +713,7 @@ impl BootstrapConfigBuilder { /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. /// - /// If this is enabled than we first fetch `unix_minute-1` and `unix_minute-1`. + /// If this is enabled, we first fetch `unix_minute-1` and `unix_minute-2`. /// /// If joining longer running, existing topics is priority, set to true. /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index bbaab2b..c8e95cd 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -33,26 +33,27 @@ pub struct TopicId([u8; 32]); impl FromStr for TopicId { type Err = anyhow::Error; - fn from_str(s: &str) -> std::result::Result { - Ok(Self::new(s.to_string())) + fn from_str(topic_name: &str) -> std::result::Result { + Ok(Self::new(topic_name.to_string())) } } impl From<&str> for TopicId { - fn from(s: &str) -> Self { - Self::new(s.to_string()) + fn from(topic_name: &str) -> Self { + Self::new(topic_name.to_string()) } } impl From for TopicId { - fn from(s: String) -> Self { - Self::new(s) + fn from(topic_name: String) -> Self { + Self::new(topic_name) } } - +/// Treats `bytes` as a topic *name* and SHA-512 hashes them. +/// For a pre-computed 32-byte hash, use [`TopicId::from_hash`] instead. impl From> for TopicId { - fn from(bytes: Vec) -> Self { - Self::new(bytes) + fn from(topic_name: Vec) -> Self { + Self::new(topic_name) } } @@ -60,14 +61,14 @@ impl TopicId { /// Create a new topic ID from a string. /// /// String is hashed with SHA512; the first 32 bytes produce the identifier. - pub fn new(raw: impl Into>) -> Self { - let mut raw_hash = sha2::Sha512::new(); - raw_hash.update(raw.into()); + pub fn new(topic_name: impl Into>) -> Self { + let mut topic_name_hash = sha2::Sha512::new(); + topic_name_hash.update(topic_name.into()); Self( - raw_hash.finalize()[..32] + topic_name_hash.finalize()[..32] .try_into() - .expect("hashing 'raw' failed"), + .expect("hashing 'topic_name' failed"), ) } @@ -164,6 +165,7 @@ impl Topic { if async_bootstrap { tracing::debug!("Bootstrap completed, now spawning workers"); if let Err(err) = spawn_workers(api, config, cancel_token.clone()).await { + cancel_token.cancel(); tracing::warn!("failed to spawn workers: {}", err); } } @@ -177,6 +179,8 @@ impl Topic { spawn_workers(Arc::downgrade(&api), config, cancel_token.clone()).await { tracing::warn!("failed to spawn workers: {}", err); + cancel_token.cancel(); + return Err(anyhow::anyhow!("failed to spawn workers: {}", err)); } tracing::debug!("Topic: bootstrap completed"); } else { @@ -247,7 +251,6 @@ async fn spawn_workers( match api.upgrade() { Some(api) => { if let Err(err) = api.call(act!(actor => actor.start_publishing())).await { - cancel_token.cancel(); return Err(anyhow::anyhow!("failed to start publisher: {err}")); } } @@ -267,7 +270,6 @@ async fn spawn_workers( match api.upgrade() { Some(api) => { if let Err(err) = api.call(act!(actor => actor.start_bubble_merge())).await { - cancel_token.cancel(); return Err(anyhow::anyhow!("failed to start bubble merge: {err}")); } } @@ -290,7 +292,6 @@ async fn spawn_workers( .call(act!(actor => actor.start_message_overlap_merge())) .await { - cancel_token.cancel(); return Err(anyhow::anyhow!( "failed to start message overlap merge: {err}" )); From 228de6ecdb9608805cf68dd0ff9fe859e0b55222 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 19:52:52 +0200 Subject: [PATCH 26/30] fix: moved TopicId to crypto + fixed interval jitter calc + check_last_minute_record_first_on_startup -> check_older_records_first_on_startup + publishing only to current unix_minute + docs fixes --- PROTOCOL.md | 10 +- README.md | 6 +- examples/chat.rs | 3 +- examples/chat_no_wait.rs | 12 ++ examples/e2e_test.rs | 1 + examples/full_config.rs | 2 +- examples/secret_rotation.rs | 1 + src/config.rs | 304 ++++++++++++++-------------- src/crypto/mod.rs | 2 +- src/crypto/record.rs | 87 +++++++- src/dht.rs | 2 +- src/gossip/merge/bubble.rs | 12 +- src/gossip/merge/message_overlap.rs | 35 ++-- src/gossip/mod.rs | 2 +- src/gossip/topic/bootstrap.rs | 57 +++--- src/gossip/topic/mod.rs | 2 +- src/gossip/topic/publisher.rs | 2 +- src/gossip/topic/topic.rs | 74 +------ src/lib.rs | 4 +- 19 files changed, 322 insertions(+), 296 deletions(-) diff --git a/PROTOCOL.md b/PROTOCOL.md index 12595fb..bb59641 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -108,9 +108,9 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne - If connected, exit bootstrap loop 3. **Time Window Selection** - - If `check_last_minute_record_first_on_startup` is enabled and this is the first attempt: check previous `unix minute-1` and `unix_minute-2` + - If `check_older_records_first_on_startup` is enabled and this is the first attempt: check previous `unix minute-1` and `unix_minute-2` - Otherwise: check current unix minute `unix_minute` and `unix_minute-1` - - Both `unix_minute` and `unix_minute-1` records are always fetched + - Two records are always fetched 4. **Record Discovery** - Call `get_records()` for both `unix_minute - 1` and `unix_minute`: @@ -147,7 +147,7 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne - Wait `discovery_poll_interval` (default 2000ms) and continue loop ### Error Handling -- DHT timeouts return empty record sets (non-fatal) +- DHT timeouts return error after retries are exhausted - Failed record decryption/verification are treated as invalid records and ignored - Failed peer connections don't interrupt the process - Publishing failures don't prevent continued bootstrapping @@ -165,7 +165,7 @@ flowchart TD D -- Yes --> Z[Return Success] D -- No --> E[Determine Unix Minute] - E --> F{First attempt AND check_last_minute_record_first?} + E --> F{First attempt AND check_older_records_first?} F -- Yes --> G[Use Previous Minute unix_minute = -1] F -- No --> H[Use Current Minute unix_minute = 0] @@ -399,7 +399,7 @@ A one-time key encryption scheme is used to protect record content while allowin ``` [4 bytes: encrypted_record_length (little-endian u32)] [variable: encrypted_record data] -[variable: encrypted_decryption_key data (88 bytes)] +[88 bytes: encrypted_decryption_key] ``` ### Security Properties diff --git a/README.md b/README.md index 474f314..4886c73 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The E2E test confirms multiple nodes discover each other via DHT and join the sa ## Upgrading from 0.2 to 0.3 -**0.3** resolves several **stability issues** present in **0.2**. Circular and dangling references between actors caused resource leaks and **tasks could outlive topic and channel handles** (after `Topic`, `GossipSender` and `GossipReceiver` were dropped). Reduced unnecessary DHT writes and reads, adjusted timeouts, **reduced time to bootstrap**. All actor lifecycles are now token-gated, references now work as expected (if all dropped, all background tasks shut down gracefully), and many more improvements. If you find any issues, please report them. +**0.3** resolves several **stability issues** present in **0.2**. Circular and dangling references between actors caused resource leaks and **tasks could outlive topic and channel handles** (after `Topic`, `GossipSender` and `GossipReceiver` were dropped). Reduced unnecessary DHT writes and reads, adjusted timeouts, **reduced time to bootstrap**. All actor lifecycles are now token-gated, references now work as expected (if all dropped, all background tasks shut down gracefully), resolved bugs in merge workers, and many more improvements. If you find any issues, please report them. **tldr: background tasks shutdown as expected, faster bootstrap time, better all around** @@ -219,7 +219,7 @@ Config::builder() BootstrapConfig::builder() .max_bootstrap_records(5) .publish_record_on_startup(true) - .check_last_minute_record_first_on_startup(false) + .check_older_records_first_on_startup(false) .discovery_poll_interval(Duration::from_millis(2000)) .no_peers_retry_interval(Duration::from_millis(1500)) .per_peer_join_settle_time(Duration::from_millis(100)) @@ -288,7 +288,7 @@ let publisher = RecordPublisher::builder("my-topic", signing_key, b"secret") ## Todo's -- [] Network degradation testing +- [ ] Network degradation testing ## License diff --git a/examples/chat.rs b/examples/chat.rs index fa7d08e..59b6e0f 100644 --- a/examples/chat.rs +++ b/examples/chat.rs @@ -52,7 +52,7 @@ async fn main() -> Result<()> { Config::builder() .bootstrap_config( BootstrapConfig::builder() - .check_last_minute_record_first_on_startup(false) + .check_older_records_first_on_startup(false) .build(), ) .build(), @@ -80,6 +80,7 @@ async fn main() -> Result<()> { println!("\nJoined by {}", &peer.to_string()[0..8]); } } + println!("\nGossip receiver stream ended"); }); // Main input loop for sending messages diff --git a/examples/chat_no_wait.rs b/examples/chat_no_wait.rs index 45f0916..eae92b8 100644 --- a/examples/chat_no_wait.rs +++ b/examples/chat_no_wait.rs @@ -6,9 +6,20 @@ use ed25519_dalek::SigningKey; // Imports from distributed-topic-tracker use distributed_topic_tracker::{AutoDiscoveryGossip, Config, RecordPublisher, TopicId}; +use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<()> { + + tracing_subscriber::fmt() + .with_thread_ids(true) + .with_ansi(true) + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("distributed_topic_tracker=debug")), + ) + .init(); + // Generate a new random secret key let secret_key = SecretKey::generate(&mut rand::rng()); let signing_key = SigningKey::from_bytes(&secret_key.to_bytes()); @@ -58,6 +69,7 @@ async fn main() -> Result<()> { println!("\nJoined by {}", &peer.to_string()[0..8]); } } + println!("\nGossip receiver stream ended"); }); // Main input loop for sending messages diff --git a/examples/e2e_test.rs b/examples/e2e_test.rs index 578e260..a3938b9 100644 --- a/examples/e2e_test.rs +++ b/examples/e2e_test.rs @@ -47,6 +47,7 @@ async fn main() -> Result<()> { while let Ok(event) = gossip_receiver.next().await { println!("event: {event:?}"); } + println!("\nGossip receiver stream ended"); }); tokio::time::sleep(std::time::Duration::from_secs(3)).await; diff --git a/examples/full_config.rs b/examples/full_config.rs index f1147a2..b7ecf6b 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -29,7 +29,7 @@ fn config_builder() -> Config { BootstrapConfig::builder() .max_bootstrap_records(5) .publish_record_on_startup(true) - .check_last_minute_record_first_on_startup(false) + .check_older_records_first_on_startup(false) .discovery_poll_interval(Duration::from_millis(2000)) .no_peers_retry_interval(Duration::from_millis(1500)) .per_peer_join_settle_time(Duration::from_millis(100)) diff --git a/examples/secret_rotation.rs b/examples/secret_rotation.rs index e2263b1..76b4e64 100644 --- a/examples/secret_rotation.rs +++ b/examples/secret_rotation.rs @@ -77,6 +77,7 @@ async fn main() -> Result<()> { println!("\nJoined by {}", &peer.to_string()[0..8]); } } + println!("\nGossip receiver stream ended"); }); // Main input loop for sending messages diff --git a/src/config.rs b/src/config.rs index ae91099..67a0af3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,21 +17,21 @@ impl TimeoutConfig { } /// How long to wait when joining a peer. - /// + /// /// Default: 5s. pub fn join_peer_timeout(&self) -> Duration { self.join_peer_timeout } /// How long to wait when broadcasting messages. - /// + /// /// Default: 5s. pub fn broadcast_timeout(&self) -> Duration { self.broadcast_timeout } /// How long to wait when broadcasting to neighbors. - /// + /// /// Default: 5s. pub fn broadcast_neighbors_timeout(&self) -> Duration { self.broadcast_neighbors_timeout @@ -56,7 +56,7 @@ pub struct TimeoutConfigBuilder { impl TimeoutConfigBuilder { /// How long to wait when joining a peer. - /// + /// /// Default: 5s. pub fn join_peer_timeout(mut self, timeout: Duration) -> Self { self.timeouts.join_peer_timeout = timeout; @@ -64,7 +64,7 @@ impl TimeoutConfigBuilder { } /// How long to wait when broadcasting messages. - /// + /// /// Default: 5s. pub fn broadcast_timeout(mut self, timeout: Duration) -> Self { self.timeouts.broadcast_timeout = timeout; @@ -72,7 +72,7 @@ impl TimeoutConfigBuilder { } /// How long to wait when broadcasting to neighbors. - /// + /// /// Default: 5s. pub fn broadcast_neighbors_timeout(mut self, timeout: Duration) -> Self { self.timeouts.broadcast_neighbors_timeout = timeout; @@ -103,9 +103,9 @@ pub struct DhtConfigBuilder { impl DhtConfigBuilder { /// Number of retries after the initial attempt. - /// + /// /// Total attempts = 1 + retries. - /// + /// /// Default: 3. pub fn retries(mut self, retries: usize) -> Self { self.config.retries = retries; @@ -113,10 +113,10 @@ impl DhtConfigBuilder { } /// Base delay between retries. No-op if `interval` is `Duration::ZERO`. - /// + /// /// If `base_retry_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_retry_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. - /// + /// /// Default: 5s. pub fn base_retry_interval(mut self, interval: Duration) -> Self { if interval > Duration::ZERO { @@ -126,7 +126,7 @@ impl DhtConfigBuilder { } /// Max random jitter added to retry interval. - /// + /// /// Default: 10s. pub fn max_retry_jitter(mut self, jitter: Duration) -> Self { self.config.max_retry_jitter = jitter; @@ -134,7 +134,7 @@ impl DhtConfigBuilder { } /// Timeout for DHT put operations. - /// + /// /// Default: 10s. pub fn put_timeout(mut self, timeout: Duration) -> Self { self.config.put_timeout = timeout; @@ -142,7 +142,7 @@ impl DhtConfigBuilder { } /// Timeout for DHT get operations. - /// + /// /// Default: 10s. pub fn get_timeout(mut self, timeout: Duration) -> Self { self.config.get_timeout = timeout; @@ -164,37 +164,37 @@ impl DhtConfig { } /// Number of retries after the initial attempt. - /// + /// /// Total attempts = 1 + retries. - /// + /// /// Default: 3. pub fn retries(&self) -> usize { self.retries } /// Base delay between retries. - /// + /// /// Default: 5s. pub fn base_retry_interval(&self) -> Duration { self.base_retry_interval } /// Max random jitter added to retry interval. - /// + /// /// Default: 10s. pub fn max_retry_jitter(&self) -> Duration { self.max_retry_jitter } /// Timeout for DHT put operations. - /// + /// /// Default: 10s. pub fn put_timeout(&self) -> Duration { self.put_timeout } /// Timeout for DHT get operations. - /// + /// /// Default: 10s. pub fn get_timeout(&self) -> Duration { self.get_timeout @@ -260,31 +260,31 @@ impl BubbleMergeConfig { } impl BubbleMergeConfigInner { - /// Base interval for bubble merge attempts. - /// + /// Base interval for bubble merge attempts. + /// /// Default: 60s. pub fn base_interval(&self) -> Duration { self.base_interval } - /// Max random jitter added to bubble merge interval. - /// + /// Max random jitter added to bubble merge interval. + /// /// Default: 120s. pub fn max_jitter(&self) -> Duration { self.max_jitter } - /// Minimum number of neighbors required to attempt a bubble merge. - /// + /// Minimum number of neighbors required to attempt a bubble merge. + /// /// Default: 4. pub fn min_neighbors(&self) -> usize { self.min_neighbors } - /// Whether to fail topic creation - /// - /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. - /// + /// Whether to fail topic creation + /// + /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. + /// /// Default: true. pub fn fail_topic_creation_on_merge_startup_failure(&self) -> bool { self.fail_topic_creation_on_merge_startup_failure @@ -293,10 +293,10 @@ impl BubbleMergeConfigInner { impl BubbleMergeConfigBuilder { /// Base interval for bubble merge attempts. No-op if `interval` is `Duration::ZERO`. - /// + /// /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. - /// + /// /// Default: 60s. pub fn base_interval(mut self, interval: Duration) -> Self { if interval > Duration::ZERO { @@ -305,26 +305,26 @@ impl BubbleMergeConfigBuilder { self } - /// Max random jitter added to bubble merge interval. - /// + /// Max random jitter added to bubble merge interval. + /// /// Default: 120s pub fn max_jitter(mut self, jitter: Duration) -> Self { self.config.max_jitter = jitter; self } - /// Minimum number of neighbors required to attempt a bubble merge. - /// + /// Minimum number of neighbors required to attempt a bubble merge. + /// /// Default: 4. pub fn min_neighbors(mut self, min_neighbors: usize) -> Self { self.config.min_neighbors = min_neighbors; self } - /// Whether to fail topic creation - /// - /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. - /// + /// Whether to fail topic creation + /// + /// If a bubble merge startup check fails (ret Err()) or just log and run topic without. + /// /// Default: true. pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { self.config.fail_topic_creation_on_merge_startup_failure = fail; @@ -382,24 +382,24 @@ impl MessageOverlapMergeConfig { } impl MessageOverlapMergeConfigInner { - /// Base interval for message overlap merge attempts. - /// + /// Base interval for message overlap merge attempts. + /// /// Default: 60s. pub fn base_interval(&self) -> Duration { self.base_interval } - /// Max random jitter added to message overlap merge interval. - /// + /// Max random jitter added to message overlap merge interval. + /// /// Default: 120s. pub fn max_jitter(&self) -> Duration { self.max_jitter } - /// Whether to fail topic creation - /// - /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. - /// + /// Whether to fail topic creation + /// + /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. + /// /// Default: true. pub fn fail_topic_creation_on_merge_startup_failure(&self) -> bool { self.fail_topic_creation_on_merge_startup_failure @@ -408,10 +408,10 @@ impl MessageOverlapMergeConfigInner { impl MessageOverlapMergeConfigBuilder { /// Base interval for message overlap merge attempts. No-op if `interval` is `Duration::ZERO`. - /// + /// /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. - /// + /// /// Default: 60s. pub fn base_interval(mut self, interval: Duration) -> Self { if interval > Duration::ZERO { @@ -420,18 +420,18 @@ impl MessageOverlapMergeConfigBuilder { self } - /// Max random jitter added to message overlap merge interval. - /// + /// Max random jitter added to message overlap merge interval. + /// /// Default: 120s. Minimum is 0s. pub fn max_jitter(mut self, jitter: Duration) -> Self { self.config.max_jitter = jitter; self } - /// Whether to fail topic creation - /// - /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. - /// + /// Whether to fail topic creation + /// + /// If a message overlap merge startup check fails (ret Err()) or just log and run topic without. + /// /// Default: true. pub fn fail_topic_creation_on_merge_startup_failure(mut self, fail: bool) -> Self { self.config.fail_topic_creation_on_merge_startup_failure = fail; @@ -491,31 +491,31 @@ impl PublisherConfig { } impl PublisherConfigInner { - /// Initial delay before starting publisher. - /// + /// Initial delay before starting publisher. + /// /// Default: 10s. pub fn initial_delay(&self) -> Duration { self.initial_delay } - /// Base interval for publisher attempts. - /// + /// Base interval for publisher attempts. + /// /// Default: 10s. pub fn base_interval(&self) -> Duration { self.base_interval } - /// Max random jitter added to publisher interval. - /// + /// Max random jitter added to publisher interval. + /// /// Default: 50s. pub fn max_jitter(&self) -> Duration { self.max_jitter } - /// Whether to fail topic creation - /// - /// If a publisher startup check fails (ret Err()) or just log and run topic without. - /// + /// Whether to fail topic creation + /// + /// If a publisher startup check fails (ret Err()) or just log and run topic without. + /// /// Default: true. pub fn fail_topic_creation_on_publishing_startup_failure(&self) -> bool { self.fail_topic_creation_on_publishing_startup_failure @@ -523,8 +523,8 @@ impl PublisherConfigInner { } impl PublisherConfigBuilder { - /// Initial delay before starting publisher. - /// + /// Initial delay before starting publisher. + /// /// Default: 10s. pub fn initial_delay(mut self, delay: Duration) -> Self { self.config.initial_delay = delay; @@ -532,10 +532,10 @@ impl PublisherConfigBuilder { } /// Base interval for publisher attempts. No-op if `interval` is `Duration::ZERO`. - /// + /// /// If `base_interval` is called only once with `Duration::ZERO`, default value prevails. /// If `base_interval` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. - /// + /// /// Default: 10s. pub fn base_interval(mut self, interval: Duration) -> Self { if interval > Duration::ZERO { @@ -544,21 +544,22 @@ impl PublisherConfigBuilder { self } - /// Max random jitter added to publisher interval. - /// + /// Max random jitter added to publisher interval. + /// /// Default: 50s. pub fn max_jitter(mut self, jitter: Duration) -> Self { self.config.max_jitter = jitter; self } - /// Whether to fail topic creation - /// - /// If a publisher startup check fails (ret Err()) or just log and run topic without. - /// + /// Whether to fail topic creation + /// + /// If a publisher startup check fails (ret Err()) or just log and run topic without. + /// /// Default: true. pub fn fail_topic_creation_on_publishing_startup_failure(mut self, fail: bool) -> Self { - self.config.fail_topic_creation_on_publishing_startup_failure = fail; + self.config + .fail_topic_creation_on_publishing_startup_failure = fail; self } @@ -568,8 +569,8 @@ impl PublisherConfigBuilder { } } -/// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records -/// and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes +/// Merge strategies run periodically in the background and attempt to merge split clusters by joining peers in DHT records +/// and message hashes for bubble detection and merging, and by joining peers in DHT records with overlapping message hashes /// for message overlap detection and merging. #[derive(Debug, Clone, Default)] pub struct MergeConfig { @@ -578,6 +579,7 @@ pub struct MergeConfig { } /// Builder for `MergeConfig`. +#[derive(Debug, Clone)] pub struct MergeConfigBuilder { config: MergeConfig, } @@ -590,15 +592,15 @@ impl MergeConfig { } } - /// Bubble merge strategy config. - /// - /// Default: BubbleMergeConfig::default() + /// Bubble merge strategy config. + /// + /// Default: BubbleMergeConfig::default() pub fn bubble_merge(&self) -> &BubbleMergeConfig { &self.bubble_merge } - /// Message overlap merge strategy config. - /// + /// Message overlap merge strategy config. + /// /// Default: MessageOverlapMergeConfig::default() pub fn message_overlap_merge(&self) -> &MessageOverlapMergeConfig { &self.message_overlap_merge @@ -606,18 +608,21 @@ impl MergeConfig { } impl MergeConfigBuilder { - /// Bubble merge strategy config. - /// - /// Default: BubbleMergeConfig::default() + /// Bubble merge strategy config. + /// + /// Default: BubbleMergeConfig::default() pub fn bubble_merge(mut self, bubble_merge: BubbleMergeConfig) -> Self { self.config.bubble_merge = bubble_merge; self } - /// Message overlap merge strategy config. - /// + /// Message overlap merge strategy config. + /// /// Default: MessageOverlapMergeConfig::default() - pub fn message_overlap_merge(mut self, message_overlap_merge: MessageOverlapMergeConfig) -> Self { + pub fn message_overlap_merge( + mut self, + message_overlap_merge: MessageOverlapMergeConfig, + ) -> Self { self.config.message_overlap_merge = message_overlap_merge; self } @@ -637,7 +642,7 @@ pub struct BootstrapConfig { join_confirmation_wait_time: Duration, discovery_poll_interval: Duration, publish_record_on_startup: bool, - check_last_minute_record_first_on_startup: bool, + check_older_records_first_on_startup: bool, } impl Default for BootstrapConfig { @@ -649,7 +654,7 @@ impl Default for BootstrapConfig { join_confirmation_wait_time: Duration::from_millis(500), discovery_poll_interval: Duration::from_millis(2000), publish_record_on_startup: true, - check_last_minute_record_first_on_startup: false, + check_older_records_first_on_startup: false, } } } @@ -661,50 +666,50 @@ pub struct BootstrapConfigBuilder { } impl BootstrapConfigBuilder { - /// Max bootstrap records per topic per minute slot. - /// - /// If zero, we don't publish (PublisherConfig will be set to Disabled). - /// + /// Max bootstrap records per topic per minute slot. + /// + /// If zero, we don't publish (PublisherConfig will be set to Disabled). + /// /// Default: 5. pub fn max_bootstrap_records(mut self, max_records: usize) -> Self { self.config.max_bootstrap_records = max_records; self } - /// How long to wait when no peers are found before retrying. - /// + /// How long to wait when no peers are found before retrying. + /// /// Default: 1500ms. pub fn no_peers_retry_interval(mut self, interval: Duration) -> Self { self.config.no_peers_retry_interval = interval; self } - /// How long to wait after joining a peer before attempting to join another. - /// + /// How long to wait after joining a peer before attempting to join another. + /// /// Default: 100ms. pub fn per_peer_join_settle_time(mut self, interval: Duration) -> Self { self.config.per_peer_join_settle_time = interval; self } - /// How long to wait after joining a peer before checking if joined successfully. - /// + /// How long to wait after joining a peer before checking if joined successfully. + /// /// Default: 500ms. pub fn join_confirmation_wait_time(mut self, interval: Duration) -> Self { self.config.join_confirmation_wait_time = interval; self } - /// How long to wait between DHT discovery attempts. - /// + /// How long to wait between DHT discovery attempts. + /// /// Default: 2000ms. pub fn discovery_poll_interval(mut self, interval: Duration) -> Self { self.config.discovery_poll_interval = interval; self } - /// Whether to publish a bootstrap record unconditionally on startup before dht get. - /// + /// Whether to publish a bootstrap record unconditionally on startup before dht get. + /// /// Default: true. pub fn publish_record_on_startup(mut self, publish: bool) -> Self { self.config.publish_record_on_startup = publish; @@ -712,16 +717,16 @@ impl BootstrapConfigBuilder { } /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. - /// + /// /// If this is enabled, we first fetch `unix_minute-1` and `unix_minute-2`. /// /// If joining longer running, existing topics is priority, set to true. - /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly /// at the same time into a topic without peers), set to false. - /// + /// /// Default: false. - pub fn check_last_minute_record_first_on_startup(mut self, check: bool) -> Self { - self.config.check_last_minute_record_first_on_startup = check; + pub fn check_older_records_first_on_startup(mut self, check: bool) -> Self { + self.config.check_older_records_first_on_startup = check; self } @@ -739,61 +744,61 @@ impl BootstrapConfig { } } - /// Max bootstrap records per topic per minute slot. - /// - /// If zero, we don't publish (PublisherConfig will be set to Disabled). - /// + /// Max bootstrap records per topic per minute slot. + /// + /// If zero, we don't publish (PublisherConfig will be set to Disabled). + /// /// Default: 5. pub fn max_bootstrap_records(&self) -> usize { self.max_bootstrap_records } - /// How long to wait when no peers are found before retrying. - /// + /// How long to wait when no peers are found before retrying. + /// /// Default: 1500ms. pub fn no_peers_retry_interval(&self) -> Duration { self.no_peers_retry_interval } - /// How long to wait after joining a peer before attempting to join another. - /// + /// How long to wait after joining a peer before attempting to join another. + /// /// Default: 100ms. pub fn per_peer_join_settle_time(&self) -> Duration { self.per_peer_join_settle_time } - /// How long to wait after joining a peer before checking if joined successfully. - /// + /// How long to wait after joining a peer before checking if joined successfully. + /// /// Default: 500ms. pub fn join_confirmation_wait_time(&self) -> Duration { self.join_confirmation_wait_time } - /// How long to wait between DHT discovery attempts. - /// + /// How long to wait between DHT discovery attempts. + /// /// Default: 2000ms. pub fn discovery_poll_interval(&self) -> Duration { self.discovery_poll_interval } - /// Whether to publish a bootstrap record unconditionally on startup before dht get. - /// + /// Whether to publish a bootstrap record unconditionally on startup before dht get. + /// /// Default: true. pub fn publish_record_on_startup(&self) -> bool { self.publish_record_on_startup } /// Whether to check `unix_minute` and `unix_minute-1` or `unix_minute-1` and `unix_minute-2` on startup. - /// - /// If this is enabled than we first fetch `unix_minute-1` and `unix_minute-1`. + /// + /// If this is enabled, we first fetch `unix_minute-1` and `unix_minute-2`. /// /// If joining longer running, existing topics is priority, set to true. - /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly + /// If minimizing bootstrap time for cluster cold starts (2+ nodes starting roughly /// at the same time into a topic without peers), set to false. - /// + /// /// Default: false. - pub fn check_last_minute_record_first_on_startup(&self) -> bool { - self.check_last_minute_record_first_on_startup + pub fn check_older_records_first_on_startup(&self) -> bool { + self.check_older_records_first_on_startup } } @@ -819,44 +824,44 @@ impl Config { } /// Publisher strategy config. - /// + /// /// Default: PublisherConfig::default(). pub fn publisher_config(&self) -> &PublisherConfig { &self.publisher_config } /// DHT operation settings. - /// + /// /// Default: DhtConfig::default(). pub fn dht_config(&self) -> &DhtConfig { &self.dht_config } /// Bootstrap strategy settings. - /// + /// /// Default: BootstrapConfig::default(). pub fn bootstrap_config(&self) -> &BootstrapConfig { &self.bootstrap_config } /// Max peers to join simultaneously. - /// + /// /// Minimum: 1. - /// + /// /// Default: 4. pub fn max_join_peer_count(&self) -> usize { self.max_join_peer_count } /// Timeout settings. - /// + /// /// Default: TimeoutConfig::default(). pub fn timeouts(&self) -> &TimeoutConfig { &self.timeouts } /// Merge strategy settings. - /// + /// /// Default: bubble and overlap merges enabled. pub fn merge_config(&self) -> &MergeConfig { &self.merge_config @@ -884,15 +889,15 @@ pub struct ConfigBuilder { impl ConfigBuilder { /// Merge strategy settings. - /// + /// /// Default: MergeConfig::default(). pub fn merge_config(mut self, merge_config: MergeConfig) -> Self { self.config.merge_config = merge_config; self } - /// Publisher strategy config. - /// + /// Publisher strategy config. + /// /// Default: PublisherConfig::default(). pub fn publisher_config(mut self, publisher_config: PublisherConfig) -> Self { self.config.publisher_config = publisher_config; @@ -900,7 +905,7 @@ impl ConfigBuilder { } /// DHT operation settings. - /// + /// /// Default: DhtConfig::default(). pub fn dht_config(mut self, dht_config: DhtConfig) -> Self { self.config.dht_config = dht_config; @@ -908,7 +913,7 @@ impl ConfigBuilder { } /// Bootstrap strategy settings. - /// + /// /// Default: BootstrapConfig::default(). pub fn bootstrap_config(mut self, bootstrap_config: BootstrapConfig) -> Self { self.config.bootstrap_config = bootstrap_config; @@ -916,9 +921,9 @@ impl ConfigBuilder { } /// Max peers to join simultaneously. - /// + /// /// Minimum: 1. - /// + /// /// Default: 4. pub fn max_join_peer_count(mut self, max_peers: NonZeroU32) -> Self { self.config.max_join_peer_count = max_peers.get() as usize; @@ -926,7 +931,7 @@ impl ConfigBuilder { } /// Timeout settings. - /// + /// /// Default: TimeoutConfig::default(). pub fn timeouts(mut self, timeouts: TimeoutConfig) -> Self { self.config.timeouts = timeouts; @@ -934,12 +939,17 @@ impl ConfigBuilder { } /// Build the `Config`. - /// + /// /// If `max_bootstrap_records` is zero, `PublisherConfig` is set to `Disabled`. pub fn build(self) -> Config { let mut config = self.config; - if config.bootstrap_config.max_bootstrap_records == 0 { + if config.bootstrap_config.max_bootstrap_records == 0 + && matches!(config.publisher_config, PublisherConfig::Enabled(_)) + { // if max_bootstrap_records is zero, we don't publish, so disable publisher to avoid confusion + tracing::warn!( + "Publisher is enabled via PublisherConfig::Enabled(_) but BootstrapConfig.max_bootstrap_records is set to 0 (we effectively never publish). Overriding PublisherConfig to PublisherConfig::Disabled." + ); config.publisher_config = PublisherConfig::Disabled; } diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index 62c415d..a0f7fe4 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -10,4 +10,4 @@ pub use keys::{ DefaultSecretRotation, RotationHandle, SecretRotation, encryption_keypair, salt, signing_keypair, }; -pub use record::{EncryptedRecord, Record, RecordPublisher}; +pub use record::{EncryptedRecord, Record, RecordPublisher, TopicId}; diff --git a/src/crypto/record.rs b/src/crypto/record.rs index e484280..fc82149 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -1,4 +1,7 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + str::FromStr, +}; use anyhow::{Result, bail}; use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; @@ -7,7 +10,72 @@ use ed25519_dalek_hpke::{Ed25519hpkeDecryption, Ed25519hpkeEncryption}; use serde::{Deserialize, Serialize}; use sha2::Digest; -use crate::{Config, TopicId}; +use crate::Config; + +/// Topic identifier derived from a string via SHA512 hashing. +/// +/// Used as the stable identifier for gossip subscriptions and DHT records. +/// +/// # Example +/// +/// ```ignore +/// let topic_id = TopicId::new("chat-room-1".to_string()); +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TopicId([u8; 32]); + +impl FromStr for TopicId { + type Err = anyhow::Error; + + fn from_str(topic_name: &str) -> std::result::Result { + Ok(Self::new(topic_name.to_string())) + } +} + +impl From<&str> for TopicId { + fn from(topic_name: &str) -> Self { + Self::new(topic_name.to_string()) + } +} + +impl From for TopicId { + fn from(topic_name: String) -> Self { + Self::new(topic_name) + } +} +/// Treats `bytes` as a topic *name* and SHA-512 hashes them. +/// For a pre-computed 32-byte hash, use [`TopicId::from_hash`] instead. +impl From> for TopicId { + fn from(topic_name: Vec) -> Self { + Self::new(topic_name) + } +} + +impl TopicId { + /// Create a new topic ID from a string. + /// + /// String is hashed with SHA512; the first 32 bytes produce the identifier. + pub fn new(topic_name: impl Into>) -> Self { + let mut topic_name_hash = sha2::Sha512::new(); + topic_name_hash.update(topic_name.into()); + + Self( + topic_name_hash.finalize()[..32] + .try_into() + .expect("hashing 'topic_name' failed"), + ) + } + + /// Create from a pre-computed 32-byte hash. + pub fn from_hash(bytes: &[u8; 32]) -> Self { + Self(*bytes) + } + + /// Get the hash bytes. + pub fn hash(&self) -> [u8; 32] { + self.0 + } +} /// DHT record encrypted with HPKE. /// @@ -224,14 +292,9 @@ impl RecordPublisher { record: Record, cached_records: Option>, ) -> Result<()> { - let records = if let Some(records) = cached_records { - records - } else { - self.get_records(record.unix_minute()) - .await? - .iter() - .cloned() - .collect::>() + let records = match cached_records { + Some(records) => records, + None => self.get_records(record.unix_minute()).await?, }; tracing::debug!( @@ -361,7 +424,9 @@ impl EncryptedRecord { buf.extend_from_slice(&self.encrypted_decryption_key); if buf.len() > Self::MAX_SIZE { - bail!("EncryptedRecord serialization exceeds maximum size, the max is set generously so this should never happen, if so your code is using it manually"); + bail!( + "EncryptedRecord serialization exceeds maximum size, the max is set generously so this should never happen, if so your code is using it manually" + ); } Ok(buf) } diff --git a/src/dht.rs b/src/dht.rs index 99578d8..ad2ee67 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -154,7 +154,7 @@ impl DhtActor { self.reset().await?; let jitter = if self.config.max_retry_jitter() > Duration::ZERO { - Duration::from_micros(rand::random::() % self.config.max_retry_jitter().as_micros() as u64) + Duration::from_nanos(rand::random::() % self.config.max_retry_jitter().as_nanos() as u64) } else { Duration::ZERO }; diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index 6dfe93b..cef960a 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -87,7 +87,7 @@ impl BubbleMergeActor { tracing::warn!("BubbleMerge: error during merge: {:?}", e); } let jitter = if self.max_jitter > Duration::ZERO { - Duration::from_micros(rand::random::() % self.max_jitter.as_micros() as u64) + Duration::from_nanos(rand::random::() % self.max_jitter.as_nanos() as u64) } else { Duration::ZERO }; @@ -158,18 +158,20 @@ impl BubbleMergeActor { }) .collect::>(); - tracing::debug!( - "BubbleMerge: found {} potential peers to join", - pub_keys.len() - ); if !pub_keys.is_empty() { + tracing::debug!( + "BubbleMerge: found {} potential peers to join", + pub_keys.len() + ); + self.gossip_sender .join_peers( pub_keys.iter().cloned().collect::>(), Some(self.max_join_peers), ) .await?; + tracing::debug!("BubbleMerge: join_peers request sent"); } } else { diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index d6874cb..82e113f 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -78,7 +78,7 @@ impl MessageOverlapMergeActor { tracing::warn!("MessageOverlapMerge: error during merge: {:?}", e); } let jitter = if self.max_jitter > Duration::ZERO { - Duration::from_micros(rand::random::() % self.max_jitter.as_micros() as u64) + Duration::from_nanos(rand::random::() % self.max_jitter.as_nanos() as u64) } else { Duration::ZERO }; @@ -164,21 +164,24 @@ impl MessageOverlapMergeActor { .filter(|pub_key| !active_neighbors.contains(pub_key)) .collect::>(); - tracing::debug!( - "MessageOverlapMerge: attempting to join {} pub_keys with no overlapping messages", - pub_keys.len() - ); - - self.gossip_sender - .join_peers( - pub_keys.iter().cloned().collect::>(), - Some(self.max_join_peers), - ) - .await?; - - tracing::debug!( - "MessageOverlapMerge: join_peers request sent for split-brain recovery" - ); + + if !pub_keys.is_empty() { + tracing::debug!( + "MessageOverlapMerge: attempting to join {} pub_keys with no overlapping messages", + pub_keys.len() + ); + + self.gossip_sender + .join_peers( + pub_keys.iter().cloned().collect::>(), + Some(self.max_join_peers), + ) + .await?; + + tracing::debug!( + "MessageOverlapMerge: join_peers request sent for split-brain recovery" + ); + } } } else { tracing::debug!( diff --git a/src/gossip/mod.rs b/src/gossip/mod.rs index 15c0ea2..043083f 100644 --- a/src/gossip/mod.rs +++ b/src/gossip/mod.rs @@ -13,7 +13,7 @@ pub use merge::{BubbleMerge, MessageOverlapMerge}; pub use receiver::GossipReceiver; pub use sender::GossipSender; use serde::{Deserialize, Serialize}; -pub use topic::{Bootstrap, Publisher, Topic, TopicId}; +pub use topic::{Bootstrap, Publisher, Topic}; use crate::{MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher}; diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index ef7c8bf..22673ad 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -102,12 +102,11 @@ impl BootstrapActor { if self.config.publish_record_on_startup() { let unix_minute = crate::unix_minute(0); tracing::debug!("Bootstrap: initial startup record publish {}", unix_minute); - last_published_unix_minute = - if self.config.check_last_minute_record_first_on_startup() { - 0 - } else { - unix_minute - }; + last_published_unix_minute = if self.config.check_older_records_first_on_startup() { + 0 + } else { + unix_minute + }; let record_creator = record_publisher.clone(); let record_content = GossipRecordContent { active_peers: [[0; 32]; MAX_RECORD_PEERS], @@ -139,26 +138,26 @@ impl BootstrapActor { break; } - // On the first try we check the prev unix minute, after that the current one + let current_unix_minute = crate::unix_minute(0); + let mut use_cached_next = true; - let unix_minute = crate::unix_minute( - if last_published_unix_minute == 0 - && bootstrap_config.check_last_minute_record_first_on_startup() - { - use_cached_next = false; - -1 - } else { - 0 - }, - ); + // last_published_unix_minute == 0 means first run + let unix_minute_offset = if last_published_unix_minute == 0 + && bootstrap_config.check_older_records_first_on_startup() + { + use_cached_next = false; + 1 + } else { + 0 + }; // Unique, verified records for the unix minute let mut records = record_publisher - .get_records(unix_minute - 1) + .get_records(current_unix_minute.saturating_sub(unix_minute_offset+1)) .await .unwrap_or_default(); let current_records = record_publisher - .get_records(unix_minute) + .get_records(current_unix_minute.saturating_sub(unix_minute_offset)) .await .unwrap_or_default(); records.extend(current_records.clone()); @@ -166,18 +165,18 @@ impl BootstrapActor { tracing::debug!( "Bootstrap: fetched {} records for unix_minute {}", records.len(), - unix_minute + current_unix_minute ); // If there are no records, invoke the publish_proc (the publishing procedure) // continue the loop after if records.is_empty() { - if unix_minute != last_published_unix_minute { + if current_unix_minute != last_published_unix_minute { tracing::debug!( "Bootstrap: no records found, publishing own record for unix_minute {}", - unix_minute + current_unix_minute ); - last_published_unix_minute = unix_minute; + last_published_unix_minute = current_unix_minute; let record_creator = record_publisher.clone(); let record_content = GossipRecordContent { active_peers: [[0; 32]; MAX_RECORD_PEERS], @@ -185,7 +184,7 @@ impl BootstrapActor { }; if let Ok(record) = Record::sign( record_publisher.topic_id().hash(), - unix_minute, + current_unix_minute, record_content, record_publisher.signing_key(), ) { @@ -214,7 +213,9 @@ impl BootstrapActor { let bootstrap_nodes = records .iter() .flat_map(|record| { + // records are already filtered by self entry let mut v = vec![record.pub_key()]; + if let Ok(record_content) = record.content::() { for peer in record_content.active_peers { if peer != [0; 32] @@ -324,16 +325,16 @@ impl BootstrapActor { } else { tracing::debug!("Bootstrap: still not joined after attempting all peers"); // If we are not connected: check if we should publish a record this minute - if unix_minute != last_published_unix_minute { + if current_unix_minute != last_published_unix_minute { tracing::debug!( "Bootstrap: publishing fallback record for unix_minute {}", - unix_minute + current_unix_minute ); - last_published_unix_minute = unix_minute; + last_published_unix_minute = current_unix_minute; let record_creator = record_publisher.clone(); if let Ok(record) = Record::sign( record_publisher.topic_id().hash(), - unix_minute, + current_unix_minute, GossipRecordContent { active_peers: [[0; 32]; MAX_RECORD_PEERS], last_message_hashes: [[0; 32]; MAX_MESSAGE_HASHES], diff --git a/src/gossip/topic/mod.rs b/src/gossip/topic/mod.rs index 2037d53..a35faa2 100644 --- a/src/gossip/topic/mod.rs +++ b/src/gossip/topic/mod.rs @@ -7,4 +7,4 @@ mod topic; pub use bootstrap::Bootstrap; pub use publisher::Publisher; -pub use topic::{Topic, TopicId}; +pub use topic::Topic; diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index cc4ad9f..1260306 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -76,7 +76,7 @@ impl PublisherActor { tracing::warn!("Publisher: failed to publish record: {:?}", e); } let jitter = if self.max_jitter > Duration::ZERO { - Duration::from_micros(rand::random::() % self.max_jitter.as_micros() as u64) + Duration::from_nanos(rand::random::() % self.max_jitter.as_nanos() as u64) } else { Duration::ZERO }; diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index c8e95cd..c940542 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -1,9 +1,6 @@ //! Main topic handle combining bootstrap, publishing, and merging. -use std::{ - str::FromStr, - sync::{Arc, Weak}, -}; +use std::sync::{Arc, Weak}; use crate::{ BubbleMergeConfig, Config, GossipSender, MessageOverlapMergeConfig, @@ -15,74 +12,8 @@ use crate::{ }; use actor_helper::{Handle, act, act_ok}; use anyhow::Result; -use sha2::Digest; use tokio_util::sync::CancellationToken; -/// Topic identifier derived from a string via SHA512 hashing. -/// -/// Used as the stable identifier for gossip subscriptions and DHT records. -/// -/// # Example -/// -/// ```ignore -/// let topic_id = TopicId::new("chat-room-1".to_string()); -/// ``` -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct TopicId([u8; 32]); - -impl FromStr for TopicId { - type Err = anyhow::Error; - - fn from_str(topic_name: &str) -> std::result::Result { - Ok(Self::new(topic_name.to_string())) - } -} - -impl From<&str> for TopicId { - fn from(topic_name: &str) -> Self { - Self::new(topic_name.to_string()) - } -} - -impl From for TopicId { - fn from(topic_name: String) -> Self { - Self::new(topic_name) - } -} -/// Treats `bytes` as a topic *name* and SHA-512 hashes them. -/// For a pre-computed 32-byte hash, use [`TopicId::from_hash`] instead. -impl From> for TopicId { - fn from(topic_name: Vec) -> Self { - Self::new(topic_name) - } -} - -impl TopicId { - /// Create a new topic ID from a string. - /// - /// String is hashed with SHA512; the first 32 bytes produce the identifier. - pub fn new(topic_name: impl Into>) -> Self { - let mut topic_name_hash = sha2::Sha512::new(); - topic_name_hash.update(topic_name.into()); - - Self( - topic_name_hash.finalize()[..32] - .try_into() - .expect("hashing 'topic_name' failed"), - ) - } - - /// Create from a pre-computed 32-byte hash. - pub fn from_hash(bytes: &[u8; 32]) -> Self { - Self(*bytes) - } - - /// Get the hash bytes. - pub fn hash(&self) -> [u8; 32] { - self.0 - } -} - /// Handle to a joined gossip topic with auto-discovery. /// /// Manages bootstrap, publishing, bubble detection, and split-brain recovery. @@ -308,7 +239,6 @@ async fn spawn_workers( Ok(()) } else { tracing::warn!("Topic: cancelled before workers could be spawned"); - cancel_token.cancel(); Err(anyhow::anyhow!("cancelled before workers could be spawned")) } } @@ -467,7 +397,7 @@ mod tests { cancel_token.cancel(); - // next() on a receiver that was alive before shutdown must return None, + // next() on a receiver that was alive before shutdown must return ChannelError, // not hang. If the broadcast channel didn't close, this would block forever let result = tokio::time::timeout(std::time::Duration::from_secs(5), survivor.next()) .await diff --git a/src/lib.rs b/src/lib.rs index ea6f69e..e2bfcd6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ mod gossip; #[cfg(feature = "iroh-gossip")] pub use gossip::{ AutoDiscoveryGossip, Bootstrap, BubbleMerge, GossipReceiver, GossipRecordContent, GossipSender, - MessageOverlapMerge, Publisher, Topic, TopicId, + MessageOverlapMerge, Publisher, Topic, }; pub use config::{ @@ -19,7 +19,7 @@ pub use config::{ }; pub use crypto::{ DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RotationHandle, - SecretRotation, encryption_keypair, salt, signing_keypair, + SecretRotation, TopicId, encryption_keypair, salt, signing_keypair, }; pub use dht::Dht; From 630e8a15f517210aa4564dc68f129996c8ca6ba6 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 20:27:12 +0200 Subject: [PATCH 27/30] chore: review fixes --- examples/chat_no_wait.rs | 8 ++++++-- src/config.rs | 18 ++++++++++++++---- src/crypto/record.rs | 2 +- src/gossip/topic/topic.rs | 2 +- src/lib.rs | 8 +++++--- 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/examples/chat_no_wait.rs b/examples/chat_no_wait.rs index eae92b8..e599c49 100644 --- a/examples/chat_no_wait.rs +++ b/examples/chat_no_wait.rs @@ -10,7 +10,6 @@ use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() -> Result<()> { - tracing_subscriber::fmt() .with_thread_ids(true) .with_ansi(true) @@ -79,7 +78,12 @@ async fn main() -> Result<()> { print!("\n> "); stdin.read_line(&mut buffer).unwrap(); gossip_sender - .broadcast(buffer.clone().replace("\n", "").into()) + .broadcast( + buffer + .trim_end_matches(&['\r', '\n'][..]) + .as_bytes() + .to_vec(), + ) .await .unwrap(); println!(" - (sent)"); diff --git a/src/config.rs b/src/config.rs index 67a0af3..b930ba9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -133,19 +133,29 @@ impl DhtConfigBuilder { self } - /// Timeout for DHT put operations. + /// Timeout for DHT put operations. No-op if `timeout` is `Duration::ZERO`. + /// + /// If `put_timeout` is called only once with `Duration::ZERO`, default value prevails. + /// If `put_timeout` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. /// /// Default: 10s. pub fn put_timeout(mut self, timeout: Duration) -> Self { - self.config.put_timeout = timeout; + if timeout > Duration::ZERO { + self.config.put_timeout = timeout; + } self } - /// Timeout for DHT get operations. + /// Timeout for DHT get operations. No-op if `timeout` is `Duration::ZERO`. + /// + /// If `get_timeout` is called only once with `Duration::ZERO`, default value prevails. + /// If `get_timeout` is first called with a > `Duration::ZERO`, and then again with `Duration::ZERO`, the first set value is kept. /// /// Default: 10s. pub fn get_timeout(mut self, timeout: Duration) -> Self { - self.config.get_timeout = timeout; + if timeout > Duration::ZERO { + self.config.get_timeout = timeout; + } self } diff --git a/src/crypto/record.rs b/src/crypto/record.rs index fc82149..e6bd797 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -372,7 +372,7 @@ impl RecordPublisher { ); let mut dedubed_records = HashMap::new(); - for item in records_iter.clone() { + for item in records_iter { if let Ok(encrypted_record) = EncryptedRecord::from_bytes(item.value().to_vec()) && let Ok(record) = encrypted_record.decrypt(&encryption_key) && record.verify(&self.topic_id.hash(), unix_minute).is_ok() diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index c940542..b54df3c 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -439,7 +439,7 @@ mod tests { .secret_key(secret_key.clone()) .bind() .await - .expect("failed to bind edpoint"); + .expect("failed to bind endpoint"); let gossip = iroh_gossip::net::Gossip::builder().spawn(endpoint.clone()); let topic_id = crate::TopicId::new("my-iroh-gossip-topic".to_string()); diff --git a/src/lib.rs b/src/lib.rs index e2bfcd6..bd76e6d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,9 +13,10 @@ pub use gossip::{ }; pub use config::{ - BootstrapConfig, BubbleMergeConfig, BubbleMergeConfigBuilder, Config, ConfigBuilder, DhtConfig, - MergeConfig, MessageOverlapMergeConfig, MessageOverlapMergeConfigBuilder, PublisherConfig, - PublisherConfigBuilder, TimeoutConfig, TimeoutConfigBuilder, + BootstrapConfig, BootstrapConfigBuilder, BubbleMergeConfig, BubbleMergeConfigBuilder, Config, + ConfigBuilder, DhtConfig, DhtConfigBuilder, MergeConfig, MergeConfigBuilder, + MessageOverlapMergeConfig, MessageOverlapMergeConfigBuilder, PublisherConfig, + PublisherConfigBuilder, TimeoutConfig, TimeoutConfigBuilder, }; pub use crypto::{ DefaultSecretRotation, EncryptedRecord, Record, RecordPublisher, RotationHandle, @@ -23,6 +24,7 @@ pub use crypto::{ }; pub use dht::Dht; +/// These are part of the on-wire format: DO NOT CHANGE without increasing protocol version. pub const MAX_RECORD_PEERS: usize = 5; pub const MAX_MESSAGE_HASHES: usize = 5; From 6d825849688c574e46cb1f0839f308d238dcf233 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 21:53:44 +0200 Subject: [PATCH 28/30] add: cancelation token to get/put record + minor review docs fixes --- ARCHITECTURE.md | 2 +- PROTOCOL.md | 2 +- src/crypto/record.rs | 206 ++++++++++++++++------------ src/gossip/merge/bubble.rs | 6 +- src/gossip/merge/message_overlap.rs | 6 +- src/gossip/topic/bootstrap.rs | 23 +++- src/gossip/topic/publisher.rs | 2 +- src/gossip/topic/topic.rs | 6 +- 8 files changed, 146 insertions(+), 107 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0577e63..75021da 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -109,7 +109,7 @@ sequenceDiagram ``` Key points: -- First iteration: optionally also check previous unix minute (`check_last_minute_record_first_on_startup`). +- First iteration: optionally check older records first (`check_older_records_first_on_startup`). - Both `unix_minute` and `unix_minute - 1` records are always fetched. - Pacing avoids bursts and "bubbles." - Keep trying until joined. diff --git a/PROTOCOL.md b/PROTOCOL.md index bb59641..3a80d19 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -94,7 +94,7 @@ The bootstrap procedure is a continuous loop that attempts to discover and conne - Final join confirmation wait: 500ms (default, configurable via `BootstrapConfig`) - Discovery poll interval: 2000ms (default, configurable via `BootstrapConfig`) - Publish on startup: true (default, configurable via `BootstrapConfig`) -- Check last minute record first on startup: false (default, configurable via `BootstrapConfig`) +- Check older records first on startup (`check_older_records_first_on_startup`): false (default, configurable via `BootstrapConfig`) ### Bootstrap Loop diff --git a/src/crypto/record.rs b/src/crypto/record.rs index e6bd797..23140bc 100644 --- a/src/crypto/record.rs +++ b/src/crypto/record.rs @@ -9,6 +9,7 @@ use ed25519_dalek::{Signer, SigningKey, VerifyingKey}; use ed25519_dalek_hpke::{Ed25519hpkeDecryption, Ed25519hpkeEncryption}; use serde::{Deserialize, Serialize}; use sha2::Digest; +use tokio_util::sync::CancellationToken; use crate::Config; @@ -279,8 +280,8 @@ impl RecordPublisher { /// /// Checks existing record count for this time slot and skips publishing if /// `self.config.bootstrap_config().max_bootstrap_records()` limit reached. - pub async fn publish_record(&self, record: Record) -> Result<()> { - self.publish_record_cached_records(record, None).await + pub async fn publish_record(&self, record: Record, cancel_token: CancellationToken) -> Result<()> { + self.publish_record_cached_records(record, None, cancel_token).await } /// Publish a record to the DHT (using cached get_records) if slot capacity allows. @@ -291,111 +292,134 @@ impl RecordPublisher { &self, record: Record, cached_records: Option>, + cancel_token: CancellationToken, ) -> Result<()> { - let records = match cached_records { - Some(records) => records, - None => self.get_records(record.unix_minute()).await?, - }; + let publish_fut = async { + let records = match cached_records { + Some(records) => records, + None => self.get_records(record.unix_minute(), cancel_token.clone()).await?, + }; + + tracing::debug!( + "RecordPublisher: found {} existing records for unix_minute {}", + records.len(), + record.unix_minute() + ); - tracing::debug!( - "RecordPublisher: found {} existing records for unix_minute {}", - records.len(), - record.unix_minute() - ); + if records.len() >= self.config.bootstrap_config().max_bootstrap_records() { + tracing::debug!( + "RecordPublisher: max records reached ({}), skipping publish", + self.config.bootstrap_config().max_bootstrap_records() + ); + return Ok(()); + } + + // Publish own records + let sign_key = crate::crypto::keys::signing_keypair(self.topic_id(), record.unix_minute); + let salt = crate::crypto::keys::salt(self.topic_id(), record.unix_minute); + let encryption_key = crate::crypto::keys::encryption_keypair( + self.topic_id(), + &self.secret_rotation.clone().unwrap_or_default(), + self.initial_secret_hash, + record.unix_minute, + ); + let encrypted_record = record.encrypt(&encryption_key); + let next_seq_num = i64::MAX; - if records.len() >= self.config.bootstrap_config().max_bootstrap_records() { tracing::debug!( - "RecordPublisher: max records reached ({}), skipping publish", - self.config.bootstrap_config().max_bootstrap_records() + "RecordPublisher: publishing record to DHT for unix_minute {}", + record.unix_minute() ); - return Ok(()); - } - // Publish own records - let sign_key = crate::crypto::keys::signing_keypair(self.topic_id(), record.unix_minute); - let salt = crate::crypto::keys::salt(self.topic_id(), record.unix_minute); - let encryption_key = crate::crypto::keys::encryption_keypair( - self.topic_id(), - &self.secret_rotation.clone().unwrap_or_default(), - self.initial_secret_hash, - record.unix_minute, - ); - let encrypted_record = record.encrypt(&encryption_key); - let next_seq_num = i64::MAX; - - tracing::debug!( - "RecordPublisher: publishing record to DHT for unix_minute {}", - record.unix_minute() - ); - - self.dht - .put_mutable( - sign_key.clone(), - Some(salt.to_vec()), - encrypted_record.to_bytes()?, - next_seq_num, - ) - .await?; - - tracing::debug!("RecordPublisher: successfully published to DHT"); - Ok(()) + self.dht + .put_mutable( + sign_key.clone(), + Some(salt.to_vec()), + encrypted_record.to_bytes()?, + next_seq_num, + ) + .await?; + + tracing::debug!("RecordPublisher: successfully published to DHT"); + Ok(()) + }; + + tokio::select! { + _ = cancel_token.cancelled() => { + anyhow::bail!("publish cancelled"); + } + res = publish_fut => { + res + } + } } /// Retrieve all verified records for a given time slot from the DHT. /// /// Filters out records from this publisher's own node ID. /// Dedup's records based on pub_key, keeping the highest sequence number per pub_key. - pub async fn get_records(&self, unix_minute: u64) -> Result> { - tracing::debug!( - "RecordPublisher: fetching records from DHT for unix_minute {}", - unix_minute - ); - - let topic_sign = crate::crypto::keys::signing_keypair(self.topic_id(), unix_minute); - let encryption_key = crate::crypto::keys::encryption_keypair( - self.topic_id(), - &self.secret_rotation.clone().unwrap_or_default(), - self.initial_secret_hash, - unix_minute, - ); - let salt = crate::crypto::keys::salt(self.topic_id(), unix_minute); - - // Get records, decrypt and verify - let records_iter = self - .dht - .get(topic_sign.verifying_key(), Some(salt.to_vec()), None) - .await?; - - tracing::debug!( - "RecordPublisher: received {} raw records from DHT", - records_iter.len() - ); - - let mut dedubed_records = HashMap::new(); - for item in records_iter { - if let Ok(encrypted_record) = EncryptedRecord::from_bytes(item.value().to_vec()) - && let Ok(record) = encrypted_record.decrypt(&encryption_key) - && record.verify(&self.topic_id.hash(), unix_minute).is_ok() - && !record.pub_key().eq(self.pub_key.as_bytes()) - { - let pub_key = record.pub_key(); - match dedubed_records.get(&pub_key) { - Some((seq, _)) if *seq >= item.seq() => {} - _ => { - dedubed_records.insert(pub_key, (item.seq(), record)); + pub async fn get_records(&self, unix_minute: u64, cancel_token: CancellationToken) -> Result> { + let get_fut = async { + tracing::debug!( + "RecordPublisher: fetching records from DHT for unix_minute {}", + unix_minute + ); + + let topic_sign = crate::crypto::keys::signing_keypair(self.topic_id(), unix_minute); + let encryption_key = crate::crypto::keys::encryption_keypair( + self.topic_id(), + &self.secret_rotation.clone().unwrap_or_default(), + self.initial_secret_hash, + unix_minute, + ); + let salt = crate::crypto::keys::salt(self.topic_id(), unix_minute); + + // Get records, decrypt and verify + let records_iter = self + .dht + .get(topic_sign.verifying_key(), Some(salt.to_vec()), None) + .await?; + + tracing::debug!( + "RecordPublisher: received {} raw records from DHT", + records_iter.len() + ); + + let mut dedubed_records = HashMap::new(); + for item in records_iter { + if let Ok(encrypted_record) = EncryptedRecord::from_bytes(item.value().to_vec()) + && let Ok(record) = encrypted_record.decrypt(&encryption_key) + && record.verify(&self.topic_id.hash(), unix_minute).is_ok() + && !record.pub_key().eq(self.pub_key.as_bytes()) + { + let pub_key = record.pub_key(); + match dedubed_records.get(&pub_key) { + Some((seq, _)) if *seq >= item.seq() => {} + _ => { + dedubed_records.insert(pub_key, (item.seq(), record)); + } } } } + tracing::debug!( + "RecordPublisher: verified {} records (filtered self)", + dedubed_records.len() + ); + + Ok(dedubed_records + .into_values() + .map(|(_, record)| record) + .collect::>()) + }; + + tokio::select! { + _ = cancel_token.cancelled() => { + anyhow::bail!("get_records cancelled"); + } + res = get_fut => { + res + } } - tracing::debug!( - "RecordPublisher: verified {} records (filtered self)", - dedubed_records.len() - ); - - Ok(dedubed_records - .into_values() - .map(|(_, record)| record) - .collect::>()) } } diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index cef960a..deb5370 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -108,8 +108,8 @@ impl BubbleMergeActor { // Cluster size as bubble indicator async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1).await?; - records.extend(self.record_publisher.get_records(unix_minute).await?); + let mut records = self.record_publisher.get_records(unix_minute - 1, self.cancel_token.clone()).await?; + records.extend(self.record_publisher.get_records(unix_minute, self.cancel_token.clone()).await?); let neighbors = self.gossip_receiver.neighbors().await?; tracing::debug!( @@ -171,7 +171,7 @@ impl BubbleMergeActor { Some(self.max_join_peers), ) .await?; - + tracing::debug!("BubbleMerge: join_peers request sent"); } } else { diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 82e113f..65d6ebb 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -98,8 +98,8 @@ impl MessageOverlapMergeActor { impl MessageOverlapMergeActor { async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1).await?; - records.extend(self.record_publisher.get_records(unix_minute).await?); + let mut records = self.record_publisher.get_records(unix_minute - 1, self.cancel_token.clone()).await?; + records.extend(self.record_publisher.get_records(unix_minute, self.cancel_token.clone()).await?); let local_hashes = self.gossip_receiver.last_message_hashes().await?; tracing::debug!( @@ -170,7 +170,7 @@ impl MessageOverlapMergeActor { "MessageOverlapMerge: attempting to join {} pub_keys with no overlapping messages", pub_keys.len() ); - + self.gossip_sender .join_peers( pub_keys.iter().cloned().collect::>(), diff --git a/src/gossip/topic/bootstrap.rs b/src/gossip/topic/bootstrap.rs index 22673ad..62f7b82 100644 --- a/src/gossip/topic/bootstrap.rs +++ b/src/gossip/topic/bootstrap.rs @@ -6,6 +6,7 @@ use actor_helper::{Handle, act, act_ok}; use anyhow::Result; use iroh::EndpointId; use tokio::time::sleep; +use tokio_util::sync::CancellationToken; use crate::{ GossipSender, MAX_MESSAGE_HASHES, MAX_RECORD_PEERS, RecordPublisher, @@ -118,7 +119,12 @@ impl BootstrapActor { record_content, record_publisher.signing_key(), ) { - publish_record_fire_and_forget(record_creator, record, None); + publish_record_fire_and_forget( + record_creator, + record, + None, + cancel_token.clone(), + ); } } @@ -153,11 +159,17 @@ impl BootstrapActor { // Unique, verified records for the unix minute let mut records = record_publisher - .get_records(current_unix_minute.saturating_sub(unix_minute_offset+1)) + .get_records( + current_unix_minute.saturating_sub(unix_minute_offset + 1), + cancel_token.clone(), + ) .await .unwrap_or_default(); let current_records = record_publisher - .get_records(current_unix_minute.saturating_sub(unix_minute_offset)) + .get_records( + current_unix_minute.saturating_sub(unix_minute_offset), + cancel_token.clone(), + ) .await .unwrap_or_default(); records.extend(current_records.clone()); @@ -196,6 +208,7 @@ impl BootstrapActor { } else { None }, + cancel_token.clone(), ); } } @@ -349,6 +362,7 @@ impl BootstrapActor { } else { None }, + cancel_token.clone(), ); } } @@ -379,10 +393,11 @@ fn publish_record_fire_and_forget( record_publisher: RecordPublisher, record: Record, cached_records: Option>, + cancel_token: CancellationToken, ) { tokio::spawn(async move { if let Err(err) = record_publisher - .publish_record_cached_records(record, cached_records) + .publish_record_cached_records(record, cached_records, cancel_token) .await { tracing::warn!("Failed to publish record: {:?}", err); diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index 1260306..caf7942 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -142,7 +142,7 @@ impl PublisherActor { .new_record(unix_minute, record_content); tracing::debug!("Publisher: created new record: {:?}", res); let record = res?; - let result = self.record_publisher.publish_record(record).await; + let result = self.record_publisher.publish_record(record, self.cancel_token.clone()).await; if let Err(ref e) = result { tracing::debug!("Publisher: failed to publish record: {:?}", e); diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index b54df3c..377ddf4 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -177,7 +177,7 @@ async fn spawn_workers( cancel_token: CancellationToken, ) -> Result<()> { if !cancel_token.is_cancelled() { - if matches!(config.publisher_config(), PublisherConfig::Enabled { .. }) { + if matches!(config.publisher_config(), PublisherConfig::Enabled(_)) { tracing::debug!("Topic: starting publisher"); match api.upgrade() { Some(api) => { @@ -195,7 +195,7 @@ async fn spawn_workers( if matches!( config.merge_config().bubble_merge(), - BubbleMergeConfig::Enabled { .. } + BubbleMergeConfig::Enabled(_) ) { tracing::debug!("Topic: starting bubble merge"); match api.upgrade() { @@ -214,7 +214,7 @@ async fn spawn_workers( if matches!( config.merge_config().message_overlap_merge(), - MessageOverlapMergeConfig::Enabled { .. } + MessageOverlapMergeConfig::Enabled(_) ) { tracing::debug!("Topic: starting message overlap merge"); match api.upgrade() { From a928778ba4248b179007e925bd2256f6ef943de8 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 23:36:13 +0200 Subject: [PATCH 29/30] chore: streamlined > 0 enforcement, removed ZeroU32 --- README.md | 4 ++-- examples/full_config.rs | 2 +- src/config.rs | 25 +++++++++++++++++-------- src/gossip/topic/topic.rs | 4 ++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4886c73..ab6821f 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ let publisher = RecordPublisher::new(topic, signing_key, None, secret, Config::d let publisher = RecordPublisher::builder(topic_id, signing_key, initial_secret) .config( Config::builder() - .max_join_peer_count(std::num::NonZeroU32::new(8).expect("must be > 0")) + .max_join_peer_count(4) .build(), ) .build(); @@ -226,7 +226,7 @@ Config::builder() .join_confirmation_wait_time(Duration::from_millis(500)) .build(), ) - .max_join_peer_count(std::num::NonZeroU32::new(4).expect("must be > 0")) + .max_join_peer_count(4) .publisher_config( PublisherConfig::builder() .initial_delay(Duration::from_secs(10)) diff --git a/examples/full_config.rs b/examples/full_config.rs index b7ecf6b..0bf54a8 100644 --- a/examples/full_config.rs +++ b/examples/full_config.rs @@ -36,7 +36,7 @@ fn config_builder() -> Config { .join_confirmation_wait_time(Duration::from_millis(500)) .build(), ) - .max_join_peer_count(std::num::NonZeroU32::new(4).expect("must be > 0")) + .max_join_peer_count(4) .publisher_config( PublisherConfig::builder() .initial_delay(Duration::from_secs(10)) diff --git a/src/config.rs b/src/config.rs index b930ba9..671fe91 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,4 @@ -use std::{num::NonZeroU32, time::Duration}; +use std::time::Duration; /// Timeout settings for gossip operations. #[derive(Debug, Clone)] @@ -271,6 +271,8 @@ impl BubbleMergeConfig { impl BubbleMergeConfigInner { /// Base interval for bubble merge attempts. + /// + /// `base_interval` > Duration::ZERO /// /// Default: 60s. pub fn base_interval(&self) -> Duration { @@ -394,6 +396,8 @@ impl MessageOverlapMergeConfig { impl MessageOverlapMergeConfigInner { /// Base interval for message overlap merge attempts. /// + /// `base_interval` > Duration::ZERO + /// /// Default: 60s. pub fn base_interval(&self) -> Duration { self.base_interval @@ -510,6 +514,8 @@ impl PublisherConfigInner { /// Base interval for publisher attempts. /// + /// `base_interval` > Duration::ZERO + /// /// Default: 10s. pub fn base_interval(&self) -> Duration { self.base_interval @@ -854,9 +860,9 @@ impl Config { &self.bootstrap_config } - /// Max peers to join simultaneously. - /// - /// Minimum: 1. + /// Max peers to join simultaneously. + /// + /// Minimum is 1. /// /// Default: 4. pub fn max_join_peer_count(&self) -> usize { @@ -930,13 +936,16 @@ impl ConfigBuilder { self } - /// Max peers to join simultaneously. + /// Max peers to join simultaneously. No-op if `max_join_peer_count` is zero. /// - /// Minimum: 1. + /// If `max_join_peer_count` is called only once with zero, default value prevails. + /// If `max_join_peer_count` is first called with a non-zero value, and then again with zero, the first set value is kept. /// /// Default: 4. - pub fn max_join_peer_count(mut self, max_peers: NonZeroU32) -> Self { - self.config.max_join_peer_count = max_peers.get() as usize; + pub fn max_join_peer_count(mut self, max_peers: usize) -> Self { + if max_peers > 0 { + self.config.max_join_peer_count = max_peers; + } self } diff --git a/src/gossip/topic/topic.rs b/src/gossip/topic/topic.rs index 377ddf4..4daab69 100644 --- a/src/gossip/topic/topic.rs +++ b/src/gossip/topic/topic.rs @@ -290,7 +290,7 @@ impl TopicActor { self.bootstrap.gossip_sender().await?, self.bootstrap.gossip_receiver().await?, self.cancel_token.clone(), - self.record_publisher.config().max_join_peer_count().max(1), + self.record_publisher.config().max_join_peer_count(), config.base_interval(), config.max_jitter(), config.min_neighbors(), @@ -331,7 +331,7 @@ impl TopicActor { self.bootstrap.gossip_sender().await?, self.bootstrap.gossip_receiver().await?, self.cancel_token.clone(), - self.record_publisher.config().max_join_peer_count().max(1), + self.record_publisher.config().max_join_peer_count(), config.base_interval(), config.max_jitter(), ) From e29f18a20887cb971b51d8257c0799c9e84b1666 Mon Sep 17 00:00:00 2001 From: rustonbsd Date: Sun, 19 Apr 2026 23:40:06 +0200 Subject: [PATCH 30/30] chore: interval jitter calculation now uses native u128 --- src/dht.rs | 4 +++- src/gossip/merge/bubble.rs | 14 ++++++++++---- src/gossip/merge/message_overlap.rs | 14 ++++++++++---- src/gossip/topic/publisher.rs | 7 +++++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/dht.rs b/src/dht.rs index ad2ee67..5b6c91c 100644 --- a/src/dht.rs +++ b/src/dht.rs @@ -154,7 +154,9 @@ impl DhtActor { self.reset().await?; let jitter = if self.config.max_retry_jitter() > Duration::ZERO { - Duration::from_nanos(rand::random::() % self.config.max_retry_jitter().as_nanos() as u64) + Duration::from_nanos( + (rand::random::() % self.config.max_retry_jitter().as_nanos()) as u64, + ) } else { Duration::ZERO }; diff --git a/src/gossip/merge/bubble.rs b/src/gossip/merge/bubble.rs index deb5370..d014354 100644 --- a/src/gossip/merge/bubble.rs +++ b/src/gossip/merge/bubble.rs @@ -87,7 +87,7 @@ impl BubbleMergeActor { tracing::warn!("BubbleMerge: error during merge: {:?}", e); } let jitter = if self.max_jitter > Duration::ZERO { - Duration::from_nanos(rand::random::() % self.max_jitter.as_nanos() as u64) + Duration::from_nanos((rand::random::() % self.max_jitter.as_nanos()) as u64) } else { Duration::ZERO }; @@ -108,8 +108,15 @@ impl BubbleMergeActor { // Cluster size as bubble indicator async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1, self.cancel_token.clone()).await?; - records.extend(self.record_publisher.get_records(unix_minute, self.cancel_token.clone()).await?); + let mut records = self + .record_publisher + .get_records(unix_minute - 1, self.cancel_token.clone()) + .await?; + records.extend( + self.record_publisher + .get_records(unix_minute, self.cancel_token.clone()) + .await?, + ); let neighbors = self.gossip_receiver.neighbors().await?; tracing::debug!( @@ -158,7 +165,6 @@ impl BubbleMergeActor { }) .collect::>(); - if !pub_keys.is_empty() { tracing::debug!( "BubbleMerge: found {} potential peers to join", diff --git a/src/gossip/merge/message_overlap.rs b/src/gossip/merge/message_overlap.rs index 65d6ebb..e9e3eb0 100644 --- a/src/gossip/merge/message_overlap.rs +++ b/src/gossip/merge/message_overlap.rs @@ -78,7 +78,7 @@ impl MessageOverlapMergeActor { tracing::warn!("MessageOverlapMerge: error during merge: {:?}", e); } let jitter = if self.max_jitter > Duration::ZERO { - Duration::from_nanos(rand::random::() % self.max_jitter.as_nanos() as u64) + Duration::from_nanos((rand::random::() % self.max_jitter.as_nanos()) as u64) } else { Duration::ZERO }; @@ -98,8 +98,15 @@ impl MessageOverlapMergeActor { impl MessageOverlapMergeActor { async fn merge(&mut self) -> Result<()> { let unix_minute = crate::unix_minute(0); - let mut records = self.record_publisher.get_records(unix_minute - 1, self.cancel_token.clone()).await?; - records.extend(self.record_publisher.get_records(unix_minute, self.cancel_token.clone()).await?); + let mut records = self + .record_publisher + .get_records(unix_minute - 1, self.cancel_token.clone()) + .await?; + records.extend( + self.record_publisher + .get_records(unix_minute, self.cancel_token.clone()) + .await?, + ); let local_hashes = self.gossip_receiver.last_message_hashes().await?; tracing::debug!( @@ -164,7 +171,6 @@ impl MessageOverlapMergeActor { .filter(|pub_key| !active_neighbors.contains(pub_key)) .collect::>(); - if !pub_keys.is_empty() { tracing::debug!( "MessageOverlapMerge: attempting to join {} pub_keys with no overlapping messages", diff --git a/src/gossip/topic/publisher.rs b/src/gossip/topic/publisher.rs index caf7942..621dc8e 100644 --- a/src/gossip/topic/publisher.rs +++ b/src/gossip/topic/publisher.rs @@ -76,7 +76,7 @@ impl PublisherActor { tracing::warn!("Publisher: failed to publish record: {:?}", e); } let jitter = if self.max_jitter > Duration::ZERO { - Duration::from_nanos(rand::random::() % self.max_jitter.as_nanos() as u64) + Duration::from_nanos((rand::random::() % self.max_jitter.as_nanos()) as u64) } else { Duration::ZERO }; @@ -142,7 +142,10 @@ impl PublisherActor { .new_record(unix_minute, record_content); tracing::debug!("Publisher: created new record: {:?}", res); let record = res?; - let result = self.record_publisher.publish_record(record, self.cancel_token.clone()).await; + let result = self + .record_publisher + .publish_record(record, self.cancel_token.clone()) + .await; if let Err(ref e) = result { tracing::debug!("Publisher: failed to publish record: {:?}", e);